web_stack_test.py 260 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163
  1. #!/usr/bin/env python3
  2. """Self-contained web-stack feature tests: app + fake saddle daemon.
  3. Unlike integration_test.py (which needs a real saddle daemon and a mock
  4. LLM), this suite spawns everything it needs:
  5. - the clanker-saddle app binary (path from argv[1], or builddir default)
  6. in a scratch directory with its own web-config.json (statum keys copied
  7. from the repo, plus `saddle_callback_base_url`) and SQLite database
  8. - in-process fake saddle daemons speaking the wire contract: POST
  9. /register (plan 10: receives {saddle_token, web_base_url}, answers
  10. {ok, ssh_public_key, version}), POST /wake (records the wake; the
  11. daemon then asks POST /api/saddle/next-task), POST /task-update
  12. (cancel/comment/subtask-status), GET /status (scripted reflection —
  13. exceptional use only), and the phase-2 push channels home:
  14. POST /api/saddle/chunks (raw PTY batches) and POST /api/saddle/results
  15. (round entries + state transitions). The retired /start, /output and
  16. /log routes 404 like a post-phase-2 daemon.
  17. It then drives the app the way statum.js does (entrypoint loads + action
  18. POSTs) and asserts the Workstream B features against the wire contract:
  19. 3. plan-review blocks hide their duplicate body preview
  20. 5. rules CRUD/attach + the effective `rules` array in assignments
  21. 6. project notes, POST /api/saddle/note, `project_notes` + `callback`
  22. in assignments
  23. plus the raw-log side: pushed chunks -> session_raw_log + terminal
  24. raw view
  25. Plan 10 phase 1 (work acquisition flip):
  26. - per-saddle tokens: registration mints + /register-delivers them,
  27. legacy tokens migrate onto the row at startup, removal revokes
  28. - token auth on every /api/saddle/* endpoint
  29. - next-task allocation: concurrent asks race exactly one winner,
  30. affinity pinning for parked tasks, resume_notice on suspended resumes
  31. - dispatcher: wake -> ask -> allocate roundtrip, the lost-wake guard
  32. (no ask after a wake -> offline, re-wake the next capable), and the
  33. pre-flip /answer + /start resume paths kept as the rollback route
  34. Plan 10 phase 2 (telemetry flip):
  35. - pushed chunks: idempotent on (session_id, seq), ordered, landing in
  36. session_raw_log so the SSE terminal keeps working
  37. - pushed results: entries store like the /output path did (redelivery
  38. idempotent on the daemon entry ids), blocked raises the block row +
  39. parks + bumps the pulse (sealed refresh shows it), done/failed
  40. complete/fail the task, suspended parks for the affinity resume
  41. - answer resumes, both daemon shapes: a SLEEPING affine saddle (its
  42. post-park ask returned 204) is woken, its ask allocating the resume
  43. with the verdict riding the assignment; a daemon parked MID-ROUND
  44. (its round holds the block's answer wait open, so it can never ask)
  45. gets the verdict delivered into that wait via POST /answer — same
  46. session, no wake. A daemon dying with a parked-with-answered task
  47. never requeues it: the verdict and session survive for the resume
  48. - staleness: a silent runner gets ONE exceptional GET /status probe —
  49. alive keeps it running, dead requeues the task with a bot comment and
  50. marks the saddle offline; a recovered daemon's ask flips offline rows
  51. back to idle
  52. - the deleted layers are gone: /output, /log and /start 404 on the
  53. daemon side, and nothing on the web side calls them
  54. Plan 10 phase 3 (capability awareness + UI):
  55. - assignment capability context: {saddle_tags, fleet_tags} on every
  56. payload, GET /api/saddle/capabilities (per-saddle token) feeding
  57. the daemon's `saddle capabilities` CLI
  58. - subtask callback capabilities: explicit-only attachment validated
  59. against the operator tag set, unknown tags rejected with the valid
  60. list in the error
  61. - task page: status block pinned to the top, blocking sections
  62. newest-first (sealed GET state)
  63. - home dashboard: pinned OPEN blocks as newest-first link-through
  64. cards (no inline answer forms) above the cross-task activity
  65. timeline (last ~50 events newest-first)
  66. - lost-wake-guard regression: a saddle that asks SLOWLY after its
  67. wake (busy winding down) still claims the task — the wake is never
  68. silently dropped, no re-wake storm fires, and the never-asks guard
  69. path (offline + re-wake of the next capable) stays green
  70. Prints PASS/FAIL lines Statum-style; exits non-zero on any failure.
  71. Plan 11 task D+E (UI + notifications), the `push` shard:
  72. - task page: progress feed above the status card, status + description
  73. in a 50/50 grid, edit expander under the history expander, plan
  74. behind an expander, answered blocks behind an expander; queue screen:
  75. New Task expander above the filters; dashboard waiting-on-drafts
  76. block with links to where Enqueue lives (pages shard)
  77. - real Web Push over VAPID: keypair generated at startup into
  78. web-config.json, GET /sw.js service worker with click-through,
  79. subscribe/unsubscribe + per-task Notify me actions, block events and
  80. waiting-on-drafts pushed globally, first start/failure/completion to
  81. opted-in users, best-effort queue with dispatcher-tick retry, 410/404
  82. endpoint pruning — payloads verified protocol-level (RFC 8291
  83. decryption + ES256 JWT signature) because the harness owns the
  84. browser-side keys
  85. Heredoc completion reports (pages shard):
  86. - terminal tasks render tasks.summary (set by done/fail) as a
  87. markdown completion-report card below the status/description grid —
  88. "Completion report" when done, "Failure summary" when failed, no
  89. card when empty; the summary renders through the shipped
  90. stm-markdown renderer (headings/code, no syntax leaks, HTML
  91. escaped) and the dashboard timeline snippets its first line
  92. Speed pass (plan 11, task 0):
  93. - SHARDING: the scenario groups are partitioned into seven independent
  94. bundles (pages, dispatch, subtasks, lineage, allocation, lifecycle,
  95. answers), each running in its own process against its own app
  96. instance, fake daemons and SQLite database. The default invocation
  97. runs all bundles in parallel and aggregates the verdicts and check
  98. counts into one exit code — meson sees a single green/red test.
  99. `--bundle NAME` runs one shard for debugging; WEBSTACK_SEQUENTIAL=1
  100. runs them in-process, one after another.
  101. - Within a bundle, scenario groups SHARE the app+daemon instance and
  102. carry per-group state as database rows (task/session/block
  103. fixtures), never process restarts; the one deliberate restart (the
  104. startup token migration) lives in the answers bundle.
  105. - EVENT-DRIVEN WAITS: wait_for polls at 50ms; the fixed sleeps were
  106. converted either to observable-condition waits (daemon-side
  107. counters, e.g. failed-delivery hits) or to quiet_window() — a
  108. bounded negative-assertion poll that FAILS LOUDLY the moment the
  109. event it guards against happens (a sleep could silently pass).
  110. - ENV KNOBS: SADDLE_DISPATCH_TICK_S=1 (dispatcher cadence; default 5
  111. in production), SADDLE_HTTP_TIMEOUT_S=2, SADDLE_LOST_WAKE_S=2,
  112. SADDLE_STALE_S=3, plus the fake daemon's scripted timings — the
  113. wire-traffic shapes stay realistic, just faster.
  114. """
  115. import json
  116. import os
  117. import re
  118. import shutil
  119. import socket
  120. import sqlite3
  121. import subprocess
  122. import sys
  123. import threading
  124. import time
  125. import urllib.error
  126. import urllib.parse
  127. import urllib.request
  128. from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
  129. # Web Push protocol verification (plan 11 task E): the test harness
  130. # owns the browser-side keys, so it can do everything the push service
  131. # does — decrypt the RFC 8291 body and verify the ES256 VAPID JWT.
  132. import base64
  133. from cryptography.hazmat.primitives import hashes
  134. from cryptography.hazmat.primitives.asymmetric import ec
  135. from cryptography.hazmat.primitives.asymmetric.utils import encode_dss_signature
  136. from cryptography.hazmat.primitives.ciphers.aead import AESGCM
  137. from cryptography.hazmat.primitives.kdf.hkdf import HKDF
  138. from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
  139. REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  140. APP = os.path.abspath(sys.argv[1]) if len(sys.argv) > 1 else os.path.join(REPO, "builddir", "clanker-saddle")
  141. # --- speed-pass tuning -----------------------------------------------------
  142. # The fake daemon's scripted timings and the dispatch-tick cadence the
  143. # quiet windows are sized against. All overridable for debugging.
  144. TICK_S = float(os.environ.get("SADDLE_DISPATCH_TICK_S", "1") or 1)
  145. FAKE_WORK_S = float(os.environ.get("WEBSTACK_FAKE_WORK_S", "1.2") or 1.2)
  146. FAKE_FAIL_S = float(os.environ.get("WEBSTACK_FAKE_FAIL_S", "0.5") or 0.5)
  147. FAKE_HEARTBEAT_S = float(os.environ.get("WEBSTACK_FAKE_HEARTBEAT_S", "0.4") or 0.4)
  148. FAKE_TRIGGER_S = float(os.environ.get("WEBSTACK_FAKE_TRIGGER_S", "0.05") or 0.05)
  149. SHARD_TIMEOUT_S = float(os.environ.get("WEBSTACK_SHARD_TIMEOUT_S", "600") or 600)
  150. def quiet_seconds():
  151. """A negative-assertion window spanning several dispatch ticks (the
  152. converted fixed sleeps were 3-7s at the old 5s tick; the bound never
  153. exceeds the old timeout)."""
  154. return min(2 * TICK_S + 0.8, 7.0)
  155. WORK = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
  156. DB_PATH = os.path.join(WORK, "web.sqlite")
  157. passes = 0
  158. failures = 0
  159. def check(condition, label):
  160. global passes, failures
  161. if condition:
  162. passes += 1
  163. print("PASS %s" % label)
  164. else:
  165. failures += 1
  166. print("FAIL %s" % label)
  167. def free_port():
  168. with socket.socket() as sock:
  169. sock.bind(("127.0.0.1", 0))
  170. return sock.getsockname()[1]
  171. APP_PORT = free_port()
  172. DAEMON_PORT = free_port()
  173. WEB = "http://127.0.0.1:%d" % APP_PORT
  174. def bundle_env(name):
  175. """Point the module's runtime globals at one shard's scratch dir,
  176. database and ports (each bundle is a process with its own app,
  177. daemons and SQLite file)."""
  178. global WORK, DB_PATH, APP_PORT, DAEMON_PORT, WEB
  179. base = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
  180. WORK = "%s-%s" % (base, name)
  181. DB_PATH = os.path.join(WORK, "web.sqlite")
  182. APP_PORT = free_port()
  183. DAEMON_PORT = free_port()
  184. WEB = "http://127.0.0.1:%d" % APP_PORT
  185. def http(method, url, data=None, headers=None, timeout=30):
  186. body = None
  187. if data is not None:
  188. pairs = []
  189. for k, v in data.items():
  190. for item in (v if isinstance(v, (list, tuple)) else [v]):
  191. pairs.append((k, item))
  192. body = "&".join("%s=%s" % (k, urllib.parse.quote(str(item))) for k, item in pairs).encode()
  193. request = urllib.request.Request(WEB + url if url.startswith("/") else url,
  194. data=body, method=method)
  195. if data is not None:
  196. request.add_header("Content-Type", "application/x-www-form-urlencoded")
  197. for key, value in (headers or {}).items():
  198. request.add_header(key, value)
  199. # Retry dropped/refused connections: the freshly started app can
  200. # close a very early keep-alive connection once while warming up,
  201. # and its first seconds can wobble under load.
  202. for attempt in range(3):
  203. try:
  204. with urllib.request.urlopen(request, timeout=timeout) as response:
  205. return response.status, response.read().decode()
  206. except urllib.error.HTTPError as error:
  207. return error.code, error.read().decode()
  208. except (urllib.error.URLError, OSError):
  209. if attempt >= 2:
  210. raise
  211. time.sleep(0.75)
  212. def directives(uri, auth=None):
  213. headers = {}
  214. if auth:
  215. headers["X-Statum-Slot"] = "key=%s; as_at=%s" % (auth["key"], auth["as_at"])
  216. status, body = http("GET", "/_statum/entrypoint?uri=" + urllib.parse.quote(uri.split("?")[0]),
  217. headers=headers)
  218. if status != 200:
  219. raise SystemExit("entrypoint %s -> HTTP %d: %s" % (uri, status, body[:300]))
  220. return json.loads(body)
  221. def slot_public(auth, uri, type_name):
  222. for directive in directives(uri, auth):
  223. if directive["type"] == "set":
  224. content = directive["snapshot_frame"].get("content")
  225. if not content:
  226. continue
  227. content = json.loads(content)
  228. if content["slot"]["type"] == type_name:
  229. return content["public"]
  230. raise SystemExit("no %s slot in entrypoint state for %s" % (type_name, uri))
  231. def act(auth, action, fields):
  232. headers = {"X-Statum-Slot": "key=%s; as_at=%s" % (auth["key"], auth["as_at"])}
  233. if action.get("private"):
  234. headers["X-Statum-Private"] = action["private"]
  235. status, body = http(action.get("method", "POST"), action["uri"], fields, headers)
  236. if status != 200:
  237. raise SystemExit("action %s -> HTTP %d: %s" % (action["uri"], status, body[:300]))
  238. return json.loads(body)
  239. def login(username, password):
  240. frames = directives("/login")
  241. content = json.loads(frames[0]["snapshot_frame"]["content"])
  242. action = content["public"]["login"]
  243. status, body = http(action.get("method", "POST"), action["uri"],
  244. {"username": username, "password": password})
  245. if status != 200:
  246. raise SystemExit("login -> HTTP %d: %s" % (status, body[:300]))
  247. for directive in json.loads(body):
  248. if directive["type"] == "set":
  249. frame = json.loads(directive["snapshot_frame"]["content"])
  250. if frame["slot"]["type"] == "auth":
  251. return {"key": frame["slot"]["key"], "as_at": frame["as_at"]}
  252. raise SystemExit("login response carried no auth slot: " + body[:300])
  253. def wait_for(predicate, timeout=60, what="condition", interval=0.05):
  254. deadline = time.monotonic() + timeout
  255. last = None
  256. while time.monotonic() < deadline:
  257. last = predicate()
  258. if last is not False and last is not None:
  259. return last
  260. time.sleep(interval)
  261. print(" (timeout waiting for %s; last: %s)" % (what, last))
  262. dump_threads()
  263. return None
  264. def quiet_window(seconds, predicate, what):
  265. """A converted fixed sleep: asserts a NEGATIVE condition (nothing may
  266. happen) across a window spanning several dispatch ticks, polling at
  267. 50ms. The moment the condition breaks the window fails loudly
  268. (diagnostics + thread dump) instead of silently passing a sleep."""
  269. deadline = time.monotonic() + seconds
  270. while time.monotonic() < deadline:
  271. value = predicate()
  272. if value is False or value is None:
  273. remaining = deadline - time.monotonic()
  274. print(" (quiet window '%s' broken after %.2fs of %.2fs — the"
  275. " guarded event DID happen)" % (what, seconds - remaining, seconds))
  276. dump_threads()
  277. return False
  278. time.sleep(0.05)
  279. return True
  280. def dump_threads():
  281. """Stacks of suite threads not parked in sleep — pinpoints a wedged
  282. fake lifecycle when a wait times out."""
  283. import traceback
  284. shown = 0
  285. for thread_id, frame in sys._current_frames().items():
  286. stack = "".join(traceback.format_stack(frame)[-4:])
  287. if ("web_stack_test.py" in stack) and ("sleep" not in stack) and shown < 6:
  288. shown += 1
  289. print(" --- thread %s ---\n%s" % (thread_id, stack))
  290. def task_state(auth, task_id):
  291. return slot_public(auth, "/tasks/%d" % task_id, "task")
  292. def wait_task_done(auth, task_id, timeout=90):
  293. return wait_for(lambda: task_state(auth, task_id).get("status") == "done" or False,
  294. timeout=timeout, what="task %d -> done" % task_id)
  295. def task_id_of(title):
  296. conn = db()
  297. try:
  298. return conn.execute("select id from tasks where title = ?", (title,)).fetchone()[0]
  299. finally:
  300. conn.close()
  301. def enqueue_task(auth, task_id):
  302. """The human-only Enqueue action (plan 11): draft -> queued."""
  303. detail = task_state(auth, task_id)
  304. assert detail["status"] == "draft", "task %d not in draft (%s)" % (task_id, detail["status"])
  305. act(auth, detail["enqueue"], {})
  306. return task_state(auth, task_id)["status"]
  307. def create_queued(auth, fields):
  308. """Create via the queue form (lands in draft) and enqueue it — the
  309. standard 'this task should dispatch' fixture of the plan-11 world."""
  310. act(auth, slot_public(auth, "/tasks", "tasks")["create"], fields)
  311. task_id = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  312. if t["title"] == fields["title"]][0])
  313. enqueue_task(auth, task_id)
  314. return task_id
  315. def saddle_api(method, path, token, payload=None):
  316. """Raw call against the saddle ingest API; returns (status, json)."""
  317. request = urllib.request.Request(WEB + path,
  318. data=json.dumps(payload or {}).encode(), method=method)
  319. request.add_header("Content-Type", "application/json")
  320. if token is not None:
  321. request.add_header("Authorization", "Bearer " + token)
  322. try:
  323. with urllib.request.urlopen(request, timeout=15) as response:
  324. return response.status, json.loads(response.read().decode() or "{}")
  325. except urllib.error.HTTPError as error:
  326. body = error.read().decode()
  327. try:
  328. return error.code, json.loads(body)
  329. except ValueError:
  330. return error.code, {"raw": body}
  331. def db():
  332. return sqlite3.connect(DB_PATH)
  333. def saddle_card(auth, name):
  334. rows = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  335. if s["name"] == name]
  336. return rows[0]
  337. def queued_count():
  338. conn = db()
  339. try:
  340. return conn.execute(
  341. "select count(*) from tasks where status = 'queued'").fetchone()[0]
  342. finally:
  343. conn.close()
  344. def command_rows(session):
  345. conn = db()
  346. try:
  347. return conn.execute(
  348. "select command_text, type from session_commands"
  349. " where session_id = ? order by id", (session,)).fetchall()
  350. finally:
  351. conn.close()
  352. class FakeSaddle:
  353. """In-process saddle daemon implementing the wire contract.
  354. Phase-2 push behaviour: the daemon asks POST /api/saddle/next-task
  355. whenever idle-awake (at registration, on POST /wake, after its task
  356. reaches a terminal state, and when a /status probe observes it
  357. parked idle — throttled), accepts assignments like POST /start used
  358. to, then PUSHES its telemetry home: raw PTY chunks via POST
  359. /api/saddle/chunks and round results with state transitions via
  360. POST /api/saddle/results (working at round start, terminal states
  361. when the scripted lifecycle ends them). /output, /log and /start
  362. are gone (404) like a post-phase-2 daemon; /task-update serves the
  363. cancel/comment/subtask-status kinds; POST /answer receives a
  364. delivered verdict when the round is parked in a block's answer
  365. wait (409 when nothing waits — the real daemon's contract; the
  366. resumed round continues after the response). Every trigger can be
  367. disabled per test for determinism; ask_now() performs a
  368. synchronous manual ask.
  369. """
  370. PROMPT = "bbarrow@fedora:/workspace/task-1$ "
  371. def __init__(self):
  372. self.lock = threading.Lock()
  373. self.start_payloads = [] # captured assignment payloads (/start + next-task 200s)
  374. self.state = "idle"
  375. self.summary = ""
  376. self.state_entered = time.monotonic()
  377. self.fail_next = False # next task drives the task to failed
  378. self.start_counts = {} # task id -> assignment count
  379. self.resumed_pending = False # crash-resume flag on the next /status
  380. self.suspend_on_title = None # this task (by title) parks as suspended
  381. self.output_seq = 100 # daemon-global entry ids for pushed results
  382. self.last_activity_epoch = int(time.time())
  383. self.subtask_status_events = [] # recorded /task-update subtask-status payloads
  384. self.subtask_fail_hits = 0 # delivery attempts that 500ed (fail_subtask_status)
  385. self.fail_subtask_status = False # exercise the web's retry path
  386. self.hang_next = {} # path suffix -> hang count (never responds)
  387. self.abort_failures = 0 # cancel task-updates failing (500) while > 0
  388. self.abort_log = [] # task ids aborted via task-update cancel
  389. self.running_task = 0 # task id the daemon believes it runs
  390. self.running_title = "" # its title (suspend targeting)
  391. self.session_id = None # the web-side run row of the running task
  392. self.hold_state = False # freeze lifecycle + heartbeat pushes
  393. self.ssh_public_key = "" # reported on /status + /register
  394. self.started_mono = time.monotonic()
  395. self.freeze_activity = False # hold last_activity_epoch still
  396. self.silent = False # suppress pushes (staleness tests)
  397. self.dead = False # kill(): no serving AND no calls home
  398. self.status_hits = 0 # GET /status arrivals (probe accounting)
  399. self.comment_log = [] # recorded /task-update comment payloads
  400. self.scripted_corpus = None # chunk batch pushed when a task starts
  401. # --- plan 10 phase 1: ask/wake/register ---
  402. self.token = None # per-saddle token from POST /register
  403. self.register_log = [] # recorded /register bodies
  404. self.wake_log = [] # recorded POST /wake arrivals
  405. self.asks = [] # [{"status": int, "payload": dict|None}]
  406. self.ask_on_register = True
  407. self.ask_on_wake = True
  408. self.ask_on_terminal = True
  409. self.ask_on_idle_status = True
  410. self.ask_delay = 0.0 # seconds; a slow daemon delays its wake-triggered ask
  411. self.serve_register = True # False: 404 /register (pre-flip daemon)
  412. self.last_ask_trigger = 0.0
  413. self.port = None
  414. self.ask_threads = [] # scheduled ask-trigger threads (settle_asks joins them)
  415. # --- answer delivery (the parked-session resume) ---
  416. self.pending_block = None # the block payload this daemon waits on (state blocked)
  417. self.answer_log = [] # recorded POST /answer bodies
  418. self.answer_push_delay = 0.2 # the resumed round's first push, after the response
  419. def set_state(self, state, summary=None):
  420. """Advances the scripted lifecycle."""
  421. self.state = state
  422. if summary is not None:
  423. self.summary = summary
  424. self.state_entered = time.monotonic()
  425. def touch_activity(self):
  426. if not self.freeze_activity:
  427. self.last_activity_epoch = int(time.time())
  428. def queue_output(self, entry_type, command, output):
  429. """Pushes a typed entry as a round-results push (the daemon's
  430. entry log riding the results channel)."""
  431. with self.lock:
  432. entry = {
  433. "id": self.output_seq,
  434. "command": command,
  435. "output": output,
  436. "exit_code": 1,
  437. "type": entry_type,
  438. "timestamp": "2026-09-06T00:00:00Z",
  439. }
  440. self.output_seq += 1
  441. self.push_results(self.state if self.state in ("working",) else "working",
  442. entries=[entry])
  443. # --- plan 10 phase 2: the push channels ---
  444. def push_chunks(self, chunks):
  445. """Pushes raw PTY chunks home: elements are texts (assigned
  446. fresh seqs) or {seq, text} dicts (a redelivery or explicit
  447. corpus). Pushes are suppressed while silent/hold_state (the
  448. staleness tests' silent runner)."""
  449. if self.dead or self.silent or self.token is None or self.session_id is None:
  450. return
  451. batch = []
  452. with self.lock:
  453. for chunk in chunks:
  454. if isinstance(chunk, dict):
  455. batch.append({"seq": int(chunk["seq"]), "text": chunk["text"]})
  456. else:
  457. batch.append({"seq": self.next_push_seq(), "text": chunk})
  458. try:
  459. saddle_api("POST", "/api/saddle/chunks", self.token,
  460. {"session_id": self.session_id, "chunks": batch})
  461. except (OSError, ValueError):
  462. pass # app restarting / shutting down mid-test
  463. def next_push_seq(self):
  464. # The daemon-side chunk counter for ad-hoc pushes.
  465. self._push_seq = getattr(self, "_push_seq", 0) + 1
  466. return self._push_seq
  467. def push_results(self, state, entries=None, block=None, summary=None, session_id=None):
  468. """Pushes one round-results payload home (state transition plus
  469. its entries). Returns the HTTP status when it went out, None
  470. when suppressed."""
  471. if self.dead or self.silent:
  472. return None
  473. if self.token is None:
  474. return None
  475. target = session_id if session_id is not None else self.session_id
  476. if target is None:
  477. return None
  478. payload = {"session_id": target, "state": state}
  479. if entries is not None:
  480. payload["entries"] = entries
  481. if block is not None:
  482. payload["block"] = block
  483. if state == "blocked":
  484. # The daemon parks its round on this block's answer wait
  485. # (the real daemon's wait_for_answer).
  486. self.pending_block = dict(block)
  487. if summary is not None:
  488. payload["summary"] = summary
  489. try:
  490. status, _ = saddle_api("POST", "/api/saddle/results", self.token, payload)
  491. if status != 200:
  492. print(" [fake %s] push %s -> HTTP %s (task=%s)"
  493. % (self.port, state, status, self.running_task))
  494. return status
  495. except (OSError, ValueError) as error:
  496. print(" [fake %s] push %s failed: %r (task=%s)"
  497. % (self.port, state, error, self.running_task))
  498. return None
  499. # --- plan 10 phase 1: the ask path ---
  500. def ask_now(self):
  501. """Synchronously asks the web for work (a phase-1 daemon's idle
  502. ask). A 200 assignment is accepted exactly like POST /start;
  503. returns (status, payload|None)."""
  504. token = self.token
  505. if token is None:
  506. raise SystemExit("fake daemon asked before receiving its token")
  507. status, payload = saddle_api("POST", "/api/saddle/next-task", token)
  508. with self.lock:
  509. self.asks.append({"status": status, "payload": payload})
  510. if status == 200:
  511. self.accept_assignment(payload)
  512. return status, payload
  513. def _trigger_ask(self, reason):
  514. """Schedules a delayed ask (the daemon woke / went idle-awake).
  515. ask_delay models a SLOW asker — a daemon still winding down its
  516. round when the wake lands: it honours the wake, but its ask
  517. arrives late (still inside the web's lost-wake guard window)."""
  518. def run():
  519. if self.state in ("working", "failing"):
  520. return # a running daemon does not ask
  521. if self.ask_delay > 0:
  522. time.sleep(self.ask_delay)
  523. for attempt in range(3):
  524. try:
  525. status, _ = self.ask_now()
  526. if status in (200, 204):
  527. return
  528. print(" (ask %s -> HTTP %s, attempt %d)" % (reason, status, attempt + 1))
  529. except Exception as error: # app restarting / blips: retry
  530. print(" (ask %s failed: %r, attempt %d)" % (reason, error, attempt + 1))
  531. time.sleep(1.0)
  532. def starter():
  533. time.sleep(FAKE_TRIGGER_S)
  534. run()
  535. thread = threading.Thread(target=starter, daemon=True)
  536. with self.lock:
  537. self.ask_threads.append(thread)
  538. thread.start()
  539. def settle_asks(self, timeout=15):
  540. """Joins every ask-trigger thread scheduled so far — the
  541. event-driven replacement for the fixed sleeps that waited for
  542. in-flight asks to land after triggers were frozen."""
  543. with self.lock:
  544. threads = [t for t in self.ask_threads if t.is_alive()]
  545. deadline = time.monotonic() + timeout
  546. for thread in threads:
  547. thread.join(timeout=max(0.1, deadline - time.monotonic()))
  548. with self.lock:
  549. self.ask_threads = [t for t in self.ask_threads if t.is_alive()]
  550. return all(not thread.is_alive() for thread in threads)
  551. def maybe_trigger_ask(self, reason, throttle=False):
  552. """Schedules an idle-awake ask. Protocol events (wake, abort,
  553. round end, park) ask singularly and immediately; only the
  554. /status-observation path throttles (a burst of probes must not
  555. hammer the ask endpoint)."""
  556. now = time.monotonic()
  557. if throttle and now - self.last_ask_trigger < 1.0:
  558. return
  559. self.last_ask_trigger = now
  560. self._trigger_ask(reason)
  561. def accept_assignment(self, payload):
  562. """A 200 next-task assignment: adopted exactly like POST /start."""
  563. self.start_payloads.append(payload)
  564. task = payload.get("task") or {}
  565. task_id = task.get("id")
  566. self.running_task = int(task_id or 0)
  567. self.running_title = task.get("title") or ""
  568. self.session_id = payload.get("session_id")
  569. # The assignment's answers resolve any block this daemon was
  570. # parked on (resume-carries-answer).
  571. self.pending_block = None
  572. count = self.start_counts.get(task_id, 0) + 1
  573. self.start_counts[task_id] = count
  574. if count > 1:
  575. # Crash-resume contract: re-pushing a task whose
  576. # workspace still exists restores it; the first
  577. # /status after the restore reports "resumed": true.
  578. self.resumed_pending = True
  579. self.set_state("failing" if self.fail_next else "working")
  580. self.fail_next = False
  581. # The round's opening telemetry: the flushed-so-far raw PTY
  582. # batch plus the working state push (>= 2 flushes per round
  583. # and the 10s timer are the real daemon's contract; the fake
  584. # pushes its batch up front and heartbeats while working).
  585. if self.scripted_corpus is not None:
  586. self.push_chunks([dict(chunk) for chunk in self.scripted_corpus])
  587. else:
  588. self.push_chunks([self.PROMPT, "echo hello\r\n", "hello\r\n"])
  589. self.push_results("working")
  590. class Handler(BaseHTTPRequestHandler):
  591. daemon = None # set on the class before serving
  592. protocol_version = "HTTP/1.1"
  593. def log_message(self, fmt, *args):
  594. pass
  595. def maybe_hang(self, path):
  596. """True when this request should hang forever (never respond),
  597. consuming one unit of the path's hang budget."""
  598. for suffix, count in list(self.daemon.hang_next.items()):
  599. if count > 0 and path.startswith(suffix):
  600. self.daemon.hang_next[suffix] = count - 1
  601. return True
  602. return False
  603. def _send(self, status, obj):
  604. body = json.dumps(obj).encode()
  605. self.send_response(status)
  606. self.send_header("Content-Type", "application/json")
  607. self.send_header("Content-Length", str(len(body)))
  608. self.end_headers()
  609. self.wfile.write(body)
  610. def do_POST(self):
  611. if self.maybe_hang(self.path):
  612. time.sleep(600)
  613. return
  614. self.daemon.touch_activity()
  615. length = int(self.headers.get("Content-Length") or 0)
  616. raw = self.rfile.read(length).decode() if length else "{}"
  617. if self.path == "/register":
  618. # Plan 10: the web delivers the per-saddle token + its
  619. # own base URL; the daemon persists the token for its
  620. # calls home and reports its ssh key + version.
  621. if not self.daemon.serve_register:
  622. self._send(404, {"error": "no such route"})
  623. return
  624. try:
  625. payload = json.loads(raw)
  626. except ValueError:
  627. payload = {"raw": raw}
  628. self.daemon.register_log.append(payload)
  629. if payload.get("saddle_token"):
  630. self.daemon.token = payload["saddle_token"]
  631. self._send(200, {"ok": True,
  632. "ssh_public_key": self.daemon.ssh_public_key,
  633. "version": "0.1-fake"})
  634. if self.daemon.ask_on_register:
  635. self.daemon.maybe_trigger_ask("register")
  636. return
  637. if self.path == "/wake":
  638. # Plan 10: a capable task is ready; a sleeping daemon
  639. # asks for work (immediately — a wake is a direct
  640. # command, never throttled).
  641. self.daemon.wake_log.append(time.monotonic())
  642. self._send(200, {"ok": True})
  643. if self.daemon.ask_on_wake:
  644. self.daemon.maybe_trigger_ask("wake")
  645. return
  646. if self.path == "/answer":
  647. # Plan 11 (daemon Task A): parks FREE the daemon, so no
  648. # block ever waits in-process — the verdict travels in
  649. # the resuming allocation's resume_notice. Always 409,
  650. # exactly like the real daemon's contract.
  651. try:
  652. payload = json.loads(raw)
  653. except ValueError:
  654. self._send(400, {"error": "bad json"})
  655. return
  656. self.daemon.answer_log.append({
  657. "refused": True,
  658. "answer": payload.get("answer", "")})
  659. self._send(409, {"error": "no block is waiting for an answer — parks resume via allocation"})
  660. return
  661. if self.path == "/task-update":
  662. # Plan 10: cancel (abort machinery), comment (injected
  663. # turn) and subtask-status (live subtask set) — delivered
  664. # directly to a RUNNING daemon, no auth.
  665. try:
  666. payload = json.loads(raw)
  667. except ValueError:
  668. self._send(400, {"error": "bad json"})
  669. return
  670. kind = payload.get("kind")
  671. if kind == "cancel":
  672. if self.daemon.abort_failures > 0:
  673. self.daemon.abort_failures -= 1
  674. self._send(500, {"error": "cancel failed (simulated)"})
  675. return
  676. aborted = self.daemon.running_task
  677. self.daemon.set_state("idle")
  678. self.daemon.summary = ""
  679. self.daemon.running_task = 0
  680. self.daemon.session_id = None
  681. self.daemon.abort_log.append(aborted)
  682. self._send(200, {"ok": True, "aborted_task_id": aborted})
  683. # An aborted daemon returns to its idle loop: it asks.
  684. self.daemon.maybe_trigger_ask("aborted")
  685. elif kind == "comment":
  686. self.daemon.comment_log.append(payload)
  687. self._send(200, {"ok": True})
  688. elif kind == "subtask-status":
  689. if self.daemon.fail_subtask_status:
  690. self.daemon.subtask_fail_hits += 1
  691. self._send(500, {"error": "simulated failure"})
  692. return
  693. self.daemon.subtask_status_events.append(payload)
  694. self._send(200, {"ok": True})
  695. else:
  696. self._send(400, {"error": "unknown task-update kind"})
  697. return
  698. self._send(404, {"error": "no such route"})
  699. self.close_connection = True
  700. def do_GET(self):
  701. if self.maybe_hang(self.path):
  702. time.sleep(600)
  703. return
  704. if self.path != "/status":
  705. # /status itself is not daemon activity: the liveness
  706. # signal must reflect loop rounds / command completions.
  707. self.daemon.touch_activity()
  708. if self.path == "/status":
  709. self.daemon.status_hits += 1
  710. response = {"state": self.daemon.state, "summary": self.daemon.summary,
  711. "uptime_seconds": int(time.monotonic() - self.daemon.started_mono),
  712. "last_activity_epoch": self.daemon.last_activity_epoch,
  713. "ssh_public_key": self.daemon.ssh_public_key}
  714. if self.daemon.resumed_pending:
  715. response["resumed"] = True
  716. self.daemon.resumed_pending = False
  717. # A parked/terminal daemon is idle-awake: a phase-1
  718. # daemon asks (and sleeps on 204). Throttled so status
  719. # poll bursts do not hammer the ask endpoint.
  720. if self.daemon.ask_on_idle_status and self.daemon.state in (
  721. "idle", "done", "failed", "suspended"):
  722. self.daemon.maybe_trigger_ask("idle-status", throttle=True)
  723. self._send(200, response)
  724. else:
  725. self._send(404, {"error": "no such route"})
  726. self.close_connection = True
  727. def run_lifecycle(self):
  728. """Applies the scripted lifecycle on a timer (like a real daemon
  729. whose task ends on its own): working heartbeats chunks (the
  730. flush timer's stand-in, keeping the staleness clock fed), works
  731. for FAKE_WORK_S, then pushes done/suspended; failing pushes
  732. failed after FAKE_FAIL_S — and a task END makes the idle-awake
  733. daemon ask again."""
  734. while True:
  735. time.sleep(0.1)
  736. if self.hold_state or self.silent:
  737. # hold_state freezes the script; silent models a daemon
  738. # mid-round with nothing to flush — its round has not
  739. # ended, so no terminal push fires while quiet.
  740. continue
  741. if self.state == "failing":
  742. if time.monotonic() - self.state_entered >= FAKE_FAIL_S:
  743. self.set_state("failed", "bash -n rejected the script (simulated)")
  744. self.push_results("failed",
  745. summary="bash -n rejected the script (simulated)")
  746. if self.ask_on_terminal:
  747. self.maybe_trigger_ask("terminal")
  748. elif self.state == "working":
  749. if time.monotonic() - self.state_entered >= FAKE_WORK_S:
  750. if self.suspend_on_title and self.suspend_on_title == self.running_title:
  751. # `saddle suspend`: park until a fresh assignment
  752. # resumes the workspace (title-targeted one-shot,
  753. # so an interleaved dispatch cannot eat the flag).
  754. self.set_state("suspended")
  755. self.suspend_on_title = None
  756. self.push_results("suspended")
  757. # A parked daemon is idle-awake: it asks (and
  758. # sleeps on 204) so a later wake can reach it.
  759. if self.ask_on_idle_status:
  760. self.maybe_trigger_ask("parked")
  761. else:
  762. self.set_state("done", "fake daemon finished")
  763. self.push_results("done", summary="fake daemon finished")
  764. if self.ask_on_terminal:
  765. self.maybe_trigger_ask("terminal")
  766. elif time.monotonic() - getattr(self, "last_heartbeat", 0) >= FAKE_HEARTBEAT_S:
  767. # The flush timer: output keeps flowing, so chunks
  768. # keep arriving (the staleness window never opens on
  769. # a healthy runner).
  770. self.last_heartbeat = time.monotonic()
  771. self.push_chunks(["working-heartbeat\r\n"])
  772. def serve(self, port=None):
  773. """Binds the daemon's HTTP server. A requested port that is
  774. already taken (free_port() hands out recyclable ports — two
  775. calls can collide) falls back to an OS-assigned one instead of
  776. silently dying and impersonating whatever owns that port."""
  777. if port is None:
  778. port = DAEMON_PORT
  779. handler = type("BoundHandler", (FakeSaddle.Handler,), {"daemon": self})
  780. self.server = None
  781. try:
  782. self.server = ThreadingHTTPServer(("127.0.0.1", port), handler)
  783. except OSError:
  784. print("(fake daemon: port %d taken, falling back)" % port)
  785. self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
  786. self.port = self.server.server_address[1]
  787. threading.Thread(target=self.server.serve_forever, daemon=True).start()
  788. threading.Thread(target=self.run_lifecycle, daemon=True).start()
  789. return self.server
  790. def kill(self):
  791. """The daemon dies: it stops listening (connections refused) AND
  792. stops calling home — no pushes, no asks (a dead process has no
  793. outbound anything; in-flight handler threads may linger but the
  794. wire contract's traffic is one-directional now)."""
  795. self.dead = True
  796. if getattr(self, "server", None) is not None:
  797. self.server.shutdown()
  798. self.server.server_close()
  799. self.server = None
  800. def start_app():
  801. config = json.load(open(os.path.join(REPO, "web-config.json")))
  802. config["saddle_callback_base_url"] = WEB
  803. with open(os.path.join(WORK, "web-config.json"), "w") as handle:
  804. json.dump(config, handle)
  805. env = dict(os.environ)
  806. env.update({
  807. "SPRY_DB_PATH": DB_PATH,
  808. "SPRY_SEED_ADMIN_USERNAME": "admin",
  809. "SPRY_SEED_ADMIN_PASSWORD": "adminpass123",
  810. "SADDLE_LLM_BASE_URL": "http://127.0.0.1:%d" % DAEMON_PORT,
  811. "SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
  812. "SADDLE_HTTP_TIMEOUT_S": "2",
  813. "SADDLE_LOST_WAKE_S": "2",
  814. "SADDLE_STALE_S": "3",
  815. "SADDLE_DISPATCH_TICK_S": "1",
  816. # Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
  817. # resolves the ancient /lib64 copies): empty makes the loader use
  818. # the binary's RUNPATH, which is the dev stack it was built
  819. # against.
  820. "LD_LIBRARY_PATH": "",
  821. })
  822. log_previous = os.path.join(WORK, "app.log.1")
  823. if os.path.exists(os.path.join(WORK, "app.log")):
  824. shutil.move(os.path.join(WORK, "app.log"), log_previous)
  825. log = open(os.path.join(WORK, "app.log"), "w")
  826. command = [APP, str(APP_PORT)]
  827. if os.environ.get("WEBSTACK_GDB"):
  828. # Debugging aid: run the app under gdb; a crash backtrace lands in
  829. # app.log. WEBSTACK_GDB=1 meson test ... to enable.
  830. command = ["gdb", "-batch", "-quiet",
  831. "-ex", "handle SIGPIPE nostop noprint pass",
  832. "-ex", "run", "-ex", "bt", "--args", APP, str(APP_PORT)]
  833. process = subprocess.Popen(command, cwd=WORK, env=env,
  834. stdout=log, stderr=subprocess.STDOUT)
  835. outcome = wait_for(lambda: (app_ready() and "ready")
  836. or (process.poll() is not None and "app-exited-%d" % process.poll())
  837. or False, timeout=30, what="app ready")
  838. if outcome is None:
  839. raise SystemExit("app did not come up; see %s/app.log" % WORK)
  840. if str(outcome).startswith("app-exited-"):
  841. raise SystemExit("app exited during startup (%s); see %s/app.log"
  842. % outcome, WORK)
  843. return process
  844. def app_ready():
  845. try:
  846. status, _ = http("GET", "/login", timeout=3)
  847. return status == 200
  848. except (urllib.error.URLError, OSError):
  849. return False
  850. def dump_app_stacks(pid):
  851. """Best-effort thread backtrace of the (possibly wedged) app."""
  852. try:
  853. fds = os.listdir("/proc/%d/fd" % pid)
  854. print("(app open fds at dump: %d)" % len(fds))
  855. kinds = {}
  856. for fd in fds[:1200]:
  857. try:
  858. target = os.readlink("/proc/%d/fd/%s" % (pid, fd))
  859. except OSError:
  860. continue
  861. key = target.split(":", 1)[0] if ":" in target else target
  862. kinds[key] = kinds.get(key, 0) + 1
  863. print("(fd kinds: %s)" % sorted(kinds.items(), key=lambda kv: -kv[1])[:6])
  864. except OSError:
  865. pass
  866. for command in (["eu-stack", "-p", str(pid)],
  867. ["gdb", "-batch", "-quiet", "-p", str(pid),
  868. "-ex", "thread apply all bt"]):
  869. try:
  870. out = subprocess.run(command, capture_output=True, timeout=60)
  871. text = out.stdout.decode(errors="replace") + out.stderr.decode(errors="replace")
  872. if text.strip():
  873. print("(app stacks via %s)\n%s" % (command[0], text[:12000]))
  874. return
  875. except (OSError, subprocess.SubprocessError):
  876. continue
  877. # =============================================================================
  878. # Fake push service + Web Push protocol helpers (plan 11 task E)
  879. # =============================================================================
  880. def b64url(raw):
  881. return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
  882. def b64url_dec(text):
  883. return base64.urlsafe_b64decode(text + "=" * ((4 - len(text) % 4) % 4))
  884. class FakePushService:
  885. """In-process Web Push endpoint server: every subscription the tests
  886. register points at a path here (a distinct "browser endpoint").
  887. Responses are scriptable per path — a fixed status (the 410-prune
  888. fixture) or N transient 500s (the retry-next-tick fixture)."""
  889. def __init__(self):
  890. self.requests = [] # [{"path", "headers", "body", "at"}]
  891. self.fail_next = {} # path -> remaining 500s
  892. self.status_for = {} # path -> fixed status
  893. self.lock = threading.Lock()
  894. self.port = None
  895. self.server = None
  896. def serve(self):
  897. service = self
  898. class Handler(BaseHTTPRequestHandler):
  899. protocol_version = "HTTP/1.1"
  900. def log_message(self, fmt, *args):
  901. pass
  902. def do_POST(self):
  903. length = int(self.headers.get("Content-Length") or 0)
  904. body = self.rfile.read(length)
  905. status = 201
  906. if self.path in service.status_for:
  907. status = service.status_for[self.path]
  908. elif service.fail_next.get(self.path, 0) > 0:
  909. service.fail_next[self.path] -= 1
  910. status = 500
  911. with service.lock:
  912. service.requests.append({
  913. "path": self.path,
  914. "headers": {key.lower(): value
  915. for key, value in self.headers.items()},
  916. "body": body,
  917. "at": time.monotonic(),
  918. })
  919. payload = b"{}"
  920. self.send_response(status)
  921. self.send_header("Content-Type", "application/json")
  922. self.send_header("Content-Length", str(len(payload)))
  923. self.end_headers()
  924. self.wfile.write(payload)
  925. self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
  926. self.port = self.server.server_address[1]
  927. threading.Thread(target=self.server.serve_forever, daemon=True).start()
  928. return self.server
  929. def url(self, path):
  930. return "http://127.0.0.1:%d%s" % (self.port, path)
  931. def hits(self, prefix):
  932. with self.lock:
  933. return [request for request in self.requests
  934. if request["path"].startswith(prefix)]
  935. def decrypt_webpush(request, client_private_key, client_public, auth_secret):
  936. """RFC 8291 decryption of one captured push: returns
  937. (record_size, payload dict) — the payload builder's exact output."""
  938. body = request["body"]
  939. salt = body[:16]
  940. record_size = int.from_bytes(body[16:20], "big")
  941. key_id_len = body[20]
  942. eph_public = body[21:21 + key_id_len]
  943. ciphertext = body[21 + key_id_len:]
  944. eph_point = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(),
  945. eph_public)
  946. shared = client_private_key.exchange(ec.ECDH(), eph_point)
  947. info = b"WebPush: info\x00" + eph_public + client_public
  948. cek = HKDF(algorithm=hashes.SHA256(), length=16, salt=salt,
  949. info=info).derive(shared + auth_secret)
  950. nonce = HKDF(algorithm=hashes.SHA256(), length=12, salt=salt,
  951. info=info).derive(shared + auth_secret)
  952. plaintext = AESGCM(cek).decrypt(nonce, ciphertext, None)
  953. return record_size, json.loads(plaintext[:-1]) # strip the 0x02 delimiter
  954. def verify_vapid(request, vapid_public, vapid_private):
  955. """Verifies the ES256 VAPID JWT on one captured push; returns its
  956. claims dict (aud/exp/sub) — raises on a bad signature."""
  957. header = request["headers"].get("authorization", "")
  958. assert header.startswith("vapid t="), "no vapid authorization header"
  959. token_part = header[len("vapid t="):].split(",")[0]
  960. sent_key = header.split("k=")[1]
  961. head_b64, claims_b64, sig_b64 = token_part.split(".")
  962. signature = b64url_dec(sig_b64)
  963. key = ec.derive_private_key(
  964. int.from_bytes(b64url_dec(vapid_private), "big"), ec.SECP256R1())
  965. key.public_key().verify(
  966. encode_dss_signature(int.from_bytes(signature[:32], "big"),
  967. int.from_bytes(signature[32:], "big")),
  968. (head_b64 + "." + claims_b64).encode(),
  969. ec.ECDSA(hashes.SHA256()))
  970. claims = json.loads(b64url_dec(claims_b64))
  971. assert sent_key == vapid_public, "the k= key is not the app's VAPID key"
  972. return claims
  973. def action_notify(result):
  974. """The message of the first notify directive an action returned."""
  975. for directive in result:
  976. if directive.get("type") == "notify":
  977. return directive.get("message", "")
  978. return None
  979. # =============================================================================
  980. # Shard orchestration
  981. # =============================================================================
  982. class Ctx:
  983. """Per-bundle shared fixtures: the app process, the admin auth slot
  984. and the primary fake saddle (registered + token delivered)."""
  985. def start_bundle(name):
  986. """Boots one shard's environment: scratch dir, database, ports, the
  987. primary fake daemon and the app, then the shared fixtures every
  988. bundle builds on (the demo project + the registered fake saddle).
  989. Scenario groups inside the bundle share this app+daemon instance
  990. and sequence their state as DB rows, never process restarts."""
  991. global APP_PORT, WEB, DAEMON_PORT
  992. bundle_env(name)
  993. fake = FakeSaddle()
  994. fake.serve()
  995. DAEMON_PORT = fake.port # the actually-bound port (fallback-safe)
  996. # A rare framework-level crash (stack corruption in Statum slot
  997. # signing during the FIRST login — shared-library codegen bug, moved
  998. # twice already) kills the app before any test runs; a clean restart
  999. # (with fresh ports) sidesteps it. Only tolerated while nothing has
  1000. # been checked yet.
  1001. auth = None
  1002. app = None
  1003. for attempt in range(3):
  1004. APP_PORT = free_port()
  1005. WEB = "http://127.0.0.1:%d" % APP_PORT
  1006. if os.path.exists(WORK):
  1007. shutil.rmtree(WORK)
  1008. os.makedirs(os.path.join(WORK, "workspace"))
  1009. try:
  1010. app = start_app()
  1011. auth = login("admin", "adminpass123")
  1012. break
  1013. except (SystemExit, OSError, urllib.error.URLError):
  1014. if passes > 0:
  1015. raise
  1016. if app is not None and app.poll() is None:
  1017. app.terminate()
  1018. try:
  1019. app.wait(timeout=5)
  1020. except subprocess.TimeoutExpired:
  1021. app.kill()
  1022. app = None
  1023. print("(app died before login; restarting cleanly, attempt %d)" % (attempt + 1))
  1024. if app is None or auth is None:
  1025. raise SystemExit("app never came up; see %s/app.log" % WORK)
  1026. state = slot_public(auth, "/projects", "projects")
  1027. act(auth, state["create"], {
  1028. "name": "demo",
  1029. "git_url": "https://example.com/demo.git",
  1030. "default_branch": "main",
  1031. })
  1032. saddles_state = slot_public(auth, "/saddles", "saddles")
  1033. act(auth, saddles_state["register"], {"name": "fake-saddle",
  1034. "host": "127.0.0.1:%d" % fake.port})
  1035. registered = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  1036. if s["name"] == "fake-saddle"]
  1037. if wait_for(lambda: len(fake.register_log) > 0 or False,
  1038. timeout=15, what="/register delivery") is None:
  1039. raise SystemExit("fake saddle never received /register")
  1040. ctx = Ctx()
  1041. ctx.name = name
  1042. ctx.app = app
  1043. ctx.auth = auth
  1044. ctx.fake = fake
  1045. ctx.fake_id = int(registered[0]["id"])
  1046. ctx.fake_token = registered[0]["token"]
  1047. return ctx
  1048. # =============================================================================
  1049. # Bundle: pages — block presentation, diff URLs, computed display state,
  1050. # terminal rendering (rejected entries, ANSI SGR), task-page layout and
  1051. # the /poll verdict channel. Fixture-driven: tasks and sessions are
  1052. # inserted as rows; only the rejected-ride scenario dispatches.
  1053. # =============================================================================
  1054. def bundle_pages(ctx):
  1055. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  1056. app = ctx.app
  1057. check(True, "login yields an auth slot")
  1058. print("== plan blocks: every task's saddle plan parks for acceptance ==")
  1059. conn = db()
  1060. conn.execute("insert into tasks (project_id, title, status, priority,"
  1061. " created_at)"
  1062. " values (1, 'Plan review task', 'working', 5,"
  1063. " strftime('%s','now'))")
  1064. plan_task = conn.execute("select max(id) from tasks").fetchone()[0]
  1065. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
  1066. " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
  1067. (plan_task, fake_id))
  1068. conn.commit()
  1069. session_id = conn.execute("select max(id) from sessions").fetchone()[0]
  1070. conn.close()
  1071. status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1072. {"type": "block", "session_id": session_id, "data": {
  1073. "question": "Plan submitted for review",
  1074. "type": "plan",
  1075. "body": "# The plan\n\n1. Do it"}})
  1076. check(status == 200 and body.get("block_id"), "plan block raised")
  1077. status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1078. {"type": "block", "session_id": session_id, "data": {
  1079. "question": "Which database?",
  1080. "type": "question",
  1081. "options": ["SQLite", "Postgres"],
  1082. "body": "# Storage choice"}})
  1083. check(status == 200, "question block raised")
  1084. detail = task_state(auth, plan_task)
  1085. blocks = {b["block_type"]: b for b in detail["blocks"]}
  1086. check(blocks["plan"].get("is_plan_review") is True,
  1087. "plan block flagged as the plan review")
  1088. check(blocks["plan"].get("show_body") is False,
  1089. "pending plan review hides the duplicate body preview")
  1090. check(blocks["question"].get("show_body") is True,
  1091. "question block keeps its body preview")
  1092. # Accept applies the submitted plan markdown to task.plan and stores
  1093. # the stable approval wording as the answer (it becomes the resume
  1094. # notice the daemon reads).
  1095. act(auth, blocks["plan"]["answer_action"],
  1096. {"answer": "Plan approved. Proceed with implementation."})
  1097. detail = task_state(auth, plan_task)
  1098. plan_block = [b for b in detail["blocks"] if b["block_type"] == "plan"][0]
  1099. check(plan_block.get("show_body") is True,
  1100. "answered plan review shows the body again (review section gone)")
  1101. check(plan_block.get("answered") is True
  1102. and plan_block.get("answer") == "Plan approved. Proceed with implementation.",
  1103. "accept stores the stable approval wording")
  1104. check(detail.get("plan", "").startswith("# The plan"),
  1105. "accepting a plan applies its markdown to task.plan")
  1106. print("== B1: highlight-to-comment quote dialog ==")
  1107. status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
  1108. check(status == 200 and "quote-comment-dialog" in page and "quote-text" in page,
  1109. "task page carries the quote-comment dialog the popover opens")
  1110. check('name="quote"' in page and 'name="return_to"' in page,
  1111. "quote dialog posts the quote and return-to fields")
  1112. status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
  1113. check(status == 200 and "quote-comment-dialog" in page,
  1114. "terminal page keeps its quote dialog")
  1115. print("== B3: review section hides the duplicate body ==")
  1116. status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1117. {"type": "block", "session_id": session_id, "data": {
  1118. "question": "Review the diff",
  1119. "type": "review",
  1120. "body": "# The change\n\nReworks the flange module."}})
  1121. check(status == 200 and body.get("block_id"), "review block raised")
  1122. detail = task_state(auth, plan_task)
  1123. review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
  1124. check(detail.get("review_pending") is True,
  1125. "review section shows while the review is unanswered")
  1126. check(review_block.get("show_body") is False,
  1127. "pending review block hides its duplicate body preview")
  1128. check(review_block.get("body", "").startswith("# The change"),
  1129. "review body itself still arrives in state (B2 counterpart)")
  1130. act(auth, review_block["answer_action"],
  1131. {"answer": "Review approved. Continue with the task."})
  1132. detail = task_state(auth, plan_task)
  1133. review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
  1134. check(review_block.get("show_body") is True,
  1135. "answered review shows the body again (banner gone)")
  1136. print("== B2: question block body above options ==")
  1137. # Contract side (A2 will restore bodies end-to-end): a question
  1138. # block raised with a heredoc body carries it; the body preview
  1139. # rides above the options (template order) and survives answering
  1140. # alongside the selected answer.
  1141. detail = task_state(auth, plan_task)
  1142. question = [b for b in detail["blocks"] if b["block_type"] == "question"][0]
  1143. check(question.get("body") == "# Storage choice"
  1144. and question.get("show_body") is True,
  1145. "question block keeps its body markdown with the preview on")
  1146. act(auth, question["answer_action"], {"option": "SQLite"})
  1147. detail = task_state(auth, plan_task)
  1148. question = [b for b in detail["blocks"] if b["block_type"] == "question"][0]
  1149. check(question.get("answered") is True and question.get("answer") == "SQLite"
  1150. and question.get("body") == "# Storage choice"
  1151. and question.get("show_body") is True,
  1152. "answered question shows body + selected answer")
  1153. print("== B5: compare-changes SSH -> HTTPS ==")
  1154. def diff_url_for_project(git_url, suffix):
  1155. state = slot_public(auth, "/projects", "projects")
  1156. act(auth, state["create"], {"name": "diff-%s" % suffix, "git_url": git_url})
  1157. project_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  1158. if p["name"] == "diff-%s" % suffix][0])
  1159. conn = db()
  1160. conn.execute("insert into tasks (project_id, title, status, task_branch,"
  1161. " branch_from, created_at) values (?, ?, 'working', 'feature/x',"
  1162. " 'main', strftime('%s','now'))", (project_id, "Diff %s" % suffix))
  1163. conn.commit()
  1164. task_id = conn.execute("select max(id) from tasks").fetchone()[0]
  1165. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at,"
  1166. " outcome) values (?, ?, strftime('%s','now'), strftime('%s','now'),"
  1167. " 'done')", (task_id, fake_id))
  1168. conn.commit()
  1169. session = conn.execute("select max(id) from sessions").fetchone()[0]
  1170. conn.close()
  1171. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1172. {"type": "block", "session_id": session, "data": {
  1173. "question": "Review the diff", "type": "review"}})
  1174. return task_state(auth, task_id).get("review_diff_url") or ""
  1175. check(diff_url_for_project("ssh://fabrica.unitatem.net:222/clanker/clanker-saddle.git",
  1176. "fabrica")
  1177. == "https://fabrica.unitatem.net/clanker/clanker-saddle/compare/main...feature/x",
  1178. "ssh://git@host:port/owner/repo.git translates (port dropped)")
  1179. check(diff_url_for_project("git@github.com:acme/widgets.git", "scp")
  1180. == "https://github.com/acme/widgets/compare/main...feature/x",
  1181. "git@host:owner/repo.git translates to the browse host")
  1182. check(diff_url_for_project("https://example.com/plain/repo", "plain")
  1183. == "https://example.com/plain/repo/compare/main...feature/x",
  1184. "HTTPS URLs pass through unchanged")
  1185. print("== B6: computed display state ==")
  1186. # Draft tasks display as draft (plan 11) and never dispatch.
  1187. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1188. "project_id": "1",
  1189. "title": "Draft display task",
  1190. "priority": "5",
  1191. })
  1192. draft_display = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1193. if t["title"] == "Draft display task"][0])
  1194. detail = task_state(auth, draft_display)
  1195. check(detail["status"] == "draft" and detail.get("display") == "draft",
  1196. "new tasks default to draft and display it")
  1197. check(detail.get("draft") is True and detail.get("enqueue") is not None
  1198. and detail.get("delete") is not None,
  1199. "draft tasks expose the enqueue + delete actions")
  1200. check(quiet_window(quiet_seconds(),
  1201. lambda: task_state(auth, draft_display)["status"] == "draft",
  1202. "draft task never dispatches"),
  1203. "a draft task is never dispatched or woken for")
  1204. # A suspended parent with subtasks still in draft shows the clear
  1205. # enqueue-pending wording (plan 11).
  1206. conn = db()
  1207. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  1208. " values (1, 'Draft-waiting parent', 'suspended', 5, strftime('%s','now'))")
  1209. waiting_parent = conn.execute("select max(id) from tasks").fetchone()[0]
  1210. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  1211. " created_at) values (1, 'Draft child', 'draft', 5, ?,"
  1212. " strftime('%s','now'))", (waiting_parent,))
  1213. conn.commit()
  1214. conn.close()
  1215. detail = task_state(auth, waiting_parent)
  1216. check(detail.get("display") == "waiting on subtasks to be enqueued"
  1217. and detail.get("subtask_draft") == 1,
  1218. "suspended parent with draft children shows the enqueue-pending wording (%s)"
  1219. % detail.get("display"))
  1220. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1221. if int(t["id"]) == waiting_parent][0]
  1222. check(queue["display"] == "waiting on subtasks to be enqueued",
  1223. "queue row carries the enqueue-pending display too")
  1224. dash = slot_public(auth, "/", "dashboard")
  1225. check(any(int(w["task_id"]) == waiting_parent and int(w["draft_subtasks"]) == 1
  1226. for w in dash.get("waiting_drafts", [])),
  1227. "dashboard data carries the waiting-on-drafts block")
  1228. waiting_view = [w for w in dash.get("waiting_drafts", [])
  1229. if int(w["task_id"]) == waiting_parent][0]
  1230. check(len(waiting_view.get("drafts", [])) == 1,
  1231. "the waiting block links the draft subtask where Enqueue lives")
  1232. status, home_page = http("GET", "/", timeout=15)
  1233. check(status == 200 and "Waiting on subtasks to be enqueued" in home_page
  1234. and "stm-attribute.href=\"'/tasks/' + waiting.task_id\"" in home_page
  1235. and "stm-attribute.href=\"'/tasks/' + draft.id\"" in home_page,
  1236. "the dashboard renders the waiting-on-drafts block with task-page links")
  1237. conn = db()
  1238. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  1239. " values (1, 'Parent', 'done', 5, strftime('%s','now'))")
  1240. parent6 = conn.execute("select max(id) from tasks").fetchone()[0]
  1241. # The post-mortem case: stored done WITH an unanswered review block.
  1242. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
  1243. " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
  1244. (parent6, fake_id))
  1245. session6 = conn.execute("select max(id) from sessions").fetchone()[0]
  1246. conn.commit()
  1247. conn.close()
  1248. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1249. {"type": "block", "session_id": session6, "data": {
  1250. "question": "Review the final diff", "type": "review"}})
  1251. detail = task_state(auth, parent6)
  1252. check(detail.get("display") == "review requested" and detail.get("terminal") is True,
  1253. "unanswered review overrides the stored done status (terminal flag keeps stored truth)")
  1254. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1255. if int(t["id"]) == parent6][0]
  1256. check(queue["display"] == "review requested",
  1257. "queue row shows the computed review-requested display")
  1258. # Delegation: with the review answered, open subtasks delegate.
  1259. conn = db()
  1260. conn.execute("update task_blocks set answered_at = strftime('%s','now'),"
  1261. " answer = 'Human approved the review' where task_id = ?", (parent6,))
  1262. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  1263. " created_at) values (1, 'Sub A', 'working', 5, ?, strftime('%s','now'))",
  1264. (parent6,))
  1265. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  1266. " created_at) values (1, 'Sub B', 'done', 5, ?, strftime('%s','now'))",
  1267. (parent6,))
  1268. conn.commit()
  1269. conn.close()
  1270. detail = task_state(auth, parent6)
  1271. check(detail.get("display") == "delegating 1/2" and detail.get("subtask_open") == 1,
  1272. "open subtasks show the delegating breakdown (%s)" % detail.get("display"))
  1273. # Flags track the stored status once the display falls through
  1274. # (subtasks terminal first — delegating leads while any are open).
  1275. conn = db()
  1276. conn.execute("update tasks set status = 'failed', completed_at = strftime('%s','now')"
  1277. " where id = ?", (parent6,))
  1278. conn.execute("update tasks set status = 'failed', completed_at = strftime('%s','now')"
  1279. " where title = 'Sub A'")
  1280. conn.commit()
  1281. conn.close()
  1282. detail = task_state(auth, parent6)
  1283. check(detail.get("display") == "failed" and detail.get("failed") is True
  1284. and detail.get("terminal") is True,
  1285. "failed task with answered review and terminal subtasks shows failed + terminal flags")
  1286. print("== rejected entries (non-fatal) ==")
  1287. # Ingest API: a rejected entry stores on the session and streams
  1288. # through SSE; the task is untouched.
  1289. status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1290. {"type": "output", "session_id": session_id, "data": {
  1291. "command": "bash -n script.sh",
  1292. "output": "syntax error near line 3 (recoverable)",
  1293. "exit_code": 1,
  1294. "type": "rejected"}})
  1295. check(status == 200, "rejected entry accepted by the ingest API")
  1296. request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
  1297. with urllib.request.urlopen(request, timeout=15) as stream:
  1298. rejected_event = None
  1299. deadline = time.monotonic() + 8
  1300. while time.monotonic() < deadline and rejected_event is None:
  1301. line = stream.readline().decode().strip()
  1302. if not line.startswith("data: ") or line == "data: ping":
  1303. continue
  1304. event = json.loads(line[6:])
  1305. if event.get("type") == "rejected":
  1306. rejected_event = event
  1307. check(rejected_event is not None
  1308. and "syntax error near line 3" in (rejected_event.get("output") or ""),
  1309. "terminal stream carries rejected entries with their reason")
  1310. # The page template renders them with the distinct non-fatal style.
  1311. status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
  1312. check(status == 200 and "term-rejected" in page,
  1313. "terminal page renders rejected entries in their own style")
  1314. print("== B8: ANSI SGR in the terminal view ==")
  1315. # Byte-exact SGR delivery: the daemon's coloured chunks flow through
  1316. # the store + SSE untouched (the client renderer styles them).
  1317. coloured = "\x1b[1;32mbold green\x1b[0m then \x1b[38;5;196m256-red\x1b[0m\n"
  1318. conn = db()
  1319. conn.execute("insert into session_raw_log (session_id, seq, chunk)"
  1320. " values (?, 900, ?)", (session_id, coloured))
  1321. conn.commit()
  1322. conn.close()
  1323. request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
  1324. with urllib.request.urlopen(request, timeout=15) as stream:
  1325. coloured_event = None
  1326. deadline = time.monotonic() + 8
  1327. while time.monotonic() < deadline and coloured_event is None:
  1328. line = stream.readline().decode().strip()
  1329. if not line.startswith("data: ") or line == "data: ping":
  1330. continue
  1331. event = json.loads(line[6:])
  1332. if event.get("type") == "raw" and "\x1b[1;32m" in (event.get("text") or ""):
  1333. coloured_event = event
  1334. check(coloured_event is not None
  1335. and "\x1b[38;5;196m" in coloured_event["text"],
  1336. "SGR sequences stream byte-exact to the client renderer")
  1337. status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
  1338. check(status == 200 and "makeAnsiRenderer" in page and "colour256" in page,
  1339. "terminal page embeds the ANSI renderer (SGR + 256-colour palette)")
  1340. # Dispatch path: a rejected entry in a round's results push flows
  1341. # into the audit trail without failing (or finishing) the task.
  1342. task_id_tmp = create_queued(auth, {
  1343. "project_id": "1",
  1344. "title": "Rejected ride",
  1345. "priority": "2",
  1346. })
  1347. rejected_task = task_id_tmp
  1348. check(wait_for(lambda: task_state(auth, rejected_task).get("status") == "working"
  1349. or False, timeout=60, what="rejected-ride working") is not None,
  1350. "rejected-ride task reaches working")
  1351. fake.queue_output("rejected", "echo 'too long' | saddle",
  1352. "max_tokens exceeded (recoverable)")
  1353. check(wait_task_done(auth, rejected_task, timeout=60) is not None,
  1354. "task completes normally past rejected entries")
  1355. conn = db()
  1356. stored = conn.execute(
  1357. "select count(*) from session_commands sc join sessions s on s.id = sc.session_id"
  1358. " where s.task_id = ? and sc.type = 'rejected'", (rejected_task,)).fetchone()[0]
  1359. conn.close()
  1360. check(stored >= 1,
  1361. "rejected entries stored in the audit trail (%d rows)" % stored)
  1362. check(task_state(auth, rejected_task)["status"] == "done"
  1363. and "fake daemon finished" in (task_state(auth, rejected_task).get("summary") or ""),
  1364. "rejected entries never fail the task")
  1365. print("== plan 11 task D: task page — progress first, 50/50 grid, expanders ==")
  1366. detail = task_state(auth, plan_task)
  1367. block_ids = [int(b["id"]) for b in detail.get("blocks", [])]
  1368. check(len(block_ids) >= 3 and block_ids == sorted(block_ids, reverse=True),
  1369. "the blocks section orders newest-first via the sealed GET state (%s)"
  1370. % block_ids)
  1371. status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
  1372. # Plan 11 task D: the progress feed sits ABOVE the status card;
  1373. # status + description share a 50/50 grid; the plan detail hides
  1374. # behind its expander below it.
  1375. check(status == 200 and "grid-2" in page
  1376. and page.index("<h3>Progress</h3>") < page.index("status-head"),
  1377. "the progress feed renders above the status card")
  1378. check(page.index("grid-2") < page.index("status-head")
  1379. and page.index("status-head") < page.index("<h3>Description</h3>"),
  1380. "status and description sit side by side inside the grid (status first)")
  1381. check("<h3>Description</h3>" in page
  1382. and page.index("<h3>Description</h3>") < page.index("plan-expander"),
  1383. "the plan detail hides behind its expander below the grid")
  1384. # Answered blocks fold behind the history expander (this fixture's
  1385. # three blocks are all answered by now).
  1386. check("blocks-history" in page and "Answered blocks" in page,
  1387. "answered blocks hide behind the blocks history expander")
  1388. conn = db()
  1389. conn.execute("insert into tasks (project_id, title, status, priority,"
  1390. " created_at) values (1, 'Progress layout task', 'working', 5,"
  1391. " strftime('%s','now','-120 seconds'))")
  1392. layout_task = conn.execute("select max(id) from tasks").fetchone()[0]
  1393. # backdated so the dashboard-timeline fixtures below stay the
  1394. # newest events
  1395. conn.execute("insert into task_progress (task_id, summary, created_at)"
  1396. " values (?, 'first update', strftime('%s','now','-90 seconds'))",
  1397. (layout_task,))
  1398. conn.execute("insert into task_progress (task_id, summary, created_at)"
  1399. " values (?, 'second update', strftime('%s','now','-89 seconds'))",
  1400. (layout_task,))
  1401. conn.commit()
  1402. conn.close()
  1403. status, history_page = http("GET", "/tasks/%d" % layout_task, timeout=15)
  1404. check(status == 200
  1405. and history_page.index("progress-latest") < history_page.index("progress-history"),
  1406. "the progress timeline renders newest-first (latest above the history)")
  1407. # The edit expander sits UNDER the "Show history" expander.
  1408. check(history_page.index("Show history") < history_page.index("edit-expander"),
  1409. "the edit expander sits under the Show history expander")
  1410. print("== heredoc completion reports: markdown card + timeline snippet ==")
  1411. # The daemon's `cat << 'EOF' | saddle done` stores a markdown
  1412. # summary (the failure path stores the same field); the task page
  1413. # renders it as a full-width completion-report card directly below
  1414. # the status/description grid — markdown via stm-markdown
  1415. # (XSS-safe), "Completion report" when done, "Failure summary"
  1416. # when failed, no card when the summary is empty. The dashboard
  1417. # timeline snippets the summary's FIRST line.
  1418. report_md = ("## Flange wired and tested\n\n"
  1419. "Ran `meson test` over the full matrix.\n\n"
  1420. "```sh\nmeson test -C builddir\n```\n\n"
  1421. "<b>not html</b> & escaped\n")
  1422. conn = db()
  1423. conn.execute("insert into tasks (project_id, title, status, priority,"
  1424. " summary, started_at, completed_at, created_at)"
  1425. " values (1, 'Report done task', 'done', 5, ?,"
  1426. " strftime('%s','now'), strftime('%s','now'),"
  1427. " strftime('%s','now'))", (report_md,))
  1428. report_done = conn.execute("select max(id) from tasks").fetchone()[0]
  1429. conn.execute("insert into tasks (project_id, title, status, priority,"
  1430. " summary, started_at, completed_at, created_at)"
  1431. " values (1, 'Report failed task', 'failed', 5,"
  1432. " 'Boom: tests failed\n\nsecond line carries the detail',"
  1433. " strftime('%s','now'), strftime('%s','now'),"
  1434. " strftime('%s','now'))")
  1435. report_failed = conn.execute("select max(id) from tasks").fetchone()[0]
  1436. conn.execute("insert into tasks (project_id, title, status, priority,"
  1437. " started_at, completed_at, created_at)"
  1438. " values (1, 'Report empty task', 'done', 5,"
  1439. " strftime('%s','now'), strftime('%s','now'),"
  1440. " strftime('%s','now'))")
  1441. report_empty = conn.execute("select max(id) from tasks").fetchone()[0]
  1442. conn.commit()
  1443. conn.close()
  1444. done_state = task_state(auth, report_done)
  1445. check(done_state["report_title"] == "Completion report"
  1446. and "## Flange wired" in (done_state.get("summary") or ""),
  1447. "a done task's summary rides the report card state")
  1448. check(task_state(auth, report_failed)["report_title"] == "Failure summary",
  1449. "a failed task's card heading reads Failure summary")
  1450. empty_state = task_state(auth, report_empty)
  1451. check(empty_state["report_title"] == "" and not (empty_state.get("summary") or ""),
  1452. "an empty summary renders no report card (no heading)")
  1453. check(task_state(auth, layout_task)["report_title"] == "",
  1454. "non-terminal tasks never show the report card")
  1455. status, page = http("GET", "/tasks/%d" % report_done, timeout=15)
  1456. check(status == 200 and "completion-report" in page
  1457. and page.index("grid-2") < page.index("completion-report")
  1458. < page.index("edit-expander"),
  1459. "the completion-report card sits below the status/description grid, full width")
  1460. card = page[page.index("completion-report"):page.index("edit-expander")]
  1461. check('stm-markdown="task.summary"' in card
  1462. and 'stm-text="task.report_title"' in card,
  1463. "the card renders the summary as markdown under its state-driven heading")
  1464. check("<strong>Summary:</strong>" not in page,
  1465. "the old plain-text summary line is gone (the card owns the summary)")
  1466. # The markdown itself renders client-side: drive the SHIPPED
  1467. # stm-markdown renderer (statum.js) on the fixture summary via
  1468. # node — headings and code come out as HTML, no markdown syntax
  1469. # leaks, raw HTML stays escaped.
  1470. status, statum_js = http("GET", "/_statum/resource/statum.js", timeout=15)
  1471. skip = None if status == 200 else "statum.js not served"
  1472. parts = []
  1473. if skip is None:
  1474. def extract_md(name, source):
  1475. start = source.index("function %s(" % name)
  1476. depth = 0
  1477. index = source.index("{", start)
  1478. end = index
  1479. while True:
  1480. if source[end] == "{":
  1481. depth += 1
  1482. elif source[end] == "}":
  1483. depth -= 1
  1484. if depth == 0:
  1485. break
  1486. end += 1
  1487. return source[start:end + 1]
  1488. try:
  1489. parts = [extract_md(name, statum_js)
  1490. for name in ("safeMarkdownUrl", "mdLinks", "mdInline", "renderMarkdown")]
  1491. except ValueError:
  1492. skip = "renderer functions missing"
  1493. rendered = ""
  1494. if skip is None:
  1495. try:
  1496. result = subprocess.run(
  1497. ["node", "-e", "%s\n%s\n%s\n%s\nconsole.log(renderMarkdown(process.argv[1]));"
  1498. % tuple(parts), report_md],
  1499. capture_output=True, text=True, timeout=20)
  1500. rendered = result.stdout
  1501. except (OSError, subprocess.SubprocessError):
  1502. skip = "node unavailable"
  1503. if skip is not None:
  1504. check(True, "stm-markdown renderer present (%s, deep check skipped)" % skip)
  1505. else:
  1506. check("<h2>Flange wired and tested</h2>" in rendered
  1507. and "<pre><code" in rendered and "<code>meson test</code>" in rendered,
  1508. "the report summary renders headings and code through stm-markdown")
  1509. check("## Flange" not in rendered and "```sh" not in rendered
  1510. and "`meson test`" not in rendered,
  1511. "no raw markdown syntax leaks into the rendered report")
  1512. check("<b>not html</b>" not in rendered and "&lt;b&gt;not html&lt;/b&gt;" in rendered
  1513. and "&amp; escaped" in rendered,
  1514. "raw HTML in the summary stays escaped (XSS-safe)")
  1515. # The dashboard timeline: completion/failure events snippet the
  1516. # summary's first line, falling back to the bare verdict text.
  1517. dash = slot_public(auth, "/", "dashboard")
  1518. done_event = next((e for e in dash.get("activity", [])
  1519. if e["kind"] == "completed" and int(e["task_id"]) == report_done), None)
  1520. check(done_event is not None
  1521. and done_event["text"] == "Task completed: ## Flange wired and tested",
  1522. "the timeline completion event snippets the summary's first line")
  1523. failed_event = next((e for e in dash.get("activity", [])
  1524. if e["kind"] == "failed" and int(e["task_id"]) == report_failed), None)
  1525. check(failed_event is not None
  1526. and failed_event["text"] == "Task failed: Boom: tests failed",
  1527. "the timeline failure event snippets the summary's first line")
  1528. empty_event = next((e for e in dash.get("activity", [])
  1529. if e["kind"] == "completed" and int(e["task_id"]) == report_empty), None)
  1530. check(empty_event is not None and empty_event["text"] == "Task completed",
  1531. "a missing summary falls back to the bare timeline text")
  1532. print("== plan 11 task D: block split + queue-screen layout ==")
  1533. # One open and one answered question: the open block renders in
  1534. # full, the answered one folds behind the history expander.
  1535. conn = db()
  1536. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  1537. " values (?, ?, strftime('%s','now'))", (layout_task, fake_id))
  1538. conn.commit()
  1539. split_session = conn.execute("select max(id) from sessions").fetchone()[0]
  1540. conn.close()
  1541. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1542. {"type": "block", "session_id": split_session, "data": {
  1543. "question": "Open question for layout?", "type": "question"}})
  1544. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1545. {"type": "block", "session_id": split_session, "data": {
  1546. "question": "Answered question for layout?", "type": "question"}})
  1547. answered_block = [b for b in task_state(auth, layout_task)["blocks"]
  1548. if b["question"] == "Answered question for layout?"][0]
  1549. act(auth, answered_block["answer_action"], {"answer": "Yes"})
  1550. detail = task_state(auth, layout_task)
  1551. check(len(detail.get("open_blocks", [])) == 1
  1552. and len(detail.get("answered_blocks", [])) == 1
  1553. and detail["open_blocks"][0]["question"] == "Open question for layout?"
  1554. and detail["answered_blocks"][0]["question"] == "Answered question for layout?",
  1555. "the task slot splits blocks into open (current) and answered lists")
  1556. status, page = http("GET", "/tasks/%d" % layout_task, timeout=15)
  1557. check(status == 200
  1558. and page.index('stm-for-block-in="task.open_blocks"') < page.index("blocks-history")
  1559. < page.index('stm-for-block-in="task.answered_blocks"'),
  1560. "current blocks render in full above the answered-behind-expander history")
  1561. check(page.index("status-head") < page.index("Cancel task"),
  1562. "the lifecycle buttons ride the status card inside the grid")
  1563. status, queue_page = http("GET", "/tasks", timeout=15)
  1564. check(status == 200
  1565. and queue_page.index("new-task-expander") < queue_page.index("filter-status")
  1566. < queue_page.index("<table"),
  1567. "the New Task expander sits above the filter controls and the table")
  1568. print("== yeehaw: ingestion, festive card, badge, timeline ==")
  1569. # `saddle yeehaw <message>` arrives as a round-results entry with the
  1570. # message in the output field: it stores like any entry type, earns
  1571. # the task its 🤠 badge (derived, nothing new stored), renders as the
  1572. # task page's festive card (last 3, newest first, "+N more") and
  1573. # surfaces on the dashboard timeline as its own event kind.
  1574. conn = db()
  1575. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  1576. " values (1, 'Yeehaw task', 'working', 5, strftime('%s','now'))")
  1577. yeehaw_task = conn.execute("select max(id) from tasks").fetchone()[0]
  1578. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  1579. " values (?, ?, strftime('%s','now'))", (yeehaw_task, fake_id))
  1580. yeehaw_session = conn.execute("select max(id) from sessions").fetchone()[0]
  1581. conn.commit()
  1582. conn.close()
  1583. now_epoch = int(time.time())
  1584. def push_yeehaw(entry_id, message, offset=0):
  1585. """One celebration riding a round-results push, timestamped
  1586. `offset` seconds from now (distinct epochs keep the timeline's
  1587. newest-first ordering observable)."""
  1588. return saddle_api("POST", "/api/saddle/results", fake_token, {
  1589. "session_id": yeehaw_session, "state": "working",
  1590. "entries": [{
  1591. "id": entry_id, "command": "saddle yeehaw", "output": message,
  1592. "exit_code": 0, "type": "yeehaw",
  1593. "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ",
  1594. time.gmtime(now_epoch + offset)),
  1595. }]})[0]
  1596. def yeehaw_rows():
  1597. conn = db()
  1598. try:
  1599. return conn.execute(
  1600. "select count(*) from session_commands where session_id = ?"
  1601. " and type = 'yeehaw'", (yeehaw_session,)).fetchone()[0]
  1602. finally:
  1603. conn.close()
  1604. check(push_yeehaw(9101, "Shipped the flange") == 200,
  1605. "yeehaw entries ingest through the results push like any entry type")
  1606. check(yeehaw_rows() == 1,
  1607. "the yeehaw entry stores as a session_commands row (message in output)")
  1608. check(push_yeehaw(9101, "Shipped the flange") == 200 and yeehaw_rows() == 1,
  1609. "a redelivered yeehaw push is idempotent on the daemon entry id")
  1610. check(task_state(auth, yeehaw_task)["status"] == "working",
  1611. "a yeehaw entry never derails the task state")
  1612. push_yeehaw(9102, "Second hoedown", 1)
  1613. push_yeehaw(9103, "Third hoedown\nsecond line carries the detail", 2)
  1614. push_yeehaw(9104, "Fourth hoedown", 3)
  1615. push_yeehaw(9105, "Fifth hoedown", 4)
  1616. detail = task_state(auth, yeehaw_task)
  1617. check(detail.get("yeehaw") is True and detail.get("yeehaw_total") == 5,
  1618. "the detail state flags the badge presence and the uncapped total")
  1619. check([y["message"] for y in detail.get("yeehaws", [])]
  1620. == ["Fifth hoedown", "Fourth hoedown",
  1621. "Third hoedown\nsecond line carries the detail"],
  1622. "the festive card keeps the last 3 yeehaws, newest first (full messages)")
  1623. check(detail.get("yeehaw_more") == 2,
  1624. "the '+N more' hint counts the capped tail")
  1625. status, page = http("GET", "/tasks/%d" % yeehaw_task, timeout=15)
  1626. # The template compiler HTML-escapes the emoji (&#129312; = 🤠).
  1627. check(status == 200 and "yeehaw-card" in page and "&#129312; Yeehaw!" in page,
  1628. "the task page renders the festive yeehaw card")
  1629. check(page.index("<h3>Progress</h3>") < page.index("yeehaw-card")
  1630. < page.index("status-head"),
  1631. "the card sits between the progress feed and the status grid")
  1632. check('stm-text="task.yeehaw_more"' in page and "more yeehaw" in page,
  1633. "the card carries the +N more hint")
  1634. check("badge-yeehaw" in page and "status-corner" in page,
  1635. "the status card corner carries the earned 🤠 badge")
  1636. plain_detail = task_state(auth, layout_task)
  1637. check(plain_detail.get("yeehaw") is False and plain_detail.get("yeehaw_total") == 0
  1638. and plain_detail.get("yeehaws", []) == [],
  1639. "tasks without yeehaw entries carry no card state")
  1640. status, plain_page = http("GET", "/tasks/%d" % layout_task, timeout=15)
  1641. check(status == 200 and 'section class="card yeehaw-card" stm-if="task.yeehaw"' in plain_page,
  1642. "the card template only renders under state gating (stm-if task.yeehaw)")
  1643. queue_rows = slot_public(auth, "/tasks", "tasks")["tasks"]
  1644. check([t for t in queue_rows if int(t["id"]) == yeehaw_task][0].get("has_yeehaw") is True,
  1645. "queue rows badge yeehaw tasks")
  1646. check([t for t in queue_rows if int(t["id"]) == layout_task][0].get("has_yeehaw") is False,
  1647. "queue rows leave yeehaw-less tasks bare")
  1648. status, queue_page = http("GET", "/tasks", timeout=15)
  1649. check(status == 200 and 'stm-if="task.has_yeehaw"' in queue_page,
  1650. "the queue template renders the row badge")
  1651. dash = slot_public(auth, "/", "dashboard")
  1652. yeehaw_events = [e for e in dash.get("activity", []) if e["kind"] == "yeehaw"]
  1653. check(len(yeehaw_events) == 5
  1654. and all(int(e["task_id"]) == yeehaw_task for e in yeehaw_events),
  1655. "every yeehaw entry surfaces as its own timeline event kind")
  1656. check(yeehaw_events[0]["text"] == "🤖 yeehaw'd: Fifth hoedown",
  1657. "the newest yeehaw event leads the timeline rows")
  1658. check(all("second line" not in e["text"] for e in yeehaw_events),
  1659. "multiline yeehaw messages snippet to their first line")
  1660. referenced = [e for e in dash.get("activity", []) if int(e["task_id"]) == yeehaw_task]
  1661. check(len(referenced) > 0 and all(e.get("task_yeehaw") is True for e in referenced),
  1662. "dashboard timeline task references carry the badge flag")
  1663. plain_refs = [e for e in dash.get("activity", []) if int(e["task_id"]) == layout_task]
  1664. check(len(plain_refs) > 0 and all(e.get("task_yeehaw") is False for e in plain_refs),
  1665. "references to yeehaw-less tasks stay bare")
  1666. status, home_page = http("GET", "/", timeout=15)
  1667. check(status == 200 and 'stm-if="item.task_yeehaw"' in home_page
  1668. and "badge-yeehaw" in home_page,
  1669. "the dashboard renders the badge on timeline task references")
  1670. print("== R10: branch_advanced on the /poll verdict channel ==")
  1671. conn = db()
  1672. conn.execute("insert into tasks (project_id, title, status, priority,"
  1673. " created_at) values (1, 'Poll grandparent', 'working', 5,"
  1674. " strftime('%s','now','-300 seconds'))")
  1675. poll_grand = conn.execute("select max(id) from tasks").fetchone()[0]
  1676. conn.execute("insert into tasks (project_id, title, status, priority,"
  1677. " parent_task_id, created_at) values (1, 'Poll answer parent',"
  1678. " 'blocked', 5, ?, strftime('%s','now','-120 seconds'))",
  1679. (poll_grand,))
  1680. poll_parent = conn.execute("select max(id) from tasks").fetchone()[0]
  1681. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  1682. " values (?, ?, strftime('%s','now','-120 seconds'))",
  1683. (poll_parent, fake_id))
  1684. conn.execute("insert into task_blocks (task_id, question, type, answer,"
  1685. " answered_at, created_at) values (?, 'Which DB?', 'question',"
  1686. " 'SQLite', strftime('%s','now','-60 seconds'),"
  1687. " strftime('%s','now','-100 seconds'))", (poll_parent,))
  1688. conn.execute("update saddles set status = 'busy', current_task_id = ?,"
  1689. " allocated_at = strftime('%s','now') where name = 'fake-saddle'",
  1690. (poll_parent,))
  1691. conn.commit()
  1692. conn.close()
  1693. status, body = saddle_api("GET", "/api/saddle/%d/poll" % fake_id, fake_token)
  1694. check(status == 200 and len(body.get("answers", [])) == 1
  1695. and body["answers"][0].get("branch_advanced") is False,
  1696. "/poll verdicts carry branch_advanced=false with no siblings")
  1697. # A sibling under the same parent completed after the park marker.
  1698. conn = db()
  1699. conn.execute("insert into tasks (project_id, title, status, priority,"
  1700. " parent_task_id, summary, completed_at, created_at)"
  1701. " values (1, 'Poll sibling', 'done', 5, ?, 'sibling finished',"
  1702. " strftime('%s','now'), strftime('%s','now','-200 seconds'))",
  1703. (poll_grand,))
  1704. conn.execute("insert into task_blocks (task_id, question, type, answer,"
  1705. " answered_at, created_at) values (?, 'Second question?',"
  1706. " 'question', 'Yes', strftime('%s','now'),"
  1707. " strftime('%s','now','-90 seconds'))", (poll_parent,))
  1708. conn.commit()
  1709. conn.close()
  1710. status, body = saddle_api("GET", "/api/saddle/%d/poll" % fake_id, fake_token)
  1711. check(status == 200 and len(body.get("answers", [])) == 1
  1712. and body["answers"][0].get("branch_advanced") is True,
  1713. "sibling completion after park flips branch_advanced to true")
  1714. # Release the staged fixture: cancel it so its busy@task row does
  1715. # not linger into the sections below.
  1716. act(auth, task_state(auth, poll_parent)["cancel"], {})
  1717. # =============================================================================
  1718. # Bundle: dispatch — the task pipeline end to end: registration tokens,
  1719. # rules, project notes, pushed chunks/results ingestion, capability
  1720. # routing, the create form, and retry/manual resume.
  1721. # =============================================================================
  1722. def bundle_dispatch(ctx):
  1723. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  1724. app = ctx.app
  1725. print("== registration: token minting and delivery ==")
  1726. registered = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  1727. if s["name"] == "fake-saddle"]
  1728. check(len(registered) == 1 and registered[0]["token"].startswith("saddle-"),
  1729. "fake saddle registered with an API token")
  1730. # Plan 10 phase 1: registration delivers the per-saddle token to
  1731. # the daemon via POST /register {saddle_token, web_base_url}.
  1732. check(wait_for(lambda: len(fake.register_log) > 0 or False,
  1733. timeout=15, what="/register delivery") is not None,
  1734. "registration POSTs /register to the daemon")
  1735. register_body = fake.register_log[0]
  1736. check(register_body.get("saddle_token") == fake_token,
  1737. "/register carries the minted per-saddle token")
  1738. check(register_body.get("web_base_url") == WEB,
  1739. "/register carries the web base URL")
  1740. check(fake.token == fake_token,
  1741. "daemon persisted the token (its asks use it)")
  1742. token_conn = db()
  1743. row_token = token_conn.execute("select token from saddles where id = ?",
  1744. (fake_id,)).fetchone()[0]
  1745. token_conn.close()
  1746. check(row_token == fake_token,
  1747. "per-saddle token lives on the saddle row")
  1748. # The register-time ask (idle-awake daemon): 204, nothing
  1749. # offerable yet — the saddle is asleep and wakeable.
  1750. check(wait_for(lambda: any(a["status"] == 204 for a in fake.asks) or False,
  1751. timeout=15, what="register-time ask") is not None,
  1752. "the daemon's idle ask returns 204 (nothing offerable)")
  1753. print("== feature 5: rules ==")
  1754. rules_state = slot_public(auth, "/rules", "rules")
  1755. for name, body in [
  1756. ("Use Invercargill", "Never use libgee; use Invercargill enumerables."),
  1757. ("Write Vala docs", "Every public class carries a doc comment."),
  1758. ("Prefer meson", "Builds go through meson; no makefiles."),
  1759. ("Co-dev rule", "Keep the wire contract exactly."),
  1760. ("Reference rule", "Must never ride along."),
  1761. ]:
  1762. act(auth, rules_state["create"], {"name": name, "body": body})
  1763. listed = {r["name"]: r for r in slot_public(auth, "/rules", "rules")["rules"]}
  1764. check(len(listed) == 5, "five rules created and listed on /rules")
  1765. def rule_id(name):
  1766. rows = [r for r in slot_public(auth, "/rules", "rules")["rules"] if r["name"] == name]
  1767. return int(rows[0]["id"])
  1768. state = slot_public(auth, "/projects", "projects")
  1769. act(auth, state["create"], {"name": "co", "git_url": "https://example.com/co.git"})
  1770. act(auth, state["create"], {"name": "ref", "git_url": "https://example.com/ref.git"})
  1771. co_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  1772. if p["name"] == "co"][0])
  1773. ref_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  1774. if p["name"] == "ref"][0])
  1775. project = slot_public(auth, "/projects/1", "project")
  1776. act(auth, project["add_link"], {"linked_project_id": str(co_id),
  1777. "relationship": "co-developed"})
  1778. act(auth, project["add_link"], {"linked_project_id": str(ref_id),
  1779. "relationship": "reference"})
  1780. act(auth, project["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
  1781. act(auth, project["attach_rule"], {"rule_id": str(rule_id("Prefer meson"))})
  1782. project = slot_public(auth, "/projects/1", "project")
  1783. check(sorted(r["name"] for r in project["rules"]) == ["Prefer meson", "Use Invercargill"],
  1784. "project page lists its attached rules")
  1785. co_project = slot_public(auth, "/projects/%d" % co_id, "project")
  1786. act(auth, co_project["attach_rule"], {"rule_id": str(rule_id("Co-dev rule"))})
  1787. ref_project = slot_public(auth, "/projects/%d" % ref_id, "project")
  1788. act(auth, ref_project["attach_rule"], {"rule_id": str(rule_id("Reference rule"))})
  1789. tasks_state = slot_public(auth, "/tasks", "tasks")
  1790. task_id_tmp = create_queued(auth, {
  1791. "project_id": "1",
  1792. "title": "Rules dispatch",
  1793. "description": "Effective rule set rides the /start payload.",
  1794. "priority": "2",
  1795. })
  1796. rule_task = task_id_tmp
  1797. detail = task_state(auth, rule_task)
  1798. act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Write Vala docs"))})
  1799. act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
  1800. detail = task_state(auth, rule_task)
  1801. check(sorted(r["name"] for r in detail["rules"]) == ["Use Invercargill", "Write Vala docs"],
  1802. "task page lists its attached rules")
  1803. check(wait_for(lambda: len(fake.start_payloads) > 0 or False,
  1804. timeout=30, what="/start push") is not None,
  1805. "dispatcher pushes the task to the fake saddle")
  1806. payload = fake.start_payloads[-1]
  1807. rule_names = [r["name"] for r in payload.get("rules", [])]
  1808. check(rule_names == ["Write Vala docs", "Use Invercargill", "Prefer meson", "Co-dev rule"],
  1809. "effective rules in /start: task, primary, co-developed order (%s)" % rule_names)
  1810. check(all(r.get("body") for r in payload.get("rules", [])),
  1811. "rules carry their bodies")
  1812. check("Reference rule" not in rule_names,
  1813. "reference project rules excluded")
  1814. detail = task_state(auth, rule_task)
  1815. act(auth, detail["rules"][0]["detach"], {})
  1816. check([r["name"] for r in task_state(auth, rule_task)["rules"]] == ["Use Invercargill"],
  1817. "detached task rule disappears")
  1818. rules_rows = slot_public(auth, "/rules", "rules")["rules"]
  1819. docs_row = [r for r in rules_rows if r["name"] == "Write Vala docs"][0]
  1820. act(auth, docs_row["update"], {"name": "Write Vala docs", "body": "Doc comments or bust."})
  1821. check(any(r["body"] == "Doc comments or bust."
  1822. for r in slot_public(auth, "/rules", "rules")["rules"]),
  1823. "rule edit saves the body")
  1824. check(wait_task_done(auth, rule_task, timeout=60) is not None,
  1825. "dispatched task completes against the fake daemon")
  1826. print("== feature 6: project notes + callback ==")
  1827. # Plan 10 phase 1: the shared saddle-callback token is gone —
  1828. # the per-saddle token authorises the callback endpoints.
  1829. callback_token = fake_token
  1830. check(callback_token.startswith("saddle-") and "saddle-callback-" not in callback_token,
  1831. "note callback authorised by the per-saddle token")
  1832. status, _ = saddle_api("POST", "/api/saddle/note", None,
  1833. {"project": "demo", "body": "x"})
  1834. check(status == 401, "note endpoint rejects a missing token")
  1835. status, _ = saddle_api("POST", "/api/saddle/note", "saddle-callback-not-real",
  1836. {"project": "demo", "body": "x"})
  1837. check(status == 401, "note endpoint rejects a wrong (legacy-style) token")
  1838. status, body = saddle_api("POST", "/api/saddle/note", callback_token,
  1839. {"task_id": rule_task, "project": "demo",
  1840. "body": "style.css tokens live in :root"})
  1841. check(status == 200 and body.get("ok") is True, "note accepted with the callback token")
  1842. status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
  1843. {"project": "no-such-project", "body": "x"})
  1844. check(status == 400, "unknown project rejected")
  1845. status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
  1846. {"project": "demo", "body": " "})
  1847. check(status == 400, "empty note body rejected")
  1848. project = slot_public(auth, "/projects/1", "project")
  1849. check(any("style.css tokens" in n["body"] and int(n["source_task_id"]) == rule_task
  1850. for n in project["notes"]),
  1851. "project page lists the note with its source task")
  1852. act(auth, [n for n in project["notes"] if "style.css tokens" in n["body"]][0]["delete"], {})
  1853. check(all("style.css tokens" not in n["body"]
  1854. for n in slot_public(auth, "/projects/1", "project")["notes"]),
  1855. "note deleted from the project page")
  1856. # /start payload: notes of primary + co-developed (not reference),
  1857. # oldest first; callback carries the configured base URL + token.
  1858. saddle_api("POST", "/api/saddle/note", callback_token,
  1859. {"project": "demo", "body": "note-one"})
  1860. saddle_api("POST", "/api/saddle/note", callback_token,
  1861. {"project": "demo", "body": "note-two"})
  1862. saddle_api("POST", "/api/saddle/note", callback_token,
  1863. {"project": "co", "body": "co-note"})
  1864. conn = db()
  1865. conn.execute("insert into project_notes (project_id, body, created_at)"
  1866. " values (?, 'ref-note-must-not-ship', strftime('%s','now'))", (ref_id,))
  1867. conn.commit()
  1868. conn.close()
  1869. task_id_tmp = create_queued(auth, {
  1870. "project_id": "1",
  1871. "title": "Notes dispatch",
  1872. "description": "Project notes ride the /start payload.",
  1873. "priority": "2",
  1874. })
  1875. check(wait_for(lambda: len(fake.start_payloads) >= 2 or False,
  1876. timeout=30, what="second /start push") is not None,
  1877. "dispatcher pushes the notes task")
  1878. payload = fake.start_payloads[-1]
  1879. notes = payload.get("project_notes", [])
  1880. check([n["body"] for n in notes] == ["note-one", "note-two", "co-note"],
  1881. "project_notes: primary + co-developed, oldest first (%s)" % [n["body"] for n in notes])
  1882. check(all(n.get("project") for n in notes)
  1883. and notes[2]["project"] == "co",
  1884. "project_notes carry the project name")
  1885. check("ref-note-must-not-ship" not in [n["body"] for n in notes],
  1886. "reference project notes excluded")
  1887. callback = payload.get("callback", {})
  1888. check(callback.get("base_url") == WEB and callback.get("token") == callback_token,
  1889. "callback carries the configured base URL and the minted token")
  1890. # entry cap: 30 of 33 candidates
  1891. conn = db()
  1892. for i in range(33):
  1893. conn.execute("insert into project_notes (project_id, body, created_at)"
  1894. " values (?, ?, strftime('%s','now'))", (1, "bulk-%02d" % i))
  1895. conn.commit()
  1896. conn.close()
  1897. create_queued(auth, {
  1898. "project_id": "1",
  1899. "title": "Entry cap dispatch",
  1900. "priority": "2",
  1901. })
  1902. check(wait_for(lambda: len(fake.start_payloads) >= 3 or False,
  1903. timeout=30, what="third /start push") is not None,
  1904. "dispatcher pushes the entry-cap task")
  1905. notes = fake.start_payloads[-1].get("project_notes", [])
  1906. check(len(notes) == 30 and notes[0]["body"] == "note-one" and notes[1]["body"] == "note-two",
  1907. "project_notes capped at 30 entries, oldest kept (%d)" % len(notes))
  1908. # char cap: the giant note is truncated so the total stays <= 20000
  1909. conn = db()
  1910. conn.execute("delete from project_notes where body like 'bulk-%' or body = 'co-note'")
  1911. conn.execute("insert into project_notes (project_id, body, created_at)"
  1912. " values (?, ?, strftime('%s','now'))", (1, "giant!" + "x" * 25000,))
  1913. conn.commit()
  1914. conn.close()
  1915. create_queued(auth, {
  1916. "project_id": "1",
  1917. "title": "Char cap dispatch",
  1918. "priority": "2",
  1919. })
  1920. check(wait_for(lambda: len(fake.start_payloads) >= 4 or False,
  1921. timeout=30, what="fourth /start push") is not None,
  1922. "dispatcher pushes the char-cap task")
  1923. notes = fake.start_payloads[-1].get("project_notes", [])
  1924. total = sum(len(n["body"]) for n in notes)
  1925. check(len(notes) == 3 and total == 20000 and notes[2]["body"].startswith("giant!"),
  1926. "project_notes char cap: head kept, giant truncated (entries=%d, chars=%d)"
  1927. % (len(notes), total))
  1928. print("== phase 2: pushed chunks -> raw log + terminal view ==")
  1929. # Wait for the last dispatch to finish so its pushes landed.
  1930. check(wait_task_done(auth, task_id_of("Char cap dispatch"), timeout=60) is not None,
  1931. "char-cap task completes")
  1932. conn = db()
  1933. raw_session = conn.execute(
  1934. "select s.id from sessions s join tasks t on t.id = s.task_id"
  1935. " where t.title = 'Char cap dispatch' order by s.id desc limit 1").fetchone()[0]
  1936. rows = conn.execute("select seq, chunk from session_raw_log where session_id = ?"
  1937. " order by seq", (raw_session,)).fetchall()
  1938. conn.close()
  1939. check(len(rows) >= 3, "session_raw_log captured the pushed chunks (%d)" % len(rows))
  1940. seqs = [r[0] for r in rows]
  1941. check(seqs == sorted(seqs) and len(set(seqs)) == len(seqs),
  1942. "pushed chunks stored once, seq strictly increasing")
  1943. text = "".join(r[1] for r in rows)
  1944. check(fake.PROMPT in text and "echo hello" in text and "hello" in text,
  1945. "pushed raw log keeps the prompt and echoed output byte-exact")
  1946. check("working-heartbeat" in text,
  1947. "the flush-timer heartbeat chunks keep arriving mid-round")
  1948. terminal = slot_public(auth, "/tasks/%d/terminal" % task_id_of("Char cap dispatch"),
  1949. "terminal")
  1950. check(terminal.get("has_raw_log") is True and int(terminal["session_id"]) == raw_session,
  1951. "terminal view selects raw rendering for the raw session")
  1952. # Old sessions (typed entries, no raw rows) keep the fallback view
  1953. # (a dedicated fixture task: its only session is the legacy one,
  1954. # so the terminal view cannot prefer a dispatched raw session).
  1955. conn = db()
  1956. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  1957. " values (1, 'Legacy terminal task', 'working', 5, strftime('%s','now'))")
  1958. legacy_task = conn.execute("select max(id) from tasks").fetchone()[0]
  1959. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
  1960. " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
  1961. (legacy_task, fake_id))
  1962. conn.commit()
  1963. legacy_session = conn.execute("select max(id) from sessions").fetchone()[0]
  1964. conn.execute("insert into session_commands (session_id, command_text, output_text,"
  1965. " type, started_at) values (?, 'ls', 'old-output', 'command',"
  1966. " strftime('%s','now'))", (legacy_session,))
  1967. conn.commit()
  1968. conn.close()
  1969. legacy = slot_public(auth, "/tasks/%d/terminal" % legacy_task, "terminal")
  1970. check(legacy.get("has_raw_log") is False,
  1971. "legacy session falls back to the entry-type terminal view")
  1972. request = urllib.request.Request(WEB + "/terminal-stream/%d" % raw_session)
  1973. with urllib.request.urlopen(request, timeout=15) as stream:
  1974. raw_events = []
  1975. deadline = time.monotonic() + 8
  1976. while time.monotonic() < deadline and len(raw_events) < 3:
  1977. line = stream.readline().decode().strip()
  1978. if not line.startswith("data: ") or line == "data: ping":
  1979. continue
  1980. event = json.loads(line[6:])
  1981. if event.get("type") == "raw":
  1982. raw_events.append(event)
  1983. check(len(raw_events) >= 3 and raw_events[0]["text"] == fake.PROMPT
  1984. and raw_events[0]["seq"] < raw_events[1]["seq"],
  1985. "terminal stream serves pushed chunks in seq order (prompt first)")
  1986. print("== phase 2: chunks/results ingestion — auth, idempotency ==")
  1987. # Token auth on both push channels (plan 10: the per-saddle
  1988. # bearer authorises every daemon->web call).
  1989. status, _ = saddle_api("POST", "/api/saddle/chunks", None,
  1990. {"session_id": raw_session, "chunks": []})
  1991. check(status == 401, "chunks endpoint rejects a missing token")
  1992. status, _ = saddle_api("POST", "/api/saddle/results", "saddle-bogus-token",
  1993. {"session_id": raw_session, "state": "working"})
  1994. check(status == 401, "results endpoint rejects a wrong token")
  1995. status, _ = saddle_api("POST", "/api/saddle/chunks", fake_token,
  1996. {"session_id": 99999, "chunks": []})
  1997. check(status == 400, "chunks for an unknown session rejected")
  1998. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  1999. {"session_id": raw_session, "state": "done"})
  2000. check(status == 200, "the owning saddle's session accepts its results")
  2001. # Pushed chunk idempotency: re-pushing a batch (the outbox's
  2002. # at-least-once redelivery after a lost response) consumes the
  2003. # duplicates and stores only the tail.
  2004. conn = db()
  2005. before = conn.execute("select count(*) from session_raw_log where session_id = ?",
  2006. (raw_session,)).fetchone()[0]
  2007. conn.close()
  2008. redelivery = [{"seq": r[0], "text": r[1]} for r in rows[:3]]
  2009. status, body = saddle_api("POST", "/api/saddle/chunks", fake_token,
  2010. {"session_id": raw_session, "chunks": redelivery})
  2011. check(status == 200 and body.get("ok") is True and int(body.get("stored") or 0) == 0,
  2012. "a fully-duplicate batch stores nothing (duplicates consumed)")
  2013. fresh = [{"seq": 9000, "text": "late-flush\r\n"},
  2014. {"seq": 9001, "text": ""}] # truly empty text: no row
  2015. status, body = saddle_api("POST", "/api/saddle/chunks", fake_token,
  2016. {"session_id": raw_session, "chunks": fresh})
  2017. check(status == 200 and int(body.get("stored") or 0) == 1,
  2018. "a partial-redelivery batch stores only the new chunks")
  2019. conn = db()
  2020. after = conn.execute("select count(*) from session_raw_log where session_id = ?",
  2021. (raw_session,)).fetchone()[0]
  2022. empty_row = conn.execute(
  2023. "select count(*) from session_raw_log where session_id = ? and seq = 9001",
  2024. (raw_session,)).fetchone()[0]
  2025. conn.close()
  2026. check(after == before + 1 and empty_row == 0,
  2027. "redelivered chunks never duplicate rows; empty text stores nothing")
  2028. # Pushed entries: today's OutputEntry shape with the daemon's
  2029. # global entry ids — a redelivered results push stores its
  2030. # entries exactly once.
  2031. def command_rows(session):
  2032. conn = db()
  2033. try:
  2034. return conn.execute(
  2035. "select command_text, type from session_commands"
  2036. " where session_id = ? order by id", (session,)).fetchall()
  2037. finally:
  2038. conn.close()
  2039. entries = [{"id": 8101, "command": "sed -n 1p README.md", "output": "# demo",
  2040. "exit_code": 0, "type": "command", "timestamp": "2026-09-06T00:00:00Z"}]
  2041. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  2042. {"session_id": raw_session, "state": "working",
  2043. "entries": entries})
  2044. check(status == 200, "entries push accepted")
  2045. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  2046. {"session_id": raw_session, "state": "working",
  2047. "entries": entries})
  2048. check(status == 200, "redelivered entries push accepted (at-least-once)")
  2049. stored = command_rows(raw_session)
  2050. check(len([r for r in stored if r[0] == "sed -n 1p README.md"]) == 1,
  2051. "redelivered entries store exactly once (entry-id cursor)")
  2052. # The retired daemon routes 404 (phase 2 deleted /output, /log
  2053. # and /start serving) — and the web app never calls them.
  2054. for method, path in (("GET", "/output?since=0"), ("GET", "/log?since=0"),
  2055. ("POST", "/start")):
  2056. request = urllib.request.Request(
  2057. "http://127.0.0.1:%d%s" % (fake.port, path),
  2058. data=b"{}" if method == "POST" else None, method=method)
  2059. try:
  2060. urllib.request.urlopen(request, timeout=5)
  2061. gone = False
  2062. except urllib.error.HTTPError as error:
  2063. gone = error.code == 404
  2064. check(gone, "the fake daemon's %s %s is deleted (404)" % (method, path))
  2065. with open(os.path.join(WORK, "app.log")) as handle:
  2066. app_log = handle.read()
  2067. check("Saddle output request" not in app_log and "Saddle log request" not in app_log,
  2068. "the web never polls /output or /log anymore")
  2069. print("== capabilities: tag-based routing ==")
  2070. saddles_state = slot_public(auth, "/saddles", "saddles")
  2071. act(auth, saddles_state["create_capability"], {"name": "gpu"})
  2072. act(auth, saddles_state["create_capability"], {"name": "bigmem"})
  2073. saddles_state = slot_public(auth, "/saddles", "saddles")
  2074. caps = {c["name"]: c for c in saddles_state["capabilities"]}
  2075. check(len(caps) == 2 and all(int(c["saddle_count"]) == 0 and int(c["task_count"]) == 0
  2076. for c in caps.values()),
  2077. "capabilities created and listed with zero usage")
  2078. fake_card = saddle_card(auth, "fake-saddle")
  2079. check(fake_card["capabilities"] == [],
  2080. "saddle card starts with no capability chips")
  2081. # A task requiring 'gpu' waits while the only idle saddle lacks it;
  2082. # a later task without requirements still dispatches (skip-and-continue).
  2083. tasks_state = slot_public(auth, "/tasks", "tasks")
  2084. task_id_tmp = create_queued(auth, {
  2085. "project_id": "1",
  2086. "title": "GPU task",
  2087. "priority": "2",
  2088. })
  2089. gpu_task = task_id_tmp
  2090. detail = task_state(auth, gpu_task)
  2091. act(auth, detail["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
  2092. detail = task_state(auth, gpu_task)
  2093. check([c["name"] for c in detail["capabilities"]] == ["gpu"],
  2094. "task page lists the required capability")
  2095. # The only idle saddle lacks the tag, so the queue flags the wait
  2096. # (read before any other task claims the idle saddle).
  2097. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2098. if int(t["id"]) == gpu_task][0]
  2099. check(queue["status"] == "queued" and queue["capabilities"] == ["gpu"]
  2100. and queue.get("waiting_for_saddle") is True,
  2101. "queue row shows the waiting task with its tag and wait flag")
  2102. task_id_tmp = create_queued(auth, {
  2103. "project_id": "1",
  2104. "title": "Any-saddle task",
  2105. "priority": "3",
  2106. })
  2107. any_task = task_id_tmp
  2108. check(wait_for(lambda: task_state(auth, any_task).get("status") != "queued" or False,
  2109. timeout=30, what="uncapable task dispatches") is not None,
  2110. "task without requirements dispatches past the waiting task")
  2111. def capability_wait_logged():
  2112. with open(os.path.join(WORK, "app.log")) as handle:
  2113. app_log = handle.read()
  2114. return "waits for a capable saddle" in app_log and "requires: gpu" in app_log
  2115. check(wait_for(capability_wait_logged, timeout=20,
  2116. what="capability wait log") is not None,
  2117. "dispatcher logs the capability wait")
  2118. # Attaching the tag to the saddle releases the task.
  2119. fake_card = saddle_card(auth, "fake-saddle")
  2120. act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
  2121. fake_card = saddle_card(auth, "fake-saddle")
  2122. check([c["name"] for c in fake_card["capabilities"]] == ["gpu"],
  2123. "saddle card shows the attached chip")
  2124. check(wait_task_done(auth, gpu_task, timeout=60) is not None,
  2125. "task dispatches once the saddle carries the required tag")
  2126. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2127. if int(t["id"]) == gpu_task][0]
  2128. check(queue.get("waiting_for_saddle") is not True,
  2129. "wait flag clears once dispatched")
  2130. # Multi-tag: gpu + bigmem required, only gpu carried → waits; adding
  2131. # bigmem completes the superset. Both requirements ride the create
  2132. # form (atomic — attaching them one at a time would race a tick
  2133. # into dispatching on the partial set).
  2134. task_id_tmp = create_queued(auth, {
  2135. "project_id": "1",
  2136. "title": "Big task",
  2137. "priority": "2",
  2138. "capability_id": [str(int(caps["gpu"]["id"])), str(int(caps["bigmem"]["id"]))],
  2139. })
  2140. big_task = task_id_tmp
  2141. check(wait_for(lambda: ([t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2142. if int(t["id"]) == big_task][0].get("waiting_for_saddle")
  2143. is True) or False, timeout=30, what="big task waits") is not None,
  2144. "task waits when the saddle's tags are not a superset")
  2145. fake_card = saddle_card(auth, "fake-saddle")
  2146. act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["bigmem"]["id"]))})
  2147. check(wait_task_done(auth, big_task, timeout=60) is not None,
  2148. "task dispatches once the superset is complete")
  2149. # Detach a requirement (on a draft — capabilities lock once a
  2150. # worker has the job); delete a capability everywhere.
  2151. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2152. "project_id": "1",
  2153. "title": "Detach probe task",
  2154. "priority": "4",
  2155. "capability_id": [str(int(caps["gpu"]["id"])), str(int(caps["bigmem"]["id"]))],
  2156. })
  2157. detach_probe = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2158. if t["title"] == "Detach probe task"][0])
  2159. act(auth, [c for c in task_state(auth, detach_probe)["capabilities"]
  2160. if c["name"] == "bigmem"][0]["detach"], {})
  2161. check([c["name"] for c in task_state(auth, detach_probe)["capabilities"]] == ["gpu"],
  2162. "detached requirement disappears from the task")
  2163. bigmem_row = [c for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  2164. if c["name"] == "bigmem"][0]
  2165. act(auth, bigmem_row["delete"], {})
  2166. saddles_state = slot_public(auth, "/saddles", "saddles")
  2167. check(all(c["name"] != "bigmem" for c in saddles_state["capabilities"]),
  2168. "deleted capability gone from the catalogue")
  2169. check([c["name"] for c in task_state(auth, big_task)["capabilities"]] == ["gpu"],
  2170. "deleting a capability drops its task requirements")
  2171. check(all(c["name"] != "bigmem"
  2172. for c in saddle_card(auth, "fake-saddle")["capabilities"]),
  2173. "deleting a capability drops it from saddles")
  2174. print("== B4: required capabilities on the create form ==")
  2175. tasks_state = slot_public(auth, "/tasks", "tasks")
  2176. gpu_option = [o for o in tasks_state.get("capability_options", [])
  2177. if o["name"] == "gpu"][0]
  2178. check(len(tasks_state.get("capability_options", [])) >= 1,
  2179. "queue page offers capability options in the create form")
  2180. conn = db()
  2181. before = conn.execute("select count(*) from tasks").fetchone()[0]
  2182. conn.close()
  2183. act(auth, tasks_state["create"], {
  2184. "project_id": "1",
  2185. "title": "Bad cap task",
  2186. "capability_id": [gpu_option["id"], "999"],
  2187. })
  2188. conn = db()
  2189. after = conn.execute("select count(*) from tasks").fetchone()[0]
  2190. conn.close()
  2191. check(after == before, "unknown capability id rejects the create")
  2192. task_id_tmp = create_queued(auth, {
  2193. "project_id": "1",
  2194. "title": "Cap-on-create task",
  2195. "capability_id": [gpu_option["id"]],
  2196. })
  2197. cap_create = task_id_tmp
  2198. detail = task_state(auth, cap_create)
  2199. check([c["name"] for c in detail["capabilities"]] == ["gpu"],
  2200. "capabilities from the create form land on the task")
  2201. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2202. if int(t["id"]) == cap_create][0]
  2203. check(queue["capabilities"] == ["gpu"],
  2204. "queue row shows the created task's requirements")
  2205. # The waiting flag must hold it back until the capable saddle frees
  2206. # up (the fake saddle still carries gpu from the routing checks).
  2207. check(queue.get("waiting_for_saddle") is not True,
  2208. "capable saddle available: no wait flag on the new task")
  2209. print("== retry + resume ==")
  2210. # A daemon-side failure fails the task with its explanation.
  2211. fake.fail_next = True
  2212. task_id_tmp = create_queued(auth, {
  2213. "project_id": "1",
  2214. "title": "Flaky task",
  2215. "priority": "2",
  2216. })
  2217. flaky = task_id_tmp
  2218. check(wait_for(lambda: task_state(auth, flaky).get("status") == "failed" or False,
  2219. timeout=60, what="flaky task fails") is not None,
  2220. "daemon failure fails the task")
  2221. detail = task_state(auth, flaky)
  2222. check("bash -n rejected" in (detail.get("summary") or ""),
  2223. "failure summary shows the daemon's explanation")
  2224. # Retry: summary moves into a comment and clears, task requeues,
  2225. # history stays, and the re-dispatch restores the workspace.
  2226. comments_before = len(detail.get("comments", []))
  2227. progress_before = len(detail.get("progress", []))
  2228. act(auth, detail["retry"], {})
  2229. detail = task_state(auth, flaky)
  2230. check(detail["status"] == "queued" and (detail.get("summary") or "") == "",
  2231. "retry requeues the task and clears the stale summary")
  2232. check(any("bash -n rejected" in c["body"] for c in detail["comments"]),
  2233. "failure explanation kept as a bot comment")
  2234. check(any(c["body"] == "Task retried by user" for c in detail["comments"]),
  2235. "retry lands its audit comment")
  2236. check(len(detail["comments"]) >= comments_before + 1,
  2237. "retry keeps the comment history")
  2238. pushes_before = len([p for p in fake.start_payloads
  2239. if int(p.get("task", {}).get("id") or 0) == flaky])
  2240. check(wait_for(lambda: len([p for p in fake.start_payloads
  2241. if int(p.get("task", {}).get("id") or 0) == flaky])
  2242. > pushes_before or False, timeout=60,
  2243. what="flaky task re-dispatched") is not None,
  2244. "retried task dispatches again")
  2245. # The resumed arc surfaces on the next /status read — in the
  2246. # push world that is the operator's manual refresh.
  2247. act(auth, saddle_card(auth, "fake-saddle")["refresh"], {})
  2248. check(wait_task_done(auth, flaky, timeout=60) is not None,
  2249. "retried task completes on the restored workspace")
  2250. detail = task_state(auth, flaky)
  2251. check(any("restored the session context" in p["summary"]
  2252. for p in detail.get("progress", [])),
  2253. "daemon resume reported on the first status after the restore")
  2254. check(len(detail["progress"]) > progress_before,
  2255. "resume note appended to the progress timeline")
  2256. # Manual resume: a task stuck working on an offline saddle.
  2257. task_id_tmp = create_queued(auth, {
  2258. "project_id": "1",
  2259. "title": "Stuck task",
  2260. "priority": "2",
  2261. })
  2262. stuck = task_id_tmp
  2263. check(wait_for(lambda: task_state(auth, stuck).get("status") == "working" or False,
  2264. timeout=60, what="stuck task starts working") is not None,
  2265. "stuck task reaches working")
  2266. # Deterministic claim for the resume leg below: the round's end,
  2267. # the refresh probe AND the dispatcher's wake for the requeued task
  2268. # all trigger self-asks that could steal it — freeze every trigger
  2269. # until the manual ask below has made the claim.
  2270. fake.ask_on_terminal = False
  2271. fake.ask_on_idle_status = False
  2272. fake.ask_on_wake = False
  2273. fake.settle_asks()
  2274. conn = db()
  2275. conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
  2276. conn.commit()
  2277. conn.close()
  2278. detail = task_state(auth, stuck)
  2279. check(detail.get("resume_available") is True,
  2280. "task page offers resume while its saddle is offline")
  2281. # The resume POSTs task-update cancel to the daemon; the abort
  2282. # handler fires an ungated re-ask that would race the manual claim
  2283. # below — slow it past the manual ask (the knob the slow-ask
  2284. # regression uses).
  2285. fake.ask_delay = 5.0
  2286. act(auth, detail["resume"], {})
  2287. detail = task_state(auth, stuck)
  2288. check(detail["status"] == "queued",
  2289. "manual resume requeues the stuck task")
  2290. check(any(c["body"] == "Saddle became unreachable; task requeued"
  2291. for c in detail["comments"]),
  2292. "manual resume lands the interruption comment")
  2293. check(all(s["status"] == "offline"
  2294. for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
  2295. "saddle stays offline after the manual requeue")
  2296. # Bringing the saddle back lets the requeued task resume and finish.
  2297. fake.state = "idle"
  2298. fake_card = saddle_card(auth, "fake-saddle")
  2299. act(auth, fake_card["refresh"], {})
  2300. check(any(s["status"] == "idle"
  2301. for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2302. if s["name"] == "fake-saddle"),
  2303. "refreshed saddle returns idle")
  2304. # The refreshed daemon is idle-awake: it asks (the real loop's
  2305. # own ask; driven directly for determinism — every self-ask trigger
  2306. # is frozen and the abort re-ask is slowed past it; joining that
  2307. # delayed thread here would fire its ask first, so it is left to
  2308. # sleep out the rest of the bundle).
  2309. status, payload = fake.ask_now()
  2310. fake.ask_on_idle_status = True
  2311. fake.ask_on_terminal = True
  2312. fake.ask_on_wake = True
  2313. fake.ask_delay = 0.0
  2314. check(status == 200, "the refreshed daemon's ask claims the requeued task")
  2315. check(wait_task_done(auth, stuck, timeout=60) is not None,
  2316. "requeued task resumes on the refreshed saddle and completes")
  2317. # =============================================================================
  2318. # Bundle: subtasks — suspension parking, the subtask status push (with
  2319. # its best-effort retry), the synchronous subtask callback, and the
  2320. # settling waits for the parked parents this shard created.
  2321. # =============================================================================
  2322. def bundle_subtasks(ctx):
  2323. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  2324. app = ctx.app
  2325. # A hold tag nothing carries: R3's second child stays queued (a
  2326. # stable parked session for the failed-delivery leg) until released.
  2327. saddles_state = slot_public(auth, "/saddles", "saddles")
  2328. act(auth, saddles_state["create_capability"], {"name": "gpu"})
  2329. print("== B7: suspended state + per-subtask resume ==")
  2330. fake.suspend_on_title = "Suspend parent"
  2331. pushes_before = len(fake.start_payloads)
  2332. task_id_tmp = create_queued(auth, {
  2333. "project_id": "1",
  2334. "title": "Suspend parent",
  2335. "priority": "2",
  2336. })
  2337. parent7 = task_id_tmp
  2338. check(wait_for(lambda: task_state(auth, parent7).get("status") == "suspended"
  2339. or False, timeout=60, what="parent parks as suspended") is not None,
  2340. "daemon 'suspended' state parks the task")
  2341. detail = task_state(auth, parent7)
  2342. check(detail.get("display") == "waiting on subtasks",
  2343. "suspended task displays as waiting on subtasks")
  2344. check(any(s["status"] == "idle"
  2345. for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
  2346. "suspension frees the saddle")
  2347. first_push = fake.start_payloads[pushes_before]
  2348. check(first_push.get("subtasks") == [],
  2349. "the /start payload carries the subtasks array (empty here)")
  2350. # Two subtask events land while parked (staged, not dispatched):
  2351. # both must coalesce into one resume with a full report.
  2352. conn = db()
  2353. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  2354. " summary, completed_at, created_at) values (1, 'Child one', 'done', 5, ?,"
  2355. " 'built and verified the flange', strftime('%s','now','+1 second'),"
  2356. " strftime('%s','now'))", (parent7,))
  2357. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  2358. " summary, completed_at, created_at) values (1, 'Child two', 'failed', 5, ?,"
  2359. " 'the harness refused to start', strftime('%s','now','+1 second'),"
  2360. " strftime('%s','now'))", (parent7,))
  2361. conn.commit()
  2362. conn.close()
  2363. def parent_push_count():
  2364. return len([p for p in fake.start_payloads
  2365. if int(p.get("task", {}).get("id") or 0) == parent7])
  2366. check(wait_for(lambda: parent_push_count() >= 2 or False,
  2367. timeout=45, what="suspended parent resumes") is not None,
  2368. "subtask events resume the suspended parent")
  2369. parent_pushes = [p for p in fake.start_payloads
  2370. if int(p.get("task", {}).get("id") or 0) == parent7]
  2371. check(len(parent_pushes) >= 2,
  2372. "the parent's resume is a fresh /start for the same task")
  2373. resume_push = parent_pushes[-1]
  2374. notice = resume_push.get("resume_notice") or ""
  2375. check("Child one" in notice and "completed" in notice
  2376. and "built and verified the flange" in notice,
  2377. "resume notice reports the finished subtask with its summary")
  2378. check("Child two" in notice and "failed" in notice
  2379. and "the harness refused to start" in notice,
  2380. "coalesced: the second event rides the same resume notice")
  2381. subtask_entries = {(s["title"], s["status"]) for s in resume_push.get("subtasks", [])}
  2382. check(("Child one", "done") in subtask_entries
  2383. and ("Child two", "failed") in subtask_entries,
  2384. "subtasks array carries the children's states at dispatch time")
  2385. # The resumed arc surfaces on a /status read while the resumed
  2386. # round still runs — the operator's manual refresh in the push
  2387. # world.
  2388. act(auth, saddle_card(auth, "fake-saddle")["refresh"], {})
  2389. check(wait_task_done(auth, parent7, timeout=60) is not None,
  2390. "resumed parent completes")
  2391. detail = task_state(auth, parent7)
  2392. check(any("restored the session context" in p["summary"]
  2393. for p in detail.get("progress", [])),
  2394. "resumed flag reported on the first status after the restore")
  2395. print("== R3 B1: subtask status push ==")
  2396. # A parked (suspended) parent gives delivery a stable open session.
  2397. fake.suspend_on_title = "Push parent"
  2398. task_id_tmp = create_queued(auth, {
  2399. "project_id": "1",
  2400. "title": "Push parent",
  2401. "priority": "2",
  2402. })
  2403. push_parent = task_id_tmp
  2404. check(wait_for(lambda: task_state(auth, push_parent).get("status") == "suspended"
  2405. or False, timeout=60, what="push parent parks") is not None,
  2406. "push parent parks with an open session")
  2407. task_id_tmp = create_queued(auth, {
  2408. "project_id": "1",
  2409. "title": "Push child",
  2410. "parent_task_id": str(push_parent),
  2411. })
  2412. push_child = task_id_tmp
  2413. def events_for(subtask_id):
  2414. return [e for e in fake.subtask_status_events
  2415. if int(e.get("subtask_id") or 0) == subtask_id]
  2416. check(wait_for(lambda: len(events_for(push_child)) > 0 or False,
  2417. timeout=30, what="queued subtask event") is not None,
  2418. "subtask creation pushes a queued status event")
  2419. first_event = events_for(push_child)[0]
  2420. check(int(first_event.get("parent_task_id") or 0) == push_parent
  2421. and first_event.get("title") == "Push child"
  2422. and first_event.get("status") in ("queued", "working"),
  2423. "status event carries parent id, title and a wire status (%s)"
  2424. % first_event.get("status"))
  2425. # Real lifecycle: the child dispatches (working) and completes (done
  2426. # with summary) — the parent's daemon hears every transition.
  2427. check(wait_for(lambda: any(e.get("status") == "done"
  2428. for e in events_for(push_child)) or False,
  2429. timeout=90, what="done subtask event") is not None,
  2430. "subtask completion pushes the done transition")
  2431. check(any(e.get("status") == "working" for e in events_for(push_child)),
  2432. "dispatch pushes a working status event")
  2433. # Best-effort retry: while the daemon 500s, events stay queued and
  2434. # land once it recovers. The parent's resumed run re-parks (the
  2435. # ask-flow resume races this section otherwise: its completion
  2436. # would close the parked session the events are delivered to).
  2437. fake.fail_subtask_status = True
  2438. fake.suspend_on_title = "Push parent"
  2439. # The child carries the hold tag (nothing can satisfy it) so it
  2440. # stays queued: its status events keep targeting the parent's
  2441. # stable parked session until the hold is released below.
  2442. gpu_cap_id = [c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  2443. if c["name"] == "gpu"][0]
  2444. task_id_tmp = create_queued(auth, {
  2445. "project_id": "1",
  2446. "title": "Push child two",
  2447. "parent_task_id": str(push_parent),
  2448. "capability_id": [str(gpu_cap_id)],
  2449. })
  2450. push_child_two = task_id_tmp
  2451. # Converted sleep(7): wait until the dispatcher has actually
  2452. # attempted (and failed) the delivery twice — two ticks' worth of
  2453. # evidence the events stay queued — instead of sleeping blindly.
  2454. check(wait_for(lambda: fake.subtask_fail_hits >= 2 or False,
  2455. timeout=15, what="failed subtask-event deliveries") is not None
  2456. and len(events_for(push_child_two)) == 0,
  2457. "failed pushes stay queued (no event delivered)")
  2458. fake.fail_subtask_status = False
  2459. check(wait_for(lambda: len(events_for(push_child_two)) > 0 or False,
  2460. timeout=30, what="retried subtask event") is not None,
  2461. "recovered pushes deliver on a later tick")
  2462. # Release the hold: the child dispatches and completes; the settle
  2463. # waits at the end of this bundle cover the parent's final resume.
  2464. act(auth, task_state(auth, push_child_two)["capabilities"][0]["detach"], {})
  2465. print("== R4 B1: synchronous subtask callback ==")
  2466. # Plan 10: the subtask callback is per-saddle-token authenticated.
  2467. r4_token = fake_token
  2468. # Parent with a live parked session (suspended) hosts the callback.
  2469. fake.suspend_on_title = "Callback parent"
  2470. task_id_tmp = create_queued(auth, {
  2471. "project_id": "1",
  2472. "title": "Callback parent",
  2473. "priority": "2",
  2474. })
  2475. cb_parent = task_id_tmp
  2476. check(wait_for(lambda: task_state(auth, cb_parent).get("status") == "suspended"
  2477. or False, timeout=60, what="callback parent parks") is not None,
  2478. "callback parent parks")
  2479. status, _ = saddle_api("POST", "/api/saddle/subtask", None,
  2480. {"parent_task_id": cb_parent, "title": "no token"})
  2481. check(status == 401, "subtask callback rejects a missing token")
  2482. conn = db()
  2483. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  2484. " values (1, 'Anchor task', 'done', 5, strftime('%s','now'))")
  2485. anchor = conn.execute("select max(id) from tasks").fetchone()[0]
  2486. conn.commit()
  2487. conn.close()
  2488. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  2489. "parent_task_id": cb_parent,
  2490. "title": "Wire the flange",
  2491. "description": "torque to spec",
  2492. "depends_on": [anchor],
  2493. "session_id": 1,
  2494. })
  2495. check(status == 200 and body.get("ok") is True and int(body.get("task_id") or 0) > 0,
  2496. "callback creates the subtask synchronously")
  2497. subtask_id = int(body.get("task_id"))
  2498. detail = task_state(auth, subtask_id)
  2499. check(detail["title"] == "Wire the flange" and int(detail["project_id"]) == 1
  2500. and int(detail["parent_task_id"]) == cb_parent,
  2501. "subtask row inherits the parent's project and linkage")
  2502. check(detail.get("task_branch") == task_state(auth, cb_parent).get("task_branch"),
  2503. "subtask shares the parent's task branch")
  2504. conn = db()
  2505. dep_rows = conn.execute(
  2506. "select depends_on_task_id from task_dependencies where task_id = ?",
  2507. (subtask_id,)).fetchall()
  2508. conn.close()
  2509. check([r[0] for r in dep_rows] == [anchor],
  2510. "depends_on wired as task_dependencies")
  2511. # Idempotency + entry-poll fallback dedup.
  2512. status, body2 = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  2513. "parent_task_id": cb_parent, "title": "Wire the flange"})
  2514. check(status == 200 and int(body2.get("task_id") or 0) == subtask_id,
  2515. "retried callback returns the same task id")
  2516. conn = db()
  2517. before = conn.execute("select count(*) from tasks where parent_task_id = ?",
  2518. (cb_parent,)).fetchone()[0]
  2519. conn.close()
  2520. conn = db()
  2521. cb_session = conn.execute(
  2522. "select id from sessions where task_id = ? and ended_at is null",
  2523. (cb_parent,)).fetchone()[0]
  2524. conn.close()
  2525. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  2526. {"type": "subtask", "session_id": cb_session, "data": {
  2527. "title": "Wire the flange", "description": "again"}})
  2528. conn = db()
  2529. after = conn.execute("select count(*) from tasks where parent_task_id = ?",
  2530. (cb_parent,)).fetchone()[0]
  2531. timeline = conn.execute(
  2532. "select count(*) from session_commands where session_id = ?"
  2533. " and type = 'subtask'", (cb_session,)).fetchone()[0]
  2534. conn.close()
  2535. check(after == before,
  2536. "entry-poll fallback dedupes against the callback's row")
  2537. check(timeline >= 1, "timeline entry recorded on the parent's session")
  2538. # Validation: unknown parent, unknown depends_on, ancestor cycle.
  2539. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
  2540. {"parent_task_id": 99999, "title": "ghost"})
  2541. check(status == 400, "unknown parent rejected")
  2542. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
  2543. {"parent_task_id": cb_parent, "title": "bad dep",
  2544. "depends_on": [99999]})
  2545. check(status == 400 and "unknown task" in body.get("error", ""),
  2546. "unknown depends_on id rejected with a clear error")
  2547. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
  2548. {"parent_task_id": cb_parent, "title": "cycle dep",
  2549. "depends_on": [cb_parent]})
  2550. check(status == 400 and "cycle" in body.get("error", ""),
  2551. "depending on an ancestor rejected as a cycle")
  2552. print("== plan 11: auto-enqueue agent-created subtasks ==")
  2553. # The per-task option: a subtask created under a parent carrying it
  2554. # enqueues immediately (via the entry-log fallback AND the
  2555. # synchronous callback); without it the subtask lands in draft.
  2556. # Either way the child inherits the parent's value as its own
  2557. # setting.
  2558. conn = db()
  2559. conn.execute("update tasks set auto_enqueue_subtasks = 1 where id = ?", (cb_parent,))
  2560. conn.commit()
  2561. cb_session = conn.execute(
  2562. "select id from sessions where task_id = ? and ended_at is null",
  2563. (cb_parent,)).fetchone()
  2564. conn.close()
  2565. check(cb_session is not None, "callback parent still parked for the fallback leg")
  2566. # The entry-log fallback applies the rule (run first: the parked
  2567. # session is guaranteed open before any enqueued child completes
  2568. # and resumes the parent).
  2569. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  2570. {"type": "subtask", "session_id": cb_session[0], "data": {
  2571. "title": "Fallback enqueued child"}})
  2572. fallback_child = task_id_of("Fallback enqueued child")
  2573. check(wait_for(lambda: task_state(auth, fallback_child)["status"] == "queued"
  2574. or False, timeout=15, what="fallback child queued") is not None,
  2575. "the entry-log fallback enqueues under the parent's option too")
  2576. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  2577. "parent_task_id": cb_parent, "title": "Auto-enqueued child"})
  2578. check(status == 200, "callback under an auto-enqueue parent accepted")
  2579. auto_enqueued = int(body["task_id"])
  2580. check(wait_for(lambda: task_state(auth, auto_enqueued)["status"] == "queued"
  2581. or False, timeout=15, what="auto-enqueued child queued") is not None,
  2582. "the parent's option enqueues the callback subtask immediately")
  2583. check(task_state(auth, auto_enqueued).get("auto_enqueue_subtasks") is True,
  2584. "the child inherits the parent's auto-enqueue setting for its own children")
  2585. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  2586. "parent_task_id": auto_enqueued, "title": "Grandchild under inheritance"})
  2587. grandchild_ae = int(body["task_id"])
  2588. check(task_state(auth, grandchild_ae)["status"] == "queued",
  2589. "inheritance applies: a subtask under the child enqueues too")
  2590. # Without the option the callback drafts (reset first — the child
  2591. # inherits the parent's value at creation time).
  2592. conn = db()
  2593. conn.execute("update tasks set auto_enqueue_subtasks = 0 where id = ?", (cb_parent,))
  2594. conn.commit()
  2595. conn.close()
  2596. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  2597. "parent_task_id": cb_parent, "title": "Draft callback child"})
  2598. draft_cb_child = int(body["task_id"])
  2599. detail = task_state(auth, draft_cb_child)
  2600. check(detail["status"] == "draft"
  2601. and detail.get("auto_enqueue_subtasks") is False,
  2602. "without the option the callback subtask lands in draft (setting off)")
  2603. print("== settle: this shard's parked parents resume and finish ==")
  2604. # Settle earlier stragglers first: the round-3/B1 suspended parents
  2605. # resume (a fresh /start) once their last subtask completes — those
  2606. # pushes would consume the one-shot suspend flag below.
  2607. check(wait_task_done(auth, push_parent, timeout=90) is not None
  2608. and wait_task_done(auth, push_child_two, timeout=90) is not None
  2609. and wait_task_done(auth, cb_parent, timeout=90) is not None,
  2610. "earlier suspended parents settle before the auto-approve run")
  2611. # =============================================================================
  2612. # Bundle: lineage — subtask auto-approve (transitive coverage, the
  2613. # toggle's descendant-only scope, the final resume), /start parent
  2614. # context and the daemon task-reading API.
  2615. # =============================================================================
  2616. def bundle_lineage(ctx):
  2617. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  2618. app = ctx.app
  2619. r4_token = fake_token
  2620. # gpu exists only as a hold (no saddle carries it): the auto-approve
  2621. # children stay queued until the final-resume leg lifts it.
  2622. saddles_state = slot_public(auth, "/saddles", "saddles")
  2623. act(auth, saddles_state["create_capability"], {"name": "gpu"})
  2624. # Parent parks; its toggle then covers all descendants. Children are
  2625. # created with an unmet 'gpu' requirement so they stay queued (and
  2626. # untouched) until the requirement is lifted for the final-resume leg.
  2627. fake.suspend_on_title = "Auto parent"
  2628. task_id_tmp = create_queued(auth, {
  2629. "project_id": "1",
  2630. "title": "Auto parent",
  2631. "priority": "2",
  2632. })
  2633. auto_parent = task_id_tmp
  2634. check(wait_for(lambda: task_state(auth, auto_parent).get("status") == "suspended"
  2635. or False, timeout=60, what="auto parent parks") is not None,
  2636. "auto parent parks")
  2637. detail = task_state(auth, auto_parent)
  2638. check(detail.get("auto_approve") is False,
  2639. "auto-approve defaults to off")
  2640. act(auth, detail["toggle_auto_approve"], {})
  2641. check(task_state(auth, auto_parent).get("auto_approve") is True,
  2642. "toggle turns auto-approve on")
  2643. gpu_option = int([o["id"] for o in slot_public(auth, "/tasks", "tasks")["capability_options"]
  2644. if o["name"] == "gpu"][0])
  2645. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
  2646. {"parent_task_id": auto_parent,
  2647. "title": "Auto child", "depends_on": []})
  2648. auto_child = int(body["task_id"])
  2649. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
  2650. {"parent_task_id": auto_child, "title": "Auto grandchild"})
  2651. auto_grand = int(body["task_id"])
  2652. # Fixture: one more held subtask whose title the task-reading API's
  2653. # q= search matches below (this bundle's "Wire the flange").
  2654. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  2655. "parent_task_id": auto_parent, "title": "Wire the flange",
  2656. "description": "torque to spec"})
  2657. flange_task = int(body["task_id"])
  2658. act(auth, task_state(auth, auto_child)["attach_capability"],
  2659. {"capability_id": str(gpu_option)})
  2660. act(auth, task_state(auth, auto_grand)["attach_capability"],
  2661. {"capability_id": str(gpu_option)})
  2662. act(auth, task_state(auth, flange_task)["attach_capability"],
  2663. {"capability_id": str(gpu_option)})
  2664. # The callback children land in draft (auto_parent carries no
  2665. # auto-enqueue option); the held gpu requirement attaches while
  2666. # editable, then the human enqueues each — they sit queued (and
  2667. # un-dispatchable) until the release loop lifts the hold.
  2668. for held in (auto_child, auto_grand, flange_task):
  2669. enqueue_task(auth, held)
  2670. # Transitive coverage: the GRANDCHILD's review is auto-answered
  2671. # (any ancestor set), answered-by 'auto', timeline entry recorded.
  2672. conn = db()
  2673. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  2674. " values (?, ?, strftime('%s','now'))", (auto_grand, fake_id))
  2675. conn.commit()
  2676. grand_session = conn.execute("select max(id) from sessions").fetchone()[0]
  2677. conn.close()
  2678. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  2679. {"type": "review", "session_id": grand_session, "data": {
  2680. "summary": "the flange is wired"}})
  2681. grand_detail = task_state(auth, auto_grand)
  2682. grand_review = [b for b in grand_detail["blocks"] if b["block_type"] == "review"][0]
  2683. check(grand_review.get("answered") is True
  2684. and grand_review.get("answer") == "Auto-approved (subtask auto-approve is on)"
  2685. and grand_review.get("answered_by") == "auto",
  2686. "covered grandchild review auto-answered as 'auto'")
  2687. conn = db()
  2688. timeline_auto = conn.execute(
  2689. "select count(*) from session_commands where session_id = ?"
  2690. " and type = 'auto-approve'", (grand_session,)).fetchone()[0]
  2691. conn.close()
  2692. check(timeline_auto >= 1, "auto action visible in the timeline")
  2693. # Questions and RFCs still block for a human.
  2694. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  2695. {"type": "block", "session_id": grand_session, "data": {
  2696. "question": "Which torque spec?", "type": "question"}})
  2697. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  2698. {"type": "block", "session_id": grand_session, "data": {
  2699. "question": "RFC: approve the design", "type": "rfc",
  2700. "body": "# Design"}})
  2701. grand_detail = task_state(auth, auto_grand)
  2702. check(all(b.get("answered") is not True for b in grand_detail["blocks"]
  2703. if b["block_type"] in ("question", "rfc")),
  2704. "questions and RFCs are never auto-answered")
  2705. # Plan 11: the transitive toggle also covers descendant PLAN blocks
  2706. # (instant approve, plan markdown applied).
  2707. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  2708. {"type": "block", "session_id": grand_session, "data": {
  2709. "question": "Plan submitted for review", "type": "plan",
  2710. "body": "# Grandchild plan\n\n1. Follow the parent plan"}})
  2711. grand_detail = task_state(auth, auto_grand)
  2712. grand_plan = [b for b in grand_detail["blocks"] if b["block_type"] == "plan"][0]
  2713. check(grand_plan.get("answered") is True
  2714. and grand_plan.get("answer") == "Plan approved. Proceed with implementation."
  2715. and grand_plan.get("answered_by") == "auto",
  2716. "the ancestor toggle instantly approves descendant plan blocks")
  2717. check(grand_detail.get("plan", "").startswith("# Grandchild plan"),
  2718. "the auto-approved plan markdown is applied to the task")
  2719. # The toggle covers DESCENDANTS only: the parent's own review waits.
  2720. conn = db()
  2721. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at,"
  2722. " outcome) values (?, ?, strftime('%s','now'), strftime('%s','now'),"
  2723. " 'done')", (auto_parent, fake_id))
  2724. conn.commit()
  2725. parent_session = conn.execute("select max(id) from sessions").fetchone()[0]
  2726. conn.close()
  2727. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  2728. {"type": "review", "session_id": parent_session, "data": {
  2729. "summary": "the whole change"}})
  2730. parent_detail = task_state(auth, auto_parent)
  2731. parent_review = [b for b in parent_detail["blocks"] if b["block_type"] == "review"][0]
  2732. check(parent_review.get("answered") is not True,
  2733. "the top-level parent's own review is never auto-approved")
  2734. # All descendants terminal → the suspended parent resumes with the
  2735. # final-review instruction. The resume fires per completion (ANY
  2736. # terminal child makes the parent offerable again), so release the
  2737. # held descendants one at a time and re-park the parent between
  2738. # releases: only the last resume sees every descendant terminal,
  2739. # and its notice then carries the final-review instruction.
  2740. for step, held_task in enumerate((auto_child, auto_grand, flange_task)):
  2741. if step < 2:
  2742. fake.suspend_on_title = "Auto parent" # interim resumes re-park
  2743. act(auth, task_state(auth, held_task)["capabilities"][0]["detach"], {})
  2744. check(wait_task_done(auth, held_task, timeout=90) is not None,
  2745. "released descendant settles before the next release (step %d)" % step)
  2746. def pushes_for(task_id):
  2747. return len([p for p in fake.start_payloads
  2748. if int(p.get("task", {}).get("id") or 0) == task_id])
  2749. before_pushes = pushes_for(auto_parent)
  2750. check(wait_for(lambda: pushes_for(auto_parent) > before_pushes or False,
  2751. timeout=90, what="auto parent resumes") is not None,
  2752. "terminal subtasks resume the suspended parent")
  2753. notice = ""
  2754. for p in fake.start_payloads:
  2755. if int(p.get("task", {}).get("id") or 0) == auto_parent:
  2756. notice = p.get("resume_notice") or ""
  2757. check("All subtasks are complete" in notice
  2758. and "ONE final review" in notice and "saddle review" in notice,
  2759. "resume notice ends with the final-review instruction")
  2760. print("== R5 B1: /start parent context ==")
  2761. # A fresh subtask dispatch carries the parent object (plan clamped).
  2762. conn = db()
  2763. conn.execute("update tasks set plan = ? where id = ?",
  2764. ("# Parent plan\n\n" + "step. " * 1200, auto_parent))
  2765. conn.commit()
  2766. conn.close()
  2767. # Barrier: the dispatch reads the parent row through the app's own
  2768. # connection — wait until the app serves the externally written
  2769. # plan before creating the dispatchable task (a stale snapshot
  2770. # would otherwise ship an empty parent plan).
  2771. check(wait_for(lambda: "Parent plan" in
  2772. (((saddle_api("GET", "/api/saddle/task/%d" % auto_parent,
  2773. r4_token)[1].get("task")) or {}).get("plan") or "")
  2774. or False, timeout=15, what="app serves the parent plan") is not None,
  2775. "the task API serves the parent plan before the lineage dispatch")
  2776. task_id_tmp = create_queued(auth, {
  2777. "project_id": "1",
  2778. "title": "Lineage subtask",
  2779. "priority": "3",
  2780. })
  2781. lineage_task = task_id_tmp
  2782. conn = db()
  2783. conn.execute("update tasks set parent_task_id = ?, description = 'torque to spec'"
  2784. " where id = ?", (auto_parent, lineage_task))
  2785. conn.commit()
  2786. conn.close()
  2787. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == lineage_task
  2788. for p in fake.start_payloads) or False,
  2789. timeout=60, what="lineage subtask dispatches") is not None,
  2790. "lineage subtask dispatches")
  2791. lineage_push = [p for p in fake.start_payloads
  2792. if int(p.get("task", {}).get("id") or 0) == lineage_task][-1]
  2793. parent_obj = lineage_push.get("parent")
  2794. check(parent_obj is not None
  2795. and int(parent_obj.get("id") or 0) == auto_parent
  2796. and parent_obj.get("title") == "Auto parent"
  2797. and parent_obj.get("status") in ("suspended", "working", "done"),
  2798. "subtask /start carries the parent object")
  2799. check(parent_obj.get("description") == "torque to spec"
  2800. or "Parent plan" in (parent_obj.get("plan") or ""),
  2801. "parent description/plan content flows through")
  2802. check(len(parent_obj.get("plan") or "") <= 4000,
  2803. "parent plan clamped to ~4000 chars (%d)"
  2804. % len(parent_obj.get("plan") or ""))
  2805. root_pushes = [p for p in fake.start_payloads
  2806. if int(p.get("task", {}).get("id") or 0) == auto_parent]
  2807. check(all("parent" not in p for p in root_pushes),
  2808. "root task /start carries no parent object")
  2809. print("== R5 B2: task-reading API ==")
  2810. status, _ = saddle_api("GET", "/api/saddle/tasks", None)
  2811. check(status == 401, "tasks endpoint rejects a missing token")
  2812. status, _ = saddle_api("GET", "/api/saddle/task/%d" % auto_child, None)
  2813. check(status == 401, "task detail endpoint rejects a missing token")
  2814. # Fixture: a second project whose tasks all carry the "Diff " title
  2815. # prefix (this bundle's own copy of the pages bundle's diff
  2816. # projects) for the project_id filter checks — created first so the
  2817. # unfiltered list has the population the count check expects.
  2818. state = slot_public(auth, "/projects", "projects")
  2819. act(auth, state["create"], {"name": "diff-fabrica",
  2820. "git_url": "ssh://fabrica.unitatem.net:222/clanker/clanker-saddle.git"})
  2821. conn = db()
  2822. other_project = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  2823. if p["name"] == "diff-fabrica"][0])
  2824. for suffix in ("fabrica", "scp"):
  2825. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  2826. " values (?, ?, 'done', 5, strftime('%s','now'))",
  2827. (other_project, "Diff %s" % suffix))
  2828. conn.commit()
  2829. conn.close()
  2830. status, body = saddle_api("GET", "/api/saddle/tasks", r4_token)
  2831. check(status == 200 and len(body.get("tasks", [])) > 5,
  2832. "tasks endpoint lists tasks without filters")
  2833. row = [t for t in body["tasks"] if int(t["id"]) == auto_child][0]
  2834. check(row["title"] == "Auto child" and int(row["parent_task_id"]) == auto_parent
  2835. and row["status"] in ("done", "queued", "working"),
  2836. "compact rows carry id/title/status/parent_task_id")
  2837. status, body = saddle_api("GET", "/api/saddle/tasks?project_id=1", r4_token)
  2838. check(all(True for _ in [1]) # project 1 dominates the data set
  2839. and not any(int(t["id"]) == 0 for t in body["tasks"]),
  2840. "project_id filter accepted")
  2841. status, body = saddle_api("GET",
  2842. "/api/saddle/tasks?project_id=%d" % other_project, r4_token)
  2843. check(all(t["title"].startswith("Diff") for t in body["tasks"]),
  2844. "other project lists only its own tasks")
  2845. status, body = saddle_api("GET", "/api/saddle/tasks?q=FLANGE", r4_token)
  2846. titles = [t["title"] for t in body["tasks"]]
  2847. check("Wire the flange" in titles,
  2848. "q matches titles case-insensitively")
  2849. status, body = saddle_api("GET", "/api/saddle/tasks?q=torque%20to%20spec", r4_token)
  2850. check(any(t["title"] == "Lineage subtask" for t in body["tasks"]),
  2851. "q matches description text")
  2852. status, body = saddle_api("GET", "/api/saddle/tasks?q=parent%20plan", r4_token)
  2853. check(any(t["title"] == "Auto parent" for t in body["tasks"]),
  2854. "q matches plan text")
  2855. status, body = saddle_api("GET", "/api/saddle/task/%d" % auto_child, r4_token)
  2856. task_view = body.get("task", {})
  2857. check(status == 200 and int(task_view.get("id") or 0) == auto_child
  2858. and task_view.get("parent_title") == "Auto parent"
  2859. and int(task_view.get("parent_task_id") or 0) == auto_parent,
  2860. "single-task view joins the parent title")
  2861. check("status" in task_view and "description" in task_view
  2862. and "plan" in task_view and "summary" in task_view,
  2863. "single-task view carries the full detail fields")
  2864. status, body = saddle_api("GET", "/api/saddle/task/99999", r4_token)
  2865. check(status == 404, "unknown task id returns 404")
  2866. # =============================================================================
  2867. # Bundle: allocation — the ask/wake/allocate machinery: concurrent ask
  2868. # race, token revocation, workspace affinity, the lost-wake guard, the
  2869. # slow-ask regression, assignment capability context + the tag
  2870. # catalogue, and the pinned-blocks/activity dashboard.
  2871. # =============================================================================
  2872. def bundle_allocation(ctx):
  2873. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  2874. app = ctx.app
  2875. r4_token = fake_token
  2876. # gpu rides the primary fake from the start (the routing bundle's
  2877. # early attach, replayed): its attach row precedes lostwake's, so
  2878. # the assignment's saddle_tags order matches the assertion in the
  2879. # capability-context section below.
  2880. saddles_state = slot_public(auth, "/saddles", "saddles")
  2881. act(auth, saddles_state["create_capability"], {"name": "gpu"})
  2882. act(auth, saddle_card(auth, "fake-saddle")["attach_capability"],
  2883. {"capability_id": str(int([c["id"] for c in
  2884. slot_public(auth, "/saddles", "saddles")["capabilities"]
  2885. if c["name"] == "gpu"][0]))})
  2886. print("== phase 1: ask/allocation — token auth + concurrent race ==")
  2887. # Plan 10 phase 1. Registration mints per-saddle tokens; the
  2888. # ask endpoint authenticates by them; concurrent asks race to
  2889. # exactly one winner through the allocation transaction.
  2890. status, _ = saddle_api("POST", "/api/saddle/next-task", None)
  2891. check(status == 401, "next-task rejects a missing token")
  2892. status, _ = saddle_api("POST", "/api/saddle/next-task", "saddle-bogus-token")
  2893. check(status == 401, "next-task rejects a wrong token")
  2894. status, body = saddle_api("POST", "/api/saddle/next-task", fake_token)
  2895. check(status == 204, "next-task answers 204 (nothing offerable — daemon sleeps)")
  2896. # Two more racers: real daemons on their own ports; fake2 reports
  2897. # its ssh key at /register so the key lands on the row directly.
  2898. fake2 = FakeSaddle()
  2899. fake2.ssh_public_key = "ssh-ed25519 AAAAFake2RacerKey saddle-2@fedora"
  2900. fake2.serve(port=free_port())
  2901. fake3 = FakeSaddle()
  2902. fake3.serve(port=free_port())
  2903. saddles_state = slot_public(auth, "/saddles", "saddles")
  2904. act(auth, saddles_state["register"], {"name": "racer-two",
  2905. "host": "127.0.0.1:%d" % fake2.port})
  2906. act(auth, saddles_state["register"], {"name": "racer-three",
  2907. "host": "127.0.0.1:%d" % fake3.port})
  2908. check(wait_for(lambda: len(fake2.register_log) > 0 and len(fake3.register_log) > 0
  2909. or False, timeout=15, what="racers registered") is not None,
  2910. "both racers received /register")
  2911. check(wait_for(lambda: "AAAAFake2RacerKey" in
  2912. [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2913. if s["name"] == "racer-two"][0].get("ssh_public_key", "") or False,
  2914. timeout=15, what="register-reported key") is not None,
  2915. "the ssh key reported at /register lands on the saddle row")
  2916. racer_cards = {}
  2917. for name, daemon in (("fake-saddle", fake), ("racer-two", fake2), ("racer-three", fake3)):
  2918. card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2919. if s["name"] == name][0]
  2920. racer_cards[daemon] = {"id": int(card["id"]), "token": card["token"]}
  2921. # Revocation: a registered saddle's token works until the saddle
  2922. # is removed, then nothing authorises its calls.
  2923. fake4 = FakeSaddle()
  2924. fake4.ask_on_register = False # a pre-flip daemon: never asks
  2925. fake4.serve(port=free_port())
  2926. saddles_state = slot_public(auth, "/saddles", "saddles")
  2927. act(auth, saddles_state["register"], {"name": "old-daemon",
  2928. "host": "127.0.0.1:%d" % fake4.port})
  2929. old_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2930. if s["name"] == "old-daemon"][0]
  2931. status, _ = saddle_api("POST", "/api/saddle/next-task", old_card["token"])
  2932. check(status in (200, 204),
  2933. "the old daemon's fresh token authenticates next-task")
  2934. status, _ = saddle_api("POST", "/api/saddle/note", old_card["token"],
  2935. {"project": "demo", "body": "from the old daemon"})
  2936. check(status == 200, "note accepts any registered saddle's token")
  2937. act(auth, old_card["unregister"], {})
  2938. status, _ = saddle_api("POST", "/api/saddle/next-task", old_card["token"])
  2939. check(status == 401, "removing the saddle revokes its token")
  2940. status, _ = saddle_api("POST", "/api/saddle/note", old_card["token"],
  2941. {"project": "demo", "body": "should fail"})
  2942. check(status == 401, "revocation covers the note endpoint too")
  2943. # Deterministic race: settle the queue first (this shard runs the
  2944. # race before any parked work exists, so only in-flight asks need
  2945. # draining), freeze every daemon's self-ask triggers, put all three
  2946. # to sleep (204), then fire one task worth of concurrent asks.
  2947. for daemon in (fake, fake2, fake3):
  2948. daemon.ask_on_register = False
  2949. daemon.ask_on_wake = False
  2950. daemon.ask_on_terminal = False
  2951. daemon.ask_on_idle_status = False
  2952. daemon.settle_asks() # converted sleep: in-flight asks land (204)
  2953. def queued_count():
  2954. conn = db()
  2955. try:
  2956. return conn.execute(
  2957. "select count(*) from tasks where status = 'queued'").fetchone()[0]
  2958. finally:
  2959. conn.close()
  2960. check(wait_for(lambda: queued_count() == 0 or False,
  2961. timeout=60, what="queue settles before the race") is not None,
  2962. "the queue settles before the race")
  2963. sleep_statuses = [daemon.ask_now()[0] for daemon in (fake, fake2, fake3)]
  2964. check(sleep_statuses == [204, 204, 204],
  2965. "all three racers sleep (204) before the task exists")
  2966. task_id_tmp = create_queued(auth, {
  2967. "project_id": "1",
  2968. "title": "Race task",
  2969. "priority": "2",
  2970. })
  2971. race_task = task_id_tmp
  2972. race_results = []
  2973. def race_ask(daemon):
  2974. # A real racing daemon accepts the assignment it wins — the
  2975. # manual ask path does exactly that on a 200.
  2976. race_results.append(daemon.ask_now())
  2977. threads = [threading.Thread(target=race_ask, args=(daemon,))
  2978. for daemon in (fake, fake2, fake3)]
  2979. for thread in threads:
  2980. thread.start()
  2981. for thread in threads:
  2982. thread.join()
  2983. race_statuses = sorted(r[0] for r in race_results)
  2984. check(race_statuses == [200, 204, 204],
  2985. "concurrent asks race to exactly one winner (%s)" % race_statuses)
  2986. winner = [r for r in race_results if r[0] == 200][0][1]
  2987. check(int(winner["task"]["id"]) == race_task, "the winner's payload carries the task")
  2988. check(int(winner["session_id"]) > 0, "the assignment carries the session id")
  2989. conn = db()
  2990. race_sessions = conn.execute(
  2991. "select count(*) from sessions where task_id = ?", (race_task,)).fetchone()[0]
  2992. conn.close()
  2993. check(race_sessions == 1,
  2994. "exactly one session row — the transaction serialised the claims (%d)"
  2995. % race_sessions)
  2996. check(wait_task_done(auth, race_task, timeout=60) is not None,
  2997. "the raced task completes on the winning daemon")
  2998. for daemon in (fake, fake2, fake3):
  2999. daemon.ask_on_register = True
  3000. daemon.ask_on_wake = True
  3001. daemon.ask_on_terminal = True
  3002. daemon.ask_on_idle_status = True
  3003. # Re-establish sleeping explicitly: the winner's post-race
  3004. # re-ask timing is not deterministic, and the affinity section
  3005. # below needs every racer provably asleep. Any stray offerable
  3006. # drains into these asks first.
  3007. def all_asleep():
  3008. return all(daemon.ask_now()[0] == 204 for daemon in (fake, fake2, fake3))
  3009. check(wait_for(lambda: all_asleep() or False, timeout=30,
  3010. what="racers re-sleep after the race") is not None,
  3011. "all racers re-sleep after the race")
  3012. print("== phase 1: workspace affinity pins parked tasks ==")
  3013. # A suspended parent parked on racer-two may be resumed ONLY by
  3014. # racer-two: a tag-capable non-affine saddle asking gets 204, and
  3015. # the wake targets the affine saddle. Only racer-two carries the
  3016. # tag at dispatch time (the race's winner is nondeterministic, so
  3017. # the park must not depend on which racers are free); racer-three
  3018. # gains it afterwards to prove the pinning is by workspace, not
  3019. # tags.
  3020. saddles_state = slot_public(auth, "/saddles", "saddles")
  3021. act(auth, saddles_state["create_capability"], {"name": "affinity-tag"})
  3022. affinity_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  3023. if c["name"] == "affinity-tag"][0])
  3024. racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3025. if s["name"] == "racer-two"][0]
  3026. racer3_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3027. if s["name"] == "racer-three"][0]
  3028. act(auth, racer2_card["attach_capability"], {"capability_id": str(affinity_cap)})
  3029. fake2.suspend_on_title = "Affinity parent"
  3030. task_id_tmp = create_queued(auth, {
  3031. "project_id": "1",
  3032. "title": "Affinity parent",
  3033. "priority": "2",
  3034. "capability_id": [str(affinity_cap)],
  3035. })
  3036. affinity_parent = task_id_tmp
  3037. check(wait_for(lambda: task_state(auth, affinity_parent).get("status") == "suspended"
  3038. or False, timeout=60, what="affinity parent parks") is not None,
  3039. "affinity parent parks (suspended) on the tag-carrying racer")
  3040. conn = db()
  3041. affinity_session = conn.execute(
  3042. "select saddle_id from sessions where task_id = ? and ended_at is null",
  3043. (affinity_parent,)).fetchone()[0]
  3044. conn.close()
  3045. check(affinity_session == int(racer2_card["id"]),
  3046. "the parked workspace sits on racer-two (the only tag-carrier)")
  3047. # Now racer-three carries the tag too — the pinning is by
  3048. # workspace, not tags.
  3049. # A terminal subtask makes the parked task resume-offerable —
  3050. # affine to racer-two only. The park is observed the moment the
  3051. # suspended push lands (no poller lag to compensate), so the
  3052. # child is inserted only after racer-two's ask triggers are
  3053. # frozen and the wake measurement armed — its completion stamped
  3054. # one second ahead so completed_at > suspended_at holds.
  3055. fake2.ask_on_wake = False
  3056. fake2.ask_on_idle_status = False
  3057. wakes_before_2 = len(fake2.wake_log)
  3058. fake2.settle_asks() # converted sleep: park-time ask lands (204)
  3059. conn = db()
  3060. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  3061. " summary, completed_at, created_at) values (1, 'Affinity child', 'done',"
  3062. " 5, ?, 'affinity child finished', strftime('%s','now','+1 second'),"
  3063. " strftime('%s','now'))", (affinity_parent,))
  3064. conn.commit()
  3065. conn.close()
  3066. act(auth, racer3_card["attach_capability"], {"capability_id": str(affinity_cap)})
  3067. status, _ = fake3.ask_now()
  3068. check(status == 204,
  3069. "a tag-capable non-affine saddle is refused the parked task (204)")
  3070. check(wait_for(lambda: len(fake2.wake_log) > wakes_before_2 or False,
  3071. timeout=30, what="affine wake") is not None,
  3072. "the dispatcher's wake targets the affine saddle")
  3073. status, resume_payload = fake2.ask_now()
  3074. fake2.ask_on_wake = True # restored whatever the claim returned
  3075. fake2.ask_on_idle_status = True
  3076. check(status == 200 and int(resume_payload["task"]["id"]) == affinity_parent,
  3077. "the affine saddle's ask claims the parked task")
  3078. check("Affinity child" in (resume_payload.get("resume_notice") or "")
  3079. and "affinity child finished" in (resume_payload.get("resume_notice") or ""),
  3080. "the resume assignment carries the subtask report notice")
  3081. check(wait_task_done(auth, affinity_parent, timeout=60) is not None,
  3082. "the resumed parent completes")
  3083. print("== phase 1: lost wake -> offline + re-wake of the next capable ==")
  3084. # racer-three sleeps through wakes (a daemon that died after its
  3085. # last 204): the guard window (SADDLE_LOST_WAKE_S=2) passes with
  3086. # no ask, the saddle goes offline, and the work moves to the
  3087. # next capable sleeping saddle.
  3088. fake3.ask_on_wake = False
  3089. fake3.ask_on_idle_status = False # sleeps through everything
  3090. saddles_state = slot_public(auth, "/saddles", "saddles")
  3091. act(auth, saddles_state["create_capability"], {"name": "lostwake"})
  3092. lostwake_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  3093. if c["name"] == "lostwake"][0])
  3094. racer3_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3095. if s["name"] == "racer-three"][0]
  3096. act(auth, racer3_card["attach_capability"], {"capability_id": str(lostwake_cap)})
  3097. # Racer-three must be asleep to be woken: park it with a manual
  3098. # 204 ask (nothing offerable for it — the task below is not
  3099. # created yet).
  3100. status, _ = fake3.ask_now()
  3101. check(status == 204, "racer-three sleeps before the lost-wake run")
  3102. task_id_tmp = create_queued(auth, {
  3103. "project_id": "1",
  3104. "title": "Lost wake task",
  3105. "priority": "2",
  3106. "capability_id": [str(lostwake_cap)],
  3107. })
  3108. lost_task = task_id_tmp
  3109. check(wait_for(lambda: len(fake3.wake_log) > 0 or False,
  3110. timeout=30, what="wake sent to racer-three") is not None,
  3111. "the only capable sleeping saddle is woken")
  3112. check(wait_for(lambda: [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3113. if s["name"] == "racer-three"][0]["status"] == "offline"
  3114. or False, timeout=30, what="lost-wake -> offline") is not None,
  3115. "no ask after the wake: the saddle goes offline (lost-wake guard)")
  3116. check(task_state(auth, lost_task)["status"] == "queued",
  3117. "the task is untouched while its wake was pending")
  3118. # Re-wake the next capable: the healthy saddle gains the tag and
  3119. # picks the work up via its own wake -> ask -> allocate roundtrip.
  3120. fake_card = saddle_card(auth, "fake-saddle")
  3121. act(auth, fake_card["attach_capability"], {"capability_id": str(lostwake_cap)})
  3122. wakes_before = len(fake.wake_log)
  3123. check(wait_for(lambda: len(fake.wake_log) > wakes_before or False,
  3124. timeout=30, what="re-wake of the next capable") is not None,
  3125. "the next capable sleeping saddle is woken for the task")
  3126. check(wait_task_done(auth, lost_task, timeout=60) is not None,
  3127. "wake -> ask -> allocate: the task completes on the healthy saddle")
  3128. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == lost_task
  3129. for p in fake.start_payloads) or False,
  3130. timeout=10, what="lost task assignment captured") is not None,
  3131. "the re-woken saddle's ask produced the assignment")
  3132. fake3.ask_on_wake = True
  3133. fake3.ask_on_idle_status = True
  3134. print("== phase 3: capability context, catalogue, subtask tags ==")
  3135. # Assignment capability context (plan 10 phase 3): every
  3136. # assignment carries {saddle_tags: the allocated saddle's tags,
  3137. # fleet_tags: every tag name known across the fleet} — the
  3138. # daemon renders it into its system prompt. fake-saddle carries
  3139. # gpu + lostwake; the catalogue is affinity-tag, gpu, lostwake.
  3140. fake2.ask_on_register = False
  3141. fake2.ask_on_wake = False
  3142. fake2.ask_on_terminal = False
  3143. fake2.ask_on_idle_status = False
  3144. fake2.settle_asks() # converted sleep: in-flight trigger asks land
  3145. task_id_tmp = create_queued(auth, {
  3146. "project_id": "1",
  3147. "title": "Capabilities context",
  3148. "priority": "2",
  3149. })
  3150. cap_task = task_id_tmp
  3151. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == cap_task
  3152. for p in fake.start_payloads) or False,
  3153. timeout=60, what="capabilities-context task dispatched") is not None,
  3154. "the capabilities-context task dispatches")
  3155. cap_payload = [p for p in fake.start_payloads
  3156. if int(p.get("task", {}).get("id") or 0) == cap_task][-1]
  3157. context = cap_payload.get("capabilities") or {}
  3158. check(context.get("saddle_tags") == ["gpu", "lostwake"],
  3159. "assignment saddle_tags = the allocated saddle's tags (%s)"
  3160. % context.get("saddle_tags"))
  3161. check(context.get("fleet_tags") == ["affinity-tag", "gpu", "lostwake"],
  3162. "assignment fleet_tags = every tag name known across the fleet (%s)"
  3163. % context.get("fleet_tags"))
  3164. check(wait_task_done(auth, cap_task, timeout=60) is not None,
  3165. "capabilities-context task completes")
  3166. # GET /api/saddle/capabilities feeds the daemon's `saddle
  3167. # capabilities` CLI: per-saddle-token auth, tag order by name,
  3168. # each tag with the saddles carrying it (name order).
  3169. status, _ = saddle_api("GET", "/api/saddle/capabilities", None)
  3170. check(status == 401, "capabilities endpoint rejects a missing token")
  3171. status, body = saddle_api("GET", "/api/saddle/capabilities", "saddle-bogus")
  3172. check(status == 401, "capabilities endpoint rejects a wrong token")
  3173. status, body = saddle_api("GET", "/api/saddle/capabilities", fake_token)
  3174. catalogue = {t["name"]: t["saddles"] for t in body.get("tags", [])}
  3175. check(status == 200 and body.get("ok") is True
  3176. and [t["name"] for t in body.get("tags", [])]
  3177. == ["affinity-tag", "gpu", "lostwake"],
  3178. "catalogue lists every tag by name")
  3179. check(catalogue.get("gpu") == ["fake-saddle"]
  3180. and catalogue.get("affinity-tag") == ["racer-three", "racer-two"]
  3181. and catalogue.get("lostwake") == ["fake-saddle", "racer-three"],
  3182. "each tag names its carrying saddles in name order (%s)" % catalogue)
  3183. # Subtask capabilities (`saddle subtask --capability`): explicit
  3184. # only — never inherited — and validated against the operator
  3185. # tag set; unknown tags reject with the valid list in the error
  3186. # (the agent reads it verbatim in the CLI output).
  3187. conn = db()
  3188. tasks_before = conn.execute("select count(*) from tasks").fetchone()[0]
  3189. conn.close()
  3190. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  3191. "parent_task_id": cap_task, "title": "Bad tag subtask",
  3192. "capabilities": ["gpu", "warp-drive"]})
  3193. conn = db()
  3194. tasks_after = conn.execute("select count(*) from tasks").fetchone()[0]
  3195. conn.close()
  3196. check(status == 400 and tasks_after == tasks_before,
  3197. "an unknown capability tag rejects the subtask call")
  3198. error_text = body.get("error", "")
  3199. check("warp-drive" in error_text and "valid tags" in error_text
  3200. and "gpu" in error_text and "lostwake" in error_text,
  3201. "the rejection names the unknown tag and the valid list (%s)" % error_text)
  3202. status, _ = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  3203. "parent_task_id": cap_task, "title": "Bad shape subtask",
  3204. "capabilities": "gpu"})
  3205. check(status == 400, "a non-array capabilities member rejects")
  3206. # Explicit-only: a parent requiring 'lostwake' (the lost-wake
  3207. # task) gets a subtask tagged just 'gpu' — nothing inherited.
  3208. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  3209. "parent_task_id": lost_task, "title": "Explicit tag subtask",
  3210. "capabilities": ["gpu"]})
  3211. check(status == 200 and body.get("ok") is True,
  3212. "a valid tag creates the subtask synchronously")
  3213. tagged_subtask = int(body["task_id"])
  3214. detail = task_state(auth, tagged_subtask)
  3215. check([c["name"] for c in detail.get("capabilities", [])] == ["gpu"],
  3216. "the subtask carries exactly the explicit tag (no inheritance)")
  3217. check([c["name"] for c in task_state(auth, lost_task).get("capabilities", [])]
  3218. == ["lostwake"],
  3219. "the parent keeps its own requirement")
  3220. conn = db()
  3221. tag_rows = conn.execute(
  3222. "select c.name from task_capabilities tc"
  3223. " join capabilities c on c.id = tc.capability_id"
  3224. " where tc.task_id = ?", (tagged_subtask,)).fetchall()
  3225. conn.close()
  3226. check([r[0] for r in tag_rows] == ["gpu"],
  3227. "the task_capabilities rows record the explicit tag")
  3228. # Without the parent's auto-enqueue option the callback's subtask
  3229. # lands in draft — the human Enqueue action queues it (plan 11).
  3230. check(detail["status"] == "draft", "callback subtask without auto-enqueue lands in draft")
  3231. enqueue_task(auth, tagged_subtask)
  3232. # The queued subtask dispatches (gpu-carrier) and settles before
  3233. # the slow-wake run needs an empty queue.
  3234. check(wait_task_done(auth, tagged_subtask, timeout=90) is not None,
  3235. "the tagged subtask dispatches and completes")
  3236. print("== phase 3: dashboard — pinned blocks + activity timeline ==")
  3237. # Pinned OPEN blocking items as link-through cards, newest
  3238. # first; no inline answer forms (the card links to the task
  3239. # page). Fixture: one open question + one open review raised on a
  3240. # local task (stand-ins for the auto-approve grandchild's
  3241. # leftovers) join the probe RFC raised below.
  3242. conn = db()
  3243. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  3244. " values (1, 'Dash fixture task', 'working', 5, strftime('%s','now'))")
  3245. dash_task = conn.execute("select max(id) from tasks").fetchone()[0]
  3246. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  3247. " values (?, ?, strftime('%s','now'))", (dash_task, fake_id))
  3248. dash_session = conn.execute("select max(id) from sessions").fetchone()[0]
  3249. conn.commit()
  3250. conn.close()
  3251. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  3252. {"type": "block", "session_id": dash_session, "data": {
  3253. "question": "Dash fixture question?", "type": "question"}})
  3254. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  3255. {"type": "review", "session_id": dash_session, "data": {
  3256. "summary": "dash fixture review"}})
  3257. # The activity timeline: last <= 50 events, newest first, fed
  3258. # from the existing tables. Fixture rows a few seconds old
  3259. # guarantee the progress kind and a human-authored comment ride
  3260. # along; the probe block below proves same-source ordering.
  3261. conn = db()
  3262. conn.execute("insert into task_comments (task_id, author, body, created_at)"
  3263. " values (?, 'user:admin', 'timeline fixture comment by a human',"
  3264. " strftime('%s','now','-5 seconds'))", (cap_task,))
  3265. conn.execute("insert into task_progress (task_id, summary, created_at)"
  3266. " values (?, 'timeline fixture progress',"
  3267. " strftime('%s','now','-5 seconds'))",
  3268. (cap_task,))
  3269. conn.commit()
  3270. conn.close()
  3271. # The probe RFC raised last: the newest pinned card and the newest
  3272. # timeline event (its raise+answer below prove same-source ordering
  3273. # both ways). The timeline sorts same-second events by source
  3274. # materialisation order, so the probe must land in a LATER clock
  3275. # second than the fixtures — wait for the boundary (bounded).
  3276. fixtures_epoch = int(time.time())
  3277. wait_for(lambda: int(time.time()) > fixtures_epoch or False,
  3278. timeout=3, what="next clock second before the probe raise")
  3279. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  3280. {"type": "block", "session_id": dash_session, "data": {
  3281. "question": "Timeline probe: approve the dashboard flow",
  3282. "type": "rfc", "body": "# Probe"}})
  3283. dash = slot_public(auth, "/", "dashboard")
  3284. pinned = dash.get("pinned", [])
  3285. pinned_ids = [int(b["id"]) for b in pinned]
  3286. check(len(pinned_ids) >= 3 and pinned_ids == sorted(pinned_ids, reverse=True),
  3287. "pinned open blocks order newest-first (%s)" % pinned_ids)
  3288. check({b["block_type"] for b in pinned} >= {"question", "rfc", "review"},
  3289. "pinned cards cover questions, RFCs and reviews")
  3290. check(all(b["question"] and b["task_title"] and int(b["task_id"]) > 0
  3291. for b in pinned)
  3292. and all("answer_action" not in b for b in pinned),
  3293. "pinned cards are link-through only (task title, kind badge, question)")
  3294. activity = dash.get("activity", [])
  3295. epochs = [int(e["at_epoch"]) for e in activity]
  3296. check(0 < len(activity) <= 50 and epochs == sorted(epochs, reverse=True),
  3297. "the timeline shows the last <=50 events newest-first (%d events)"
  3298. % len(activity))
  3299. kinds = {e["kind"] for e in activity}
  3300. check({"created", "started", "dispatch", "comment", "progress", "completed"} <= kinds,
  3301. "the timeline covers status changes, dispatches, comments, progress and"
  3302. " completions (%s)" % sorted(kinds))
  3303. check(any(e["kind"] == "comment" and e["text"].startswith("user:admin:")
  3304. and "fixture comment" in e["text"] for e in activity),
  3305. "comment events carry author + snippet")
  3306. # The freshly raised probe pinned as the newest card and landed
  3307. # atop the timeline as block-raised; answering it records
  3308. # block-answered.
  3309. check(dash["pinned"][0]["question"] == "Timeline probe: approve the dashboard flow"
  3310. and int(dash["pinned"][0]["task_id"]) == dash_task,
  3311. "the freshest open block pins as the first card")
  3312. check(dash["activity"][0]["kind"] == "block-raised"
  3313. and "Timeline probe" in dash["activity"][0]["text"],
  3314. "the raised block lands atop the timeline")
  3315. probe_block = [b for b in task_state(auth, dash_task)["blocks"]
  3316. if b["question"] == "Timeline probe: approve the dashboard flow"][0]
  3317. act(auth, probe_block["answer_action"], {"answer": "Ship the dashboard"})
  3318. dash = slot_public(auth, "/", "dashboard")
  3319. activity = dash.get("activity", [])
  3320. raised_at = next((i for i, e in enumerate(activity)
  3321. if e["kind"] == "block-raised" and "Timeline probe" in e["text"]), -1)
  3322. answered_at = next((i for i, e in enumerate(activity)
  3323. if e["kind"] == "block-answered"
  3324. and "Ship the dashboard" in e["text"]), -1)
  3325. check(answered_at >= 0 and raised_at >= 0 and answered_at < raised_at,
  3326. "the answered event sorts above its raised event (newest-first, same source)")
  3327. check(all(b["question"] != "Timeline probe: approve the dashboard flow"
  3328. for b in dash["pinned"]),
  3329. "the answered block leaves the pinned cards")
  3330. print("== phase 3 regression: a slow ask after a wake is never dropped ==")
  3331. # The daemon worker's lost-wake-guard regression: a saddle whose
  3332. # ask arrives SLOWLY after the wake (busy winding down when the
  3333. # wake lands) must not lose the wake — the delayed ask (inside
  3334. # the guard window) claims the task, no re-wake storm fires
  3335. # while it is pending, and the saddle stays healthy. The
  3336. # never-asks path (guard -> offline + re-wake of the next
  3337. # capable) is proven by the racer-three section above.
  3338. check(wait_for(lambda: queued_count() == 0 or False,
  3339. timeout=60, what="queue settles before the slow-wake run") is not None,
  3340. "the queue settles before the slow-wake run")
  3341. status, _ = fake2.ask_now()
  3342. check(status == 204, "the slow-ask racer sleeps before its wake")
  3343. saddles_state = slot_public(auth, "/saddles", "saddles")
  3344. act(auth, saddles_state["create_capability"], {"name": "slowwake"})
  3345. slowwake_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")
  3346. ["capabilities"] if c["name"] == "slowwake"][0])
  3347. racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3348. if s["name"] == "racer-two"][0]
  3349. act(auth, racer2_card["attach_capability"], {"capability_id": str(slowwake_cap)})
  3350. fake2.ask_delay = 1.2 # slow: asks well after the wake, inside the 2s guard
  3351. fake2.ask_on_wake = True
  3352. wakes_before_slow = len(fake2.wake_log)
  3353. task_id_tmp = create_queued(auth, {
  3354. "project_id": "1",
  3355. "title": "Slow wake task",
  3356. "priority": "2",
  3357. "capability_id": [str(slowwake_cap)],
  3358. })
  3359. slow_task = task_id_tmp
  3360. check(wait_for(lambda: len(fake2.wake_log) > wakes_before_slow or False,
  3361. timeout=30, what="the slow-ask racer woken") is not None,
  3362. "the only capable sleeping saddle is woken")
  3363. check(wait_task_done(auth, slow_task, timeout=120) is not None,
  3364. "the delayed ask claims the task — the wake was not dropped")
  3365. check(any(a.get("status") == 200
  3366. and int((a.get("payload") or {}).get("task", {}).get("id") or 0)
  3367. == slow_task for a in fake2.asks),
  3368. "the slow ask's 200 carries the task")
  3369. check(len(fake2.wake_log) == wakes_before_slow + 1,
  3370. "the pending-wake skip re-wakes nobody while the slow ask is in flight")
  3371. racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3372. if s["name"] == "racer-two"][0]
  3373. check(racer2_card["status"] in ("idle", "busy"),
  3374. "the slow asker never went offline (guard did not fire)")
  3375. fake2.ask_delay = 0.0
  3376. fake2.ask_on_wake = False # stays out of the pool below
  3377. # =============================================================================
  3378. # Bundle: lifecycle — offline requeues, the hung-request tick survival,
  3379. # parked-refresh bookkeeping, cancellation, the health panel, raw-log
  3380. # losslessness, git identity and the live blocked-push roundtrip.
  3381. # =============================================================================
  3382. def bundle_lifecycle(ctx):
  3383. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  3384. app = ctx.app
  3385. print("== plan 11: draft state machine, edit matrix, enqueue/delete guards ==")
  3386. # New tasks default to draft; Enqueue is the only path to the
  3387. # queue (there is no agent path — the ingest API refuses to queue).
  3388. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3389. "project_id": "1",
  3390. "title": "Draft lifecycle task",
  3391. "priority": "4",
  3392. })
  3393. draft_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3394. if t["title"] == "Draft lifecycle task"][0])
  3395. detail = task_state(auth, draft_task)
  3396. check(detail["status"] == "draft", "a created task lands in draft")
  3397. check(detail.get("enqueue") is not None and detail.get("delete") is not None,
  3398. "draft exposes the enqueue + delete actions")
  3399. check(detail.get("cancel") is not None, "the cancel action stays authored")
  3400. status, page = http("GET", "/tasks/%d" % draft_task, timeout=15)
  3401. check(status == 200 and "Enqueue task" in page and ">Delete<" in page,
  3402. "the task page shows Enqueue + Delete in draft")
  3403. check("Mark done" not in page,
  3404. "the user Mark-complete action is gone (bot-only completion)")
  3405. # A draft dependency blocks enqueue with a shown error naming it.
  3406. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3407. "project_id": "1",
  3408. "title": "Draft dependency",
  3409. "priority": "4",
  3410. })
  3411. draft_dep = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3412. if t["title"] == "Draft dependency"][0])
  3413. act(auth, detail["dependency"], {"depends_on_task_id": str(draft_dep)})
  3414. directives_result = act(auth, task_state(auth, draft_task)["enqueue"], {})
  3415. check(any(d.get("type") == "notify" and d.get("kind") == "error"
  3416. and "Draft dependency" in (d.get("message") or "")
  3417. for d in directives_result),
  3418. "enqueue is refused with an error naming the draft dependency (%s)"
  3419. % [d.get("message") for d in directives_result if d.get("type") == "notify"])
  3420. check(task_state(auth, draft_task)["status"] == "draft",
  3421. "the blocked enqueue leaves the task in draft")
  3422. # The edit matrix starts in DRAFT: every field except status.
  3423. act(auth, detail["update"], {
  3424. "title": "Draft lifecycle task (edited)",
  3425. "description": "Now with a description",
  3426. "priority": "2",
  3427. "task_branch": "feature/draft",
  3428. "branch_from": "main",
  3429. "plan_auto_approve": "on",
  3430. "auto_enqueue_subtasks": "on",
  3431. "auto_approve": "on",
  3432. })
  3433. detail = task_state(auth, draft_task)
  3434. check(detail["title"] == "Draft lifecycle task (edited)"
  3435. and detail["description"] == "Now with a description"
  3436. and detail["priority"] == 2
  3437. and detail["task_branch"] == "feature/draft"
  3438. and detail["branch_from"] == "main",
  3439. "draft edits apply to every field")
  3440. check(detail.get("plan_auto_approve") is True
  3441. and detail.get("auto_enqueue_subtasks") is True
  3442. and detail.get("auto_approve") is True,
  3443. "draft edits apply the task options")
  3444. # A hold tag nothing carries keeps the dependent QUEUED (never
  3445. # dispatched) through the remaining edit-matrix checks.
  3446. saddles_state = slot_public(auth, "/saddles", "saddles")
  3447. act(auth, saddles_state["create_capability"], {"name": "holdlock"})
  3448. holdlock_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")
  3449. ["capabilities"] if c["name"] == "holdlock"][0])
  3450. act(auth, detail["attach_capability"], {"capability_id": str(holdlock_cap)})
  3451. # Enqueue the dependency, then the dependent goes through.
  3452. enqueue_task(auth, draft_dep)
  3453. enqueue_task(auth, draft_task)
  3454. check(task_state(auth, draft_task)["status"] == "queued",
  3455. "enqueue succeeds once no dependency is in draft")
  3456. # Drafts are never dispatched even with an idle saddle available.
  3457. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3458. "project_id": "1",
  3459. "title": "Never-dispatched draft",
  3460. "priority": "1",
  3461. })
  3462. silent_draft = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3463. if t["title"] == "Never-dispatched draft"][0])
  3464. check(quiet_window(quiet_seconds(),
  3465. lambda: task_state(auth, silent_draft)["status"] == "draft",
  3466. "draft never dispatched"),
  3467. "an idle saddle never picks a draft up (no agent enqueue path)")
  3468. # Once work can begin (queued here), only priority is editable.
  3469. act(auth, detail["update"], {
  3470. "title": "Should not apply",
  3471. "priority": "3",
  3472. })
  3473. detail = task_state(auth, draft_task)
  3474. check(detail["title"] == "Draft lifecycle task (edited)" and detail["priority"] == 3,
  3475. "non-draft edits apply priority only (title locked)")
  3476. # A terminal task edits nothing.
  3477. conn = db()
  3478. conn.execute("update tasks set status = 'done', completed_at = strftime('%s','now')"
  3479. " where id = ?", (draft_task,))
  3480. conn.commit()
  3481. conn.close()
  3482. act(auth, task_state(auth, draft_task)["update"], {"priority": "8"})
  3483. check(task_state(auth, draft_task)["priority"] == 3,
  3484. "terminal tasks edit nothing (priority locked)")
  3485. # Capabilities lock once a worker has the job: editable while
  3486. # queued, locked once working.
  3487. act(auth, saddles_state["create_capability"], {"name": "editlock"})
  3488. editlock_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")
  3489. ["capabilities"] if c["name"] == "editlock"][0])
  3490. act(auth, saddle_card(auth, "fake-saddle")["attach_capability"],
  3491. {"capability_id": str(editlock_cap)})
  3492. locked_task = create_queued(auth, {"project_id": "1", "title": "Caplock task",
  3493. "priority": "2"})
  3494. detail = task_state(auth, locked_task)
  3495. act(auth, detail["attach_capability"], {"capability_id": str(editlock_cap)})
  3496. check(any(c["name"] == "editlock"
  3497. for c in task_state(auth, locked_task)["capabilities"]),
  3498. "capabilities attach while queued (no worker yet)")
  3499. check(wait_for(lambda: task_state(auth, locked_task).get("status") == "working"
  3500. or False, timeout=60, what="caplock working") is not None,
  3501. "the caplock task reaches working")
  3502. detach_directives = act(auth, task_state(auth, locked_task)["capabilities"][0]["detach"], {})
  3503. check(any(d.get("type") == "notify" and d.get("kind") == "error"
  3504. for d in detach_directives),
  3505. "capabilities are locked once a worker has the job")
  3506. # The ingest API has no queue transition either (only working and
  3507. # blocked are accepted states).
  3508. conn = db()
  3509. caplock_session = conn.execute(
  3510. "select id from sessions where task_id = ? and ended_at is null",
  3511. (locked_task,)).fetchone()
  3512. conn.close()
  3513. if caplock_session is not None:
  3514. status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  3515. {"type": "status", "session_id": caplock_session[0],
  3516. "data": {"state": "queued"}})
  3517. check(status == 200
  3518. and task_state(auth, locked_task)["status"] in ("working", "done"),
  3519. "the ingest API's status push never queues a task")
  3520. # Delete guards: a clean draft deletes; non-draft refuses; a draft
  3521. # with dependents is refused naming them.
  3522. act(auth, task_state(auth, silent_draft)["delete"], {})
  3523. check(not any(t["id"] == silent_draft
  3524. for t in slot_public(auth, "/tasks", "tasks")["tasks"]),
  3525. "a clean draft task deletes")
  3526. delete_directives = act(auth, task_state(auth, draft_task)["delete"], {})
  3527. check(any(d.get("type") == "notify" and d.get("kind") == "error"
  3528. for d in delete_directives),
  3529. "a non-draft task refuses the delete")
  3530. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3531. "project_id": "1", "title": "Guarded draft", "priority": "4"})
  3532. guarded = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3533. if t["title"] == "Guarded draft"][0])
  3534. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3535. "project_id": "1", "title": "Dependent namer", "priority": "4"})
  3536. namer = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3537. if t["title"] == "Dependent namer"][0])
  3538. act(auth, task_state(auth, namer)["dependency"], {"depends_on_task_id": str(guarded)})
  3539. delete_directives = act(auth, task_state(auth, guarded)["delete"], {})
  3540. check(any(d.get("type") == "notify" and d.get("kind") == "error"
  3541. and "Dependent namer" in (d.get("message") or "")
  3542. for d in delete_directives),
  3543. "deleting a task with dependents is refused naming them")
  3544. print("== offline requeues ==")
  3545. # Tasks caught working on a saddle that went offline requeue with a
  3546. # bot comment each tick.
  3547. task_id_tmp = create_queued(auth, {
  3548. "project_id": "1",
  3549. "title": "Offline saddle task",
  3550. "priority": "2",
  3551. })
  3552. offline_task = task_id_tmp
  3553. check(wait_for(lambda: task_state(auth, offline_task).get("status") == "working"
  3554. or False, timeout=60, what="offline task starts working") is not None,
  3555. "task reaches working before its saddle disappears")
  3556. # Hold the daemon mid-round across the requeue window (the
  3557. # original relied on its 8s work period outlasting the 5s tick):
  3558. # the task completing on its own would race the offline pass and
  3559. # skip the requeue the check exists to observe.
  3560. fake.hold_state = True
  3561. conn = db()
  3562. conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
  3563. conn.commit()
  3564. conn.close()
  3565. # The requeue frees the saddle in the same tick, so 'queued' is
  3566. # transient; the durable evidence is the bot comment (and the task
  3567. # completing again on the healthy daemon).
  3568. check(wait_for(lambda: any(c["body"] == "saddle went offline; task requeued"
  3569. for c in task_state(auth, offline_task).get("comments", []))
  3570. or False, timeout=30, what="offline requeue comment") is not None,
  3571. "offline saddle requeues the task")
  3572. fake.hold_state = False # release: the round ends, the daemon re-asks
  3573. check(wait_task_done(auth, offline_task, timeout=60) is not None,
  3574. "requeued task completes again on the healthy daemon")
  3575. print("== R6 F1: a hanging request cannot freeze the tick ==")
  3576. # One wedged /task-update (never responds) blocks the comment/
  3577. # subtask-event delivery pass; the Soup timeout must fail it and
  3578. # later ticks keep dispatching.
  3579. conn = db()
  3580. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  3581. " values (1, 'Hang survivor', 'queued', 2, strftime('%s','now'))")
  3582. hang_task = conn.execute("select max(id) from tasks").fetchone()[0]
  3583. conn.close()
  3584. task_id_tmp = create_queued(auth, {
  3585. "project_id": "1",
  3586. "title": "Hang runner",
  3587. "priority": "3",
  3588. })
  3589. hang_runner = task_id_tmp
  3590. # A human comment on the running task gives the tick a
  3591. # /task-update delivery to make; the next one hangs.
  3592. check(wait_for(lambda: task_state(auth, hang_runner).get("status") == "working"
  3593. or False, timeout=60, what="hang runner working") is not None,
  3594. "hang runner reaches working")
  3595. act(auth, task_state(auth, hang_runner)["comment"],
  3596. {"body": "operator ping while wedged"})
  3597. fake.hang_next["/task-update"] = 1
  3598. check(wait_task_done(auth, hang_runner, timeout=90) is not None,
  3599. "dispatching task survives a hung daemon request")
  3600. check(wait_for(lambda: any(c.get("text") == "operator ping while wedged"
  3601. or c.get("body") == "operator ping while wedged"
  3602. for c in fake.comment_log) or False,
  3603. timeout=30, what="wedged comment eventually delivered") is not None,
  3604. "the comment delivery retries after the hang")
  3605. # The hung delivery itself fails after the timeout; the system
  3606. # kept moving either way.
  3607. with open(os.path.join(WORK, "app.log")) as handle:
  3608. app_log = handle.read()
  3609. check("tick exceeded" not in app_log or "Dispatch tick failed" in app_log,
  3610. "a hung request never freezes the tick latch")
  3611. print("== R6 F2: refresh must not corrupt parked bookkeeping ==")
  3612. fake.suspend_on_title = "Parked refresh parent"
  3613. task_id_tmp = create_queued(auth, {
  3614. "project_id": "1",
  3615. "title": "Parked refresh parent",
  3616. "priority": "2",
  3617. })
  3618. parked = task_id_tmp
  3619. check(wait_for(lambda: task_state(auth, parked).get("status") == "suspended"
  3620. or False, timeout=60, what="parked parent suspends") is not None,
  3621. "parent parks before the refresh")
  3622. fake_card = saddle_card(auth, "fake-saddle")
  3623. act(auth, fake_card["refresh"], {})
  3624. fake_card = saddle_card(auth, "fake-saddle")
  3625. check(fake_card["status"] == "idle" and int(fake_card["current_task_id"] or 0) == 0,
  3626. "refresh on a parked (suspended) daemon leaves the saddle idle")
  3627. # The parked task keeps its open session and resumes normally.
  3628. conn = db()
  3629. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  3630. " summary, completed_at, created_at) values (1, 'Parked child', 'done', 5,"
  3631. " ?, 'child finished', strftime('%s','now','+1 second'),"
  3632. " strftime('%s','now'))", (parked,))
  3633. conn.commit()
  3634. conn.close()
  3635. check(wait_task_done(auth, parked, timeout=90) is not None,
  3636. "parked parent still resumes after the refresh")
  3637. print("== R7 B1: cancel reaches the daemon ==")
  3638. # Plan 10 phase 2: cancellation is POST /task-update {kind: cancel}
  3639. # to the RUNNING daemon (the abort machinery, workspace kept).
  3640. task_id_tmp = create_queued(auth, {
  3641. "project_id": "1",
  3642. "title": "Cancel live",
  3643. "priority": "2",
  3644. })
  3645. cancel_live = task_id_tmp
  3646. check(wait_for(lambda: task_state(auth, cancel_live).get("status") == "working"
  3647. or False, timeout=60, what="cancel-live working") is not None,
  3648. "task reaches working before the cancel")
  3649. act(auth, task_state(auth, cancel_live)["cancel"], {})
  3650. check(wait_for(lambda: cancel_live in fake.abort_log or False,
  3651. timeout=30, what="daemon abort recorded") is not None,
  3652. "cancelling POSTs task-update cancel to the daemon")
  3653. check(any(c["body"] == "cancelled; daemon aborted"
  3654. for c in task_state(auth, cancel_live).get("comments", [])),
  3655. "the abort outcome is narrated")
  3656. check(task_state(auth, cancel_live)["status"] == "failed",
  3657. "cancelled task lands failed as before")
  3658. # A 500 on the cancel marks the saddle offline (no fence layer
  3659. # anymore); nothing dispatches to it while offline, and the
  3660. # daemon's own ask flips the row back to idle (proof of life).
  3661. task_id_tmp = create_queued(auth, {
  3662. "project_id": "1",
  3663. "title": "Fence runner",
  3664. "priority": "2",
  3665. })
  3666. fence_runner = task_id_tmp
  3667. check(wait_for(lambda: task_state(auth, fence_runner).get("status") == "working"
  3668. or False, timeout=60, what="fence runner working") is not None,
  3669. "second task reaches working")
  3670. fake.abort_failures = 1
  3671. act(auth, task_state(auth, fence_runner)["cancel"], {})
  3672. check(any(c["body"] == "cancelled; daemon unreachable — saddle marked offline"
  3673. for c in task_state(auth, fence_runner).get("comments", [])),
  3674. "failed cancel narrates the offline marking")
  3675. check(saddle_card(auth, "fake-saddle")["status"] == "offline",
  3676. "saddle marked offline after the failed cancel")
  3677. task_id_tmp = create_queued(auth, {
  3678. "project_id": "1",
  3679. "title": "During fence",
  3680. "priority": "3",
  3681. })
  3682. during_fence = task_id_tmp
  3683. # Converted sleep(7): a quiet window spanning several dispatch
  3684. # ticks in which the task MUST stay queued — if the offline
  3685. # saddle ever dispatches it, this fails loudly. The daemon is held
  3686. # mid-round for the window (the original relied on its 8s work
  3687. # period outlasting the sleep): a finishing round would ask, prove
  3688. # liveness and flip the offline row — only after the window may it.
  3689. fake.hold_state = True
  3690. check(quiet_window(quiet_seconds(),
  3691. lambda: task_state(auth, during_fence)["status"] == "queued",
  3692. "offline saddle does not dispatch during_fence"),
  3693. "offline saddle is not dispatchable")
  3694. fake.hold_state = False
  3695. # The (unaborted) daemon finishes its round, goes idle-awake and
  3696. # asks — the ask is the proof of life that flips the row back.
  3697. check(wait_for(lambda: saddle_card(auth, "fake-saddle")["status"] == "idle" or False,
  3698. timeout=30, what="ask flips offline to idle") is not None,
  3699. "the daemon's ask flips the offline row back to idle")
  3700. check(wait_task_done(auth, during_fence, timeout=60) is not None,
  3701. "queued work dispatches once the saddle proves it is alive")
  3702. print("== R7 B3: health panel + pushed raw log from the first round ==")
  3703. # A normally dispatched task pushes its chunks immediately: raw
  3704. # rows exist from the first round (the session-4 gap stays
  3705. # fixed without any probe machinery).
  3706. task_id_tmp = create_queued(auth, {
  3707. "project_id": "1",
  3708. "title": "Raw from first push",
  3709. "priority": "2",
  3710. })
  3711. raw_first = task_id_tmp
  3712. def raw_count(task_id):
  3713. conn = db()
  3714. try:
  3715. return conn.execute(
  3716. "select count(*) from session_raw_log where session_id in"
  3717. " (select id from sessions where task_id = ?)", (task_id,)).fetchone()[0]
  3718. finally:
  3719. conn.close()
  3720. check(wait_for(lambda: raw_count(raw_first) > 0 or False,
  3721. timeout=60, what="raw rows from first push") is not None,
  3722. "pushed chunks land from the first round (session-4 gap)")
  3723. check(wait_task_done(auth, raw_first, timeout=60) is not None,
  3724. "raw-first task completes")
  3725. # Health panel: a manual refresh populates the snapshot.
  3726. fake_card = saddle_card(auth, "fake-saddle")
  3727. act(auth, fake_card["refresh"], {})
  3728. card = saddle_card(auth, "fake-saddle")
  3729. check(card.get("health") in ("healthy", "stale")
  3730. and card.get("daemon_state") in ("working", "idle", "done")
  3731. and card.get("uptime_display", "") != ""
  3732. and card.get("last_contact_display", "") not in ("", "never"),
  3733. "saddles page shows daemon state, uptime and last contact")
  3734. print("== R8: pushed chunk ingestion is lossless ==")
  3735. # The daemon's opening flush: a coloured prompt run, plain text,
  3736. # a bare CRLF between prompts and the tail — every non-empty
  3737. # chunk must land, exactly once, in seq order. A mid-flight
  3738. # redelivery of the whole batch (the outbox's at-least-once
  3739. # retry) consumes the duplicates.
  3740. corpus = [
  3741. {"seq": 1, "text": "\x1b[01;32muser@host\x1b[00m:\x1b[01;34m~/task-1\x1b[00m$ "},
  3742. {"seq": 2, "text": "echo hello\r\n"},
  3743. {"seq": 3, "text": "\r\n"},
  3744. {"seq": 4, "text": "hello\r\n"},
  3745. {"seq": 5, "text": "\x1b[01;32muser@host\x1b[00m:\x1b[01;34m~/task-1\x1b[00m$ "},
  3746. {"seq": 6, "text": "done\r\n"},
  3747. ]
  3748. fake.scripted_corpus = corpus
  3749. fake.hold_state = True
  3750. task_id_tmp = create_queued(auth, {
  3751. "project_id": "1",
  3752. "title": "Lossless log task",
  3753. "priority": "2",
  3754. })
  3755. lossless = task_id_tmp
  3756. def stored_seqs(task_id):
  3757. conn = db()
  3758. try:
  3759. return [r[0] for r in conn.execute(
  3760. "select seq from session_raw_log where session_id in"
  3761. " (select id from sessions where task_id = ?)"
  3762. " order by seq", (task_id,))]
  3763. finally:
  3764. conn.close()
  3765. check(wait_for(lambda: task_state(auth, lossless).get("status") == "working"
  3766. or False, timeout=60, what="lossless task working") is not None,
  3767. "lossless task dispatches and pushes its opening flush")
  3768. check(wait_for(lambda: stored_seqs(lossless) == [1, 2, 3, 4, 5, 6] or False,
  3769. timeout=30, what="all six chunks stored") is not None,
  3770. "every pushed chunk lands in session_raw_log, CRLF-only included (%s)"
  3771. % stored_seqs(lossless))
  3772. status, _ = saddle_api("POST", "/api/saddle/chunks", fake.token,
  3773. {"session_id": fake.session_id,
  3774. "chunks": [dict(chunk) for chunk in corpus]})
  3775. check(status == 200 and stored_seqs(lossless) == [1, 2, 3, 4, 5, 6],
  3776. "a full-batch redelivery stores nothing new (exactly-once)")
  3777. request = urllib.request.Request(
  3778. WEB + "/terminal-stream/%d" % [r[0] for r in db().execute(
  3779. "select id from sessions where task_id = ?", (lossless,))][0])
  3780. with urllib.request.urlopen(request, timeout=15) as stream:
  3781. texts = []
  3782. deadline = time.monotonic() + 8
  3783. while time.monotonic() < deadline and len(texts) < 6:
  3784. line = stream.readline().decode().strip()
  3785. if not line.startswith("data: ") or line == "data: ping":
  3786. continue
  3787. event = json.loads(line[6:])
  3788. if event.get("type") == "raw":
  3789. texts.append(event.get("text") or "")
  3790. check("\r\n" in texts and len(texts) == 6,
  3791. "SSE serves the whitespace chunk unfiltered")
  3792. fake.hold_state = False
  3793. fake.scripted_corpus = None
  3794. check(wait_task_done(auth, lossless, timeout=60) is not None,
  3795. "lossless task completes")
  3796. print("== R9 B1/B2: git identity + ssh key ==")
  3797. # The key travels on /status into the saddle row and the page view
  3798. # (the manual refresh's exceptional probe).
  3799. fake.ssh_public_key = "ssh-ed25519 AAAAC3NzaC1lZDI1teStKey saddle-1@fedora"
  3800. fake_card = saddle_card(auth, "fake-saddle")
  3801. check(fake_card.get("ssh_public_key", "") == "",
  3802. "no key shown before the daemon reports one")
  3803. act(auth, fake_card["refresh"], {})
  3804. check(wait_for(lambda: "AAAAC3NzaC1lZDI1teStKey" in
  3805. saddle_card(auth, "fake-saddle").get("ssh_public_key", "") or False,
  3806. timeout=30, what="key lands on the row") is not None,
  3807. "reported ssh_public_key persists onto the saddle row")
  3808. conn = db()
  3809. row_key = conn.execute("select ssh_public_key from saddles where name = 'fake-saddle'"
  3810. ).fetchone()[0]
  3811. conn.close()
  3812. check(row_key == fake.ssh_public_key,
  3813. "the key is stored verbatim in the database")
  3814. # Identity edit flows through the UI into the assignment payload.
  3815. fake_card = saddle_card(auth, "fake-saddle")
  3816. act(auth, fake_card["update_identity"],
  3817. {"git_name": "clanker-bot", "git_email": "bot@unitatem.net"})
  3818. fake_card = saddle_card(auth, "fake-saddle")
  3819. check(fake_card.get("git_name") == "clanker-bot"
  3820. and fake_card.get("git_email") == "bot@unitatem.net",
  3821. "inline edit saves the git identity")
  3822. task_id_tmp = create_queued(auth, {
  3823. "project_id": "1",
  3824. "title": "Identity task",
  3825. "priority": "2",
  3826. })
  3827. identity_task = task_id_tmp
  3828. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == identity_task
  3829. for p in fake.start_payloads) or False,
  3830. timeout=60, what="identity task dispatched") is not None,
  3831. "identity task dispatches")
  3832. payload = [p for p in fake.start_payloads
  3833. if int(p.get("task", {}).get("id") or 0) == identity_task][-1]
  3834. identity = payload.get("git_identity") or {}
  3835. check(identity.get("name") == "clanker-bot"
  3836. and identity.get("email") == "bot@unitatem.net",
  3837. "the assignment carries the saddle's git_identity")
  3838. # Earlier assignments (identity unset) carry no git_identity member.
  3839. check(all("git_identity" not in p for p in fake.start_payloads[:-1]),
  3840. "unset identity omits the member (daemon fallback applies)")
  3841. # Clearing the identity omits it from later assignments.
  3842. fake_card = saddle_card(auth, "fake-saddle")
  3843. act(auth, fake_card["update_identity"], {"git_name": "", "git_email": ""})
  3844. check(saddle_card(auth, "fake-saddle").get("git_name", "") == "",
  3845. "clearing the identity empties the row")
  3846. check(wait_task_done(auth, identity_task, timeout=60) is not None,
  3847. "identity task completes")
  3848. print("== phase 2: blocked push -> block row + pulse ==")
  3849. # The browser flow on each pulse: statum.act(task.refresh) — the
  3850. # sealed GET action must rehydrate the task slot with the new
  3851. # block the daemon just PUSHED (state blocked + block payload).
  3852. task_id_tmp = create_queued(auth, {
  3853. "project_id": "1",
  3854. "title": "Live block task",
  3855. "priority": "2",
  3856. })
  3857. live_task = task_id_tmp
  3858. check(wait_for(lambda: task_state(auth, live_task).get("status") == "working"
  3859. or False, timeout=60, what="live task working") is not None,
  3860. "live-block task reaches working")
  3861. fake.hold_state = True
  3862. fake.state = "blocked" # the daemon parks on its question
  3863. status = fake.push_results("blocked", block={
  3864. "question": "Pick one", "type": "question", "options": ["A", "B"]})
  3865. check(status == 200, "blocked state push accepted")
  3866. # The parked daemon is idle-awake: it asks (and sleeps on 204).
  3867. fake.maybe_trigger_ask("parked")
  3868. check(wait_for(lambda: any(b.get("question") == "Pick one"
  3869. for b in task_state(auth, live_task).get("blocks", []))
  3870. or False, timeout=30, what="block lands in state") is not None,
  3871. "the pushed block reaches the sealed refresh without a reload")
  3872. detail = task_state(auth, live_task)
  3873. check(detail.get("status") == "blocked",
  3874. "the blocked push flips the task to blocked")
  3875. pushed_block = [b for b in detail["blocks"] if b.get("question") == "Pick one"][0]
  3876. conn = db()
  3877. options_json = conn.execute(
  3878. "select options from task_blocks where id = ?", (int(pushed_block["id"]),)
  3879. ).fetchone()[0]
  3880. conn.close()
  3881. check(json.loads(options_json) == ["A", "B"],
  3882. "the block row carries its options payload")
  3883. check(any(s["status"] == "idle"
  3884. for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
  3885. "the blocked push parks the saddle")
  3886. detail = task_state(auth, live_task)
  3887. check(detail.get("refresh") is not None,
  3888. "the sealed GET refresh action is still authored")
  3889. status, _ = http("GET", detail["refresh"]["uri"], timeout=15)
  3890. check(status in (200, 401),
  3891. "the refresh URI resolves as a GET (client roundtrip shape)")
  3892. act(auth, pushed_block["answer_action"], {"answer": "A"})
  3893. # The answer makes the parked task resume-offerable: the affine
  3894. # saddle is woken, asks, and its assignment carries the answer.
  3895. pushes_before = len([p for p in fake.start_payloads
  3896. if int(p.get("task", {}).get("id") or 0) == live_task])
  3897. check(wait_for(lambda: len([p for p in fake.start_payloads
  3898. if int(p.get("task", {}).get("id") or 0) == live_task])
  3899. > pushes_before or False, timeout=60,
  3900. what="blocked task resumes") is not None,
  3901. "answering the pushed block resumes the parked workspace")
  3902. fake.hold_state = False # the resumed round runs to completion
  3903. resume_payload = [p for p in fake.start_payloads
  3904. if int(p.get("task", {}).get("id") or 0) == live_task][-1]
  3905. answers = resume_payload.get("task", {}).get("answers", [])
  3906. check(any(a.get("answer") == "A" for a in answers),
  3907. "the resume assignment carries the human's answer")
  3908. check(wait_task_done(auth, live_task, timeout=60) is not None,
  3909. "live-block task completes after answering")
  3910. # =============================================================================
  3911. # Bundle: answers — the answer-resume machinery in both daemon shapes
  3912. # (sleeping affine wake, mid-round verdict delivery), offline requeue
  3913. # keeping parked-with-answered tasks, the staleness probe, and the
  3914. # startup token migration (this shard's one deliberate app restart).
  3915. # =============================================================================
  3916. def bundle_answers(ctx):
  3917. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  3918. app = ctx.app
  3919. print("== phase 2: answer resume — wake -> ask -> affinity claim ==")
  3920. # The full live sequence against a daemon that asks after its
  3921. # park: ask -> assignment -> blocked push (park) -> ask -> 204
  3922. # -> sleeps (row idle) -> human answers -> the dispatcher WAKES
  3923. # the sleeping affine saddle -> its ask allocates the resume,
  3924. # the verdict riding the assignment -> the session continues.
  3925. # A private capability tag keeps the fleet's other sleeping
  3926. # saddles from claiming these tasks.
  3927. saddles_state = slot_public(auth, "/saddles", "saddles")
  3928. act(auth, saddles_state["create_capability"], {"name": "answerflow"})
  3929. answerflow_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  3930. if c["name"] == "answerflow"][0])
  3931. def answerflow_daemon(**kwargs):
  3932. daemon = FakeSaddle()
  3933. daemon.ask_on_register = False # manual, deterministic asks
  3934. daemon.hold_state = True # the tests drive the lifecycle
  3935. for key, value in kwargs.items():
  3936. setattr(daemon, key, value)
  3937. daemon.serve()
  3938. return daemon
  3939. def register_answerflow(daemon, name):
  3940. act(auth, slot_public(auth, "/saddles", "saddles")["register"], {
  3941. "name": name, "host": "127.0.0.1:%d" % daemon.port})
  3942. act(auth, saddle_card(auth, name)["attach_capability"],
  3943. {"capability_id": str(answerflow_cap)})
  3944. def create_answerflow_task(title):
  3945. return create_queued(auth, {
  3946. "project_id": "1", "title": title, "priority": "2",
  3947. "capability_id": [str(answerflow_cap)]})
  3948. def retire_daemon(daemon, name):
  3949. # deterministic teardown: a dead daemon and an offline row
  3950. # never interfere with the sections that follow
  3951. daemon.kill()
  3952. act(auth, saddle_card(auth, name)["refresh"], {})
  3953. sleeper = answerflow_daemon(ask_on_wake=True)
  3954. register_answerflow(sleeper, "resume-sleeper")
  3955. sleeping_task = create_answerflow_task("Sleeping resume task")
  3956. ask_status, assignment = sleeper.ask_now()
  3957. check(ask_status == 200 and int(assignment["task"]["id"]) == sleeping_task,
  3958. "the daemon's ask claims the queued task")
  3959. sleeping_session = assignment.get("session_id")
  3960. sleeper.state = "blocked"
  3961. check(sleeper.push_results("blocked", block={
  3962. "question": "Pick a side", "type": "question",
  3963. "options": ["Port", "Starboard"]}) == 200,
  3964. "the blocked push parks the session")
  3965. check(wait_for(lambda: task_state(auth, sleeping_task).get("status") == "blocked"
  3966. or False, timeout=30, what="sleeping task parked") is not None,
  3967. "the task parks as blocked")
  3968. check(saddle_card(auth, "resume-sleeper")["status"] == "idle",
  3969. "the parked session's saddle row reads idle")
  3970. ask_status, _ = sleeper.ask_now()
  3971. check(ask_status == 204,
  3972. "the parked daemon's ask returns 204 — it sleeps until woken")
  3973. # Converted sleep(3): ticks pass with the daemon asleep and no
  3974. # wake may arrive — a wake during the window fails loudly.
  3975. check(quiet_window(quiet_seconds(),
  3976. lambda: len(sleeper.wake_log) == 0,
  3977. "sleeping daemon never woken"),
  3978. "the sleeping daemon is never woken while the block waits")
  3979. detail = task_state(auth, sleeping_task)
  3980. side_block = [b for b in detail["blocks"] if b["question"] == "Pick a side"][0]
  3981. act(auth, side_block["answer_action"], {"option": "Starboard"})
  3982. check(wait_for(lambda: len(sleeper.wake_log) > 0 or False,
  3983. timeout=30, what="wake after the answer") is not None,
  3984. "answering the block wakes the sleeping affine saddle")
  3985. # Answering = allocating (plan 11): the resume assignment carries
  3986. # the verdict as resume_notice (stable wording) AND a FRESH
  3987. # session id — the parked row closed, the daemon keys its resumed
  3988. # pushes on the new one.
  3989. check(wait_for(lambda: any(
  3990. int(p.get("task", {}).get("id") or 0) == sleeping_task
  3991. and p.get("session_id") != sleeping_session
  3992. and p.get("resume_notice") == "Human selected: Starboard"
  3993. and any(a.get("answer") == "Starboard"
  3994. for a in p.get("task", {}).get("answers", []))
  3995. for p in sleeper.start_payloads[1:]) or False,
  3996. timeout=30, what="resume allocation carries the verdict") is not None,
  3997. "the woken daemon's ask allocates the resume — fresh session, notice carried")
  3998. # Barrier: the resume's own working push must commit before the
  3999. # manual done push (a late working write would otherwise race the
  4000. # completion's full-row write).
  4001. check(wait_for(lambda: task_state(auth, sleeping_task).get("status") == "working"
  4002. or False, timeout=15, what="resumed round working") is not None,
  4003. "the resumed round reports working before its completion push")
  4004. # The resumed round runs to completion keyed on the fresh session.
  4005. sleeper.push_results("done", summary="sleep-resumed round complete")
  4006. check(wait_task_done(auth, sleeping_task, timeout=30) is not None,
  4007. "the resumed session continues to completion")
  4008. conn = db()
  4009. rows = conn.execute(
  4010. "select id, outcome, ended_at is not null from sessions where task_id = ?"
  4011. " order by id", (sleeping_task,)).fetchall()
  4012. conn.close()
  4013. check(len(rows) == 2 and rows[0][1] == "resumed" and rows[0][2]
  4014. and rows[1][1] == "done" and rows[1][2],
  4015. "parked session closed 'resumed'; the resumed round ran on a fresh row to done (%s)"
  4016. % (rows,))
  4017. retire_daemon(sleeper, "resume-sleeper")
  4018. print("== phase 2: answer resume — the parked daemon is woken (POST /answer retired) ==")
  4019. # Post Task A every park FREES the daemon: after the blocked push
  4020. # it lands idle and asks (204). The retired POST /answer channel
  4021. # always refuses; the verdict reaches the parked workspace purely
  4022. # through the wake -> ask -> affinity allocation carrying the
  4023. # resume_notice + a fresh session id.
  4024. parked = answerflow_daemon(ask_on_wake=True,
  4025. ask_on_terminal=False)
  4026. register_answerflow(parked, "resume-parked")
  4027. parked_task = create_answerflow_task("Parked resume task")
  4028. ask_status, assignment = parked.ask_now()
  4029. check(ask_status == 200 and int(assignment["task"]["id"]) == parked_task,
  4030. "the daemon's ask claims the task")
  4031. parked_session = assignment.get("session_id")
  4032. parked.state = "blocked"
  4033. check(parked.push_results("blocked", block={
  4034. "question": "Which database?", "type": "question",
  4035. "options": ["SQLite", "Postgres"]}) == 200,
  4036. "the blocked push parks the session mid-round")
  4037. check(wait_for(lambda: task_state(auth, parked_task).get("status") == "blocked"
  4038. or False, timeout=30, what="parked task blocked") is not None,
  4039. "the task parks as blocked")
  4040. check(saddle_card(auth, "resume-parked")["status"] == "idle",
  4041. "the parked session's saddle row reads idle")
  4042. # The freed daemon re-asks (its idle loop) and sleeps on 204.
  4043. ask_status, _ = parked.ask_now()
  4044. check(ask_status == 204,
  4045. "the freed parked daemon asks again and sleeps (204)")
  4046. # The retired channel always refuses — the web never depends on it.
  4047. request = urllib.request.Request(
  4048. "http://127.0.0.1:%d/answer" % parked.port,
  4049. data=json.dumps({"answer": "SQLite", "branch_advanced": False}).encode(),
  4050. method="POST")
  4051. request.add_header("Content-Type", "application/json")
  4052. try:
  4053. urllib.request.urlopen(request, timeout=5)
  4054. refused = False
  4055. except urllib.error.HTTPError as error:
  4056. refused = error.code == 409
  4057. check(refused, "POST /answer always refuses (409) — parks resume via allocation")
  4058. detail = task_state(auth, parked_task)
  4059. db_block = [b for b in detail["blocks"] if b["question"] == "Which database?"][0]
  4060. act(auth, db_block["answer_action"], {"option": "SQLite"})
  4061. check(wait_for(lambda: len(parked.wake_log) > 0 or False,
  4062. timeout=30, what="wake after the answer") is not None,
  4063. "answering the block wakes the parked (idle) affine saddle")
  4064. check(wait_for(lambda: any(
  4065. int(p.get("task", {}).get("id") or 0) == parked_task
  4066. and p.get("session_id") != parked_session
  4067. and p.get("resume_notice") == "Human selected: SQLite"
  4068. for p in parked.start_payloads[1:]) or False,
  4069. timeout=30, what="parked resume allocation") is not None,
  4070. "the parked daemon's ask allocates the resume — fresh session, formatted notice")
  4071. # The hold_state daemon drives its round manually: the resumed
  4072. # round completes on the fresh session row.
  4073. parked.push_results("done", summary="parked round resumed and finished")
  4074. check(wait_task_done(auth, parked_task, timeout=30) is not None,
  4075. "the parked session continues to completion")
  4076. conn = db()
  4077. row = conn.execute("select outcome from sessions where id = ?",
  4078. (parked_session,)).fetchone()
  4079. session_rows = conn.execute(
  4080. "select id, outcome from sessions where task_id = ? order by id",
  4081. (parked_task,)).fetchall()
  4082. conn.close()
  4083. check(row is not None and row[0] == "resumed"
  4084. and len(session_rows) == 2 and session_rows[0] == (parked_session, "resumed"),
  4085. "the parked session closed 'resumed' and the resumed round kept its own row (%s)"
  4086. % (session_rows,))
  4087. retire_daemon(parked, "resume-parked")
  4088. print("== plan 11: plan acceptance — accept / reject / auto-approve ==")
  4089. # Every task's `saddle plan` parks with block.type 'plan'; Accept
  4090. # and Reject-with-feedback drive the same wake -> ask ->
  4091. # allocation resume as any verdict, the stable wording riding the
  4092. # resume_notice; the per-task auto-approve option answers the plan
  4093. # block instantly web-side.
  4094. planner = answerflow_daemon(ask_on_wake=True, ask_on_terminal=False)
  4095. register_answerflow(planner, "plan-planner")
  4096. def park_on_plan(task_id, markdown):
  4097. planner.state = "blocked"
  4098. check(planner.push_results("blocked", block={
  4099. "question": "Plan submitted for review", "type": "plan",
  4100. "body": markdown}) == 200,
  4101. "the plan push parks the session")
  4102. # Wait for the block row itself (the auto-approve variant is
  4103. # answered — and may resume — before any status poll lands).
  4104. check(wait_for(lambda: any(b["block_type"] == "plan"
  4105. for b in task_state(auth, task_id)["blocks"])
  4106. or False, timeout=30, what="plan block raised") is not None,
  4107. "the plan block reaches the task")
  4108. return [b for b in task_state(auth, task_id)["blocks"]
  4109. if b["block_type"] == "plan"][0]
  4110. # (a) Accept applies the plan markdown and resumes with approval.
  4111. accept_task = create_answerflow_task("Plan accept task")
  4112. ask_status, accept_assignment = planner.ask_now()
  4113. check(ask_status == 200 and int(accept_assignment["task"]["id"]) == accept_task,
  4114. "the planner claims the accept task")
  4115. accept_parked_session = accept_assignment.get("session_id")
  4116. plan_block = park_on_plan(accept_task, "# Steps\n\n1. Wire\n2. Test")
  4117. check(plan_block.get("answered") is not True,
  4118. "the plan block waits for a human verdict")
  4119. act(auth, plan_block["answer_action"],
  4120. {"answer": "Plan approved. Proceed with implementation."})
  4121. detail = task_state(auth, accept_task)
  4122. check(detail.get("plan", "").startswith("# Steps"),
  4123. "accept applies the submitted markdown to task.plan")
  4124. check(wait_for(lambda: any(
  4125. int(p.get("task", {}).get("id") or 0) == accept_task
  4126. and p.get("session_id") != accept_parked_session
  4127. and p.get("resume_notice") == "Plan approved. Proceed with implementation."
  4128. for p in planner.start_payloads[1:]) or False,
  4129. timeout=30, what="accept resume allocation") is not None,
  4130. "accept resumes the parked daemon with the approval notice")
  4131. planner.push_results("done", summary="plan accepted and implemented")
  4132. check(wait_task_done(auth, accept_task, timeout=30) is not None,
  4133. "the accepted plan's task completes")
  4134. planner.set_state("idle")
  4135. # (b) Reject-with-feedback loops the feedback notice back.
  4136. reject_task = create_answerflow_task("Plan reject task")
  4137. ask_status, reject_assignment = planner.ask_now()
  4138. check(ask_status == 200 and int(reject_assignment["task"]["id"]) == reject_task,
  4139. "the planner claims the reject task")
  4140. reject_parked_session = reject_assignment.get("session_id")
  4141. plan_block = park_on_plan(reject_task, "# Draft\n\nJust wing it")
  4142. act(auth, plan_block["answer_action"], {"feedback": "Add a test section"})
  4143. detail = task_state(auth, reject_task)
  4144. reject_block = [b for b in detail["blocks"] if b["block_type"] == "plan"][0]
  4145. check(reject_block.get("answer")
  4146. == "Plan feedback received:\nAdd a test section",
  4147. "reject stores the feedback in the stable wording (%s)"
  4148. % reject_block.get("answer"))
  4149. check(wait_for(lambda: any(
  4150. int(p.get("task", {}).get("id") or 0) == reject_task
  4151. and p.get("session_id") != reject_parked_session
  4152. and p.get("resume_notice") == "Plan feedback received:\nAdd a test section"
  4153. for p in planner.start_payloads[1:]) or False,
  4154. timeout=30, what="reject resume allocation") is not None,
  4155. "reject resumes the parked daemon with the feedback notice")
  4156. planner.push_results("done", summary="plan revised and delivered")
  4157. check(wait_task_done(auth, reject_task, timeout=30) is not None,
  4158. "the revised plan's task completes")
  4159. planner.set_state("idle")
  4160. # (c) The per-task option answers the plan block instantly
  4161. # web-side — the agent never sees a human gate, and the resume
  4162. # still rides the allocation with the approval notice.
  4163. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  4164. "project_id": "1", "title": "Auto-approved plan task", "priority": "2",
  4165. "plan_auto_approve": "on",
  4166. "capability_id": [str(answerflow_cap)],
  4167. })
  4168. auto_plan = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  4169. if t["title"] == "Auto-approved plan task"][0])
  4170. check(task_state(auth, auto_plan).get("plan_auto_approve") is True,
  4171. "the create form stores the plan-auto-approve option")
  4172. enqueue_task(auth, auto_plan)
  4173. ask_status, _ = planner.ask_now()
  4174. check(ask_status == 200, "the planner claims the auto-approve task")
  4175. auto_parked_session = planner.session_id
  4176. plan_block = park_on_plan(auto_plan, "# Auto plan\n\nStraightforward.")
  4177. detail = task_state(auth, auto_plan)
  4178. auto_block = [b for b in detail["blocks"] if b["block_type"] == "plan"][0]
  4179. check(auto_block.get("answered") is True
  4180. and auto_block.get("answer") == "Plan approved. Proceed with implementation."
  4181. and auto_block.get("answered_by") == "auto",
  4182. "the plan block is answered instantly web-side (answered-by auto)")
  4183. check(detail.get("plan", "").startswith("# Auto plan"),
  4184. "the auto-approval applies the plan markdown")
  4185. check(wait_for(lambda: any(
  4186. int(p.get("task", {}).get("id") or 0) == auto_plan
  4187. and p.get("session_id") != auto_parked_session
  4188. and p.get("resume_notice") == "Plan approved. Proceed with implementation."
  4189. for p in planner.start_payloads[1:]) or False,
  4190. timeout=30, what="auto-approve resume allocation") is not None,
  4191. "the auto-approved plan resumes through the allocation with the notice")
  4192. planner.push_results("done", summary="auto-approved plan implemented")
  4193. check(wait_task_done(auth, auto_plan, timeout=30) is not None,
  4194. "the auto-approved plan's task completes")
  4195. retire_daemon(planner, "plan-planner")
  4196. print("== phase 2: offline requeue keeps parked-with-answered tasks ==")
  4197. # A daemon dying while its parked task carries an answered
  4198. # verdict: going offline must not requeue the task or close its
  4199. # session — the resume machinery (delivery once the daemon
  4200. # returns, or the affinity allocation on its next ask) needs
  4201. # both. The question park leaves the row idle; the review park
  4202. # keeps it busy@task (`saddle review` blocks the round without
  4203. # a parking state push), exercising the unreachable path's
  4204. # requeue decision both ways.
  4205. stranded = answerflow_daemon()
  4206. register_answerflow(stranded, "resume-stranded")
  4207. stranded_task = create_answerflow_task("Stranded answered task")
  4208. ask_status, _ = stranded.ask_now()
  4209. check(ask_status == 200, "the stranded daemon claims its task")
  4210. stranded.state = "blocked"
  4211. stranded.push_results("blocked", block={
  4212. "question": "Hold or fold?", "type": "question", "options": ["Hold", "Fold"]})
  4213. check(wait_for(lambda: task_state(auth, stranded_task).get("status") == "blocked"
  4214. or False, timeout=30, what="stranded task parked") is not None,
  4215. "the stranded task parks as blocked")
  4216. detail = task_state(auth, stranded_task)
  4217. hold_block = [b for b in detail["blocks"] if b["question"] == "Hold or fold?"][0]
  4218. act(auth, hold_block["answer_action"], {"option": "Hold"})
  4219. stranded.kill() # the daemon dies before the verdict is delivered
  4220. act(auth, saddle_card(auth, "resume-stranded")["refresh"], {})
  4221. check(wait_for(lambda: saddle_card(auth, "resume-stranded")["status"] == "offline"
  4222. or False, timeout=30, what="stranded saddle offline") is not None,
  4223. "the dead daemon's saddle row goes offline")
  4224. # Converted sleep(7): offline ticks run while the task MUST stay
  4225. # parked — a requeue during the window fails loudly.
  4226. check(quiet_window(quiet_seconds(),
  4227. lambda: task_state(auth, stranded_task)["status"] == "blocked",
  4228. "stranded task not requeued"),
  4229. "the parked-with-answered task is not requeued by the offline pass")
  4230. conn = db()
  4231. open_session = conn.execute(
  4232. "select count(*) from sessions where task_id = ? and ended_at is null",
  4233. (stranded_task,)).fetchone()[0]
  4234. conn.close()
  4235. check(open_session == 1,
  4236. "the parked-with-answered task keeps its open session for the resume")
  4237. reviewer = answerflow_daemon(ask_on_terminal=False)
  4238. register_answerflow(reviewer, "resume-reviewer")
  4239. review_task = create_answerflow_task("Review park task")
  4240. ask_status, _ = reviewer.ask_now()
  4241. check(ask_status == 200, "the reviewer claims its task")
  4242. # Post Task A the review park frees the daemon: the blocked push
  4243. # carries the review block and parks the row idle.
  4244. reviewer.state = "blocked"
  4245. check(reviewer.push_results("blocked", block={
  4246. "question": "Review requested", "type": "review",
  4247. "body": "Ready for review: wired and tested."}) == 200,
  4248. "the review park push carries the review block")
  4249. check(wait_for(lambda: task_state(auth, review_task).get("status") == "review_requested"
  4250. or False, timeout=30, what="review park raised") is not None,
  4251. "the review block parks the task as review_requested (row freed)")
  4252. # The verdict lands (nothing delivers it anymore); THEN the daemon
  4253. # dies and the manual refresh offlines it — the parked-with-answered
  4254. # task must keep its session for the eventual resume.
  4255. detail = task_state(auth, review_task)
  4256. verdict_block = [b for b in detail["blocks"]
  4257. if b["block_type"] == "review"][0]
  4258. act(auth, verdict_block["answer_action"],
  4259. {"answer": "Review approved. Continue with the task."})
  4260. reviewer.kill()
  4261. act(auth, saddle_card(auth, "resume-reviewer")["refresh"], {})
  4262. check(wait_for(lambda: saddle_card(auth, "resume-reviewer")["status"] == "offline"
  4263. or False, timeout=30, what="reviewer offline") is not None,
  4264. "the dead reviewer goes offline on the failed probe")
  4265. # Converted sleep(7): offline ticks run while the task MUST stay
  4266. # review-parked — a requeue during the window fails loudly.
  4267. check(quiet_window(quiet_seconds(),
  4268. lambda: task_state(auth, review_task)["status"] == "review_requested",
  4269. "review-parked task not requeued"),
  4270. "the review-parked task with an answered verdict is never requeued")
  4271. conn = db()
  4272. open_session = conn.execute(
  4273. "select count(*) from sessions where task_id = ? and ended_at is null",
  4274. (review_task,)).fetchone()[0]
  4275. conn.close()
  4276. check(open_session == 1,
  4277. "the review-parked task keeps its open session for the resume")
  4278. print("== phase 2: staleness — one probe decides ==")
  4279. # A silent-but-alive runner (long think, nothing to flush): after
  4280. # SADDLE_STALE_S (3s here) of silence it owes ONE exceptional
  4281. # GET /status; the answer keeps it running and the silence
  4282. # episode gets no second probe.
  4283. task_id_tmp = create_queued(auth, {
  4284. "project_id": "1",
  4285. "title": "Silent thinker",
  4286. "priority": "2",
  4287. })
  4288. thinker = task_id_tmp
  4289. check(wait_for(lambda: task_state(auth, thinker).get("status") == "working"
  4290. or False, timeout=60, what="thinker working") is not None,
  4291. "silent thinker reaches working")
  4292. fake.silent = True # the daemon goes quiet mid-round (alive)
  4293. probes_at_start = fake.status_hits
  4294. check(wait_for(lambda: fake.status_hits > probes_at_start or False,
  4295. timeout=30, what="exceptional probe arrives") is not None,
  4296. "a silent runner gets its exceptional GET /status probe")
  4297. # Converted sleep(8)+sleep(6): a quiet window across several
  4298. # ticks in which the probe count MUST NOT move — a second probe
  4299. # (the poll drumbeat this guards against) fails loudly.
  4300. second_window = fake.status_hits
  4301. check(quiet_window(quiet_seconds(),
  4302. lambda: fake.status_hits == second_window,
  4303. "no second staleness probe"),
  4304. "the silence episode is probed exactly once (no poll drumbeat)")
  4305. check(task_state(auth, thinker).get("status") == "working"
  4306. and saddle_card(auth, "fake-saddle")["status"] == "busy",
  4307. "the answered probe keeps the task running (no requeue)")
  4308. # Output flows again: the episode ends, the clock resets.
  4309. fake.silent = False
  4310. fake.push_chunks(["resumed output\r\n"])
  4311. check(wait_task_done(auth, thinker, timeout=60) is not None,
  4312. "the silent thinker completes once output flows again")
  4313. # The daemon vanishing mid-task: silence for the window, the
  4314. # probe fails, the task requeues with a bot comment and the
  4315. # saddle goes offline.
  4316. task_id_tmp = create_queued(auth, {
  4317. "project_id": "1",
  4318. "title": "Vanish victim",
  4319. "priority": "2",
  4320. })
  4321. vanish = task_id_tmp
  4322. check(wait_for(lambda: task_state(auth, vanish).get("status") == "working"
  4323. or False, timeout=60, what="vanish victim working") is not None,
  4324. "vanish victim reaches working")
  4325. fake.kill()
  4326. check(wait_for(lambda: saddle_card(auth, "fake-saddle")["status"] == "offline" or False,
  4327. timeout=60, what="daemon death -> offline") is not None,
  4328. "vanishing daemon drives the saddle offline (failed staleness probe)")
  4329. check(wait_for(lambda: any("unreachable" in c["body"]
  4330. for c in task_state(auth, vanish).get("comments", []))
  4331. or False, timeout=30, what="unreachable comment") is not None,
  4332. "the affected task gets an unreachable bot comment")
  4333. check(task_state(auth, vanish)["status"] == "queued",
  4334. "the task requeues after the daemon death")
  4335. card = saddle_card(auth, "fake-saddle")
  4336. check(card.get("health") == "offline",
  4337. "health badge reports offline")
  4338. # The unreachable row flag marks the detection window: a task the
  4339. # web still sees working while its saddle is offline/fenced, before
  4340. # the requeue pass moves it. Controlled window (the real
  4341. # transitions above are atomic with their requeues).
  4342. conn = db()
  4343. conn.execute("insert into tasks (project_id, title, status, priority,"
  4344. " created_at) values (1, 'Unreachable row', 'working', 5,"
  4345. " strftime('%s','now'))")
  4346. unreachable_row = conn.execute("select max(id) from tasks").fetchone()[0]
  4347. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  4348. " values (?, ?, strftime('%s','now'))", (unreachable_row, fake_id))
  4349. conn.commit()
  4350. conn.close()
  4351. row = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  4352. if int(t["id"]) == unreachable_row][0]
  4353. check(row.get("unreachable") is True,
  4354. "queue row shows the unreachable state in the window")
  4355. print("== phase 1: startup migrates the per-saddle token ==")
  4356. # Saddles registered before the flip hold legacy api_tokens rows.
  4357. # On startup every saddle row gets its canonical token: the
  4358. # legacy value it already holds (daemons keep their credential)
  4359. # or a freshly minted one. Fixture: this shard's own racer.
  4360. fake2 = FakeSaddle()
  4361. fake2.ask_on_register = False
  4362. fake2.serve(port=free_port())
  4363. saddles_state = slot_public(auth, "/saddles", "saddles")
  4364. act(auth, saddles_state["register"], {"name": "racer-two",
  4365. "host": "127.0.0.1:%d" % fake2.port})
  4366. racer_two_id = int([s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  4367. if s["name"] == "racer-two"][0]["id"])
  4368. conn = db()
  4369. conn.execute("insert into api_tokens (saddle_id, token, created_at)"
  4370. " values (?, 'saddle-legacy-copied-token', strftime('%s','now'))",
  4371. (racer_two_id,))
  4372. conn.execute("update saddles set token = null where name = 'racer-two'")
  4373. conn.execute("insert into saddles (name, host, status, current_task_id)"
  4374. " values ('tokenless-orphan', '127.0.0.1:1', 'offline', 0)")
  4375. conn.commit()
  4376. conn.close()
  4377. app.terminate()
  4378. app.wait(timeout=10)
  4379. app = start_app()
  4380. auth = login("admin", "adminpass123")
  4381. ctx.app = app # teardown tracks the restarted process
  4382. ctx.auth = auth
  4383. def saddle_row(name):
  4384. conn = db()
  4385. try:
  4386. return conn.execute("select id, token from saddles where name = ?",
  4387. (name,)).fetchone()
  4388. finally:
  4389. conn.close()
  4390. check(wait_for(lambda: saddle_row("tokenless-orphan") is not None
  4391. and (saddle_row("tokenless-orphan")[1] or "").startswith("saddle-")
  4392. or False, timeout=20, what="tokenless row minted") is not None,
  4393. "startup mints a token for a saddle row without one")
  4394. check(wait_for(lambda: saddle_row("racer-two") is not None
  4395. and saddle_row("racer-two")[1] == "saddle-legacy-copied-token"
  4396. or False, timeout=20, what="legacy token copied") is not None,
  4397. "startup copies the legacy api_tokens value onto the row")
  4398. fake2.token = "saddle-legacy-copied-token" # the daemon asks with it
  4399. status, _ = saddle_api("POST", "/api/saddle/next-task", "saddle-legacy-copied-token")
  4400. check(status in (200, 204),
  4401. "the legacy token still authenticates (copied, not replaced)")
  4402. conn = db()
  4403. conn.execute("delete from saddles where name = 'tokenless-orphan'")
  4404. conn.commit()
  4405. conn.close()
  4406. # The restart wiped the in-memory ask bookkeeping; the fleet's
  4407. # daemons re-establish sleeping with a fresh ask each (the real
  4408. # daemons' idle-awake ask / 15-minute safety self-ask).
  4409. for daemon in (fake2,):
  4410. daemon.ask_now()
  4411. check(True, "fleet re-asks after the restart (sleeping re-established)")
  4412. # =============================================================================
  4413. # Bundle: push — plan 11 task E: real Web Push over VAPID. VAPID
  4414. # persistence, the service worker, subscription CRUD + validation,
  4415. # notify-me prefs, the block/lifecycle dispatch matrix (global vs
  4416. # opted-in), 410 pruning, retry-next-tick delivery and protocol-level
  4417. # payload verification (RFC 8291 decryption + ES256 JWT) against the
  4418. # fake push service.
  4419. # =============================================================================
  4420. def bundle_push(ctx):
  4421. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  4422. def subscription_count(endpoint):
  4423. conn = db()
  4424. try:
  4425. return conn.execute(
  4426. "select count(*) from push_subscriptions where endpoint = ?",
  4427. (endpoint,)).fetchone()[0]
  4428. finally:
  4429. conn.close()
  4430. print("== task E: service worker asset ==")
  4431. status, sw = http("GET", "/sw.js", timeout=15)
  4432. check(status == 200 and "addEventListener('push'" in sw
  4433. and "notificationclick" in sw and "clients.openWindow" in sw,
  4434. "the service worker serves at the origin root with push + click routing")
  4435. print("== task E: VAPID keypair generation + persistence ==")
  4436. config = json.load(open(os.path.join(WORK, "web-config.json")))
  4437. vapid = config.get("vapid") or {}
  4438. check(bool(vapid.get("public_key")) and bool(vapid.get("private_key")),
  4439. "the VAPID keypair persists into web-config.json's vapid section")
  4440. vapid_private_key = ec.derive_private_key(
  4441. int.from_bytes(b64url_dec(vapid["private_key"]), "big"), ec.SECP256R1())
  4442. derived_public = vapid_private_key.public_key().public_bytes(
  4443. Encoding.X962, PublicFormat.UncompressedPoint)
  4444. check(b64url_dec(vapid["public_key"]) == derived_public,
  4445. "the persisted public key matches the private scalar (a valid P-256 pair)")
  4446. dash = slot_public(auth, "/", "dashboard")
  4447. check(dash.get("push_vapid_key") == vapid["public_key"],
  4448. "the dashboard exposes the same VAPID key to subscribers")
  4449. check(dash.get("push_subscribed") is False and bool(dash.get("push_subscribe")),
  4450. "the bell starts unsubscribed and carries the subscribe action")
  4451. print("== task E: subscription CRUD + auth validation ==")
  4452. browser = ec.generate_private_key(ec.SECP256R1())
  4453. browser_public = browser.public_key().public_bytes(
  4454. Encoding.X962, PublicFormat.UncompressedPoint)
  4455. browser_auth = os.urandom(16)
  4456. subscribe_fields = {"p256dh": b64url(browser_public), "auth": b64url(browser_auth)}
  4457. valid_endpoint = "http://127.0.0.1:1/x"
  4458. check("absolute http(s)" in (action_notify(act(auth, dash["push_subscribe"],
  4459. {"endpoint": "ftp://example.com/push", **subscribe_fields})) or ""),
  4460. "a non-http endpoint is rejected")
  4461. check("p256dh" in (action_notify(act(auth, dash["push_subscribe"],
  4462. {"endpoint": valid_endpoint, "p256dh": b64url(os.urandom(64)),
  4463. "auth": subscribe_fields["auth"]})) or ""),
  4464. "a malformed p256dh point is rejected")
  4465. check("auth" in (action_notify(act(auth, dash["push_subscribe"],
  4466. {"endpoint": valid_endpoint, "p256dh": subscribe_fields["p256dh"],
  4467. "auth": b64url(os.urandom(15))})) or ""),
  4468. "a malformed auth secret is rejected")
  4469. push = FakePushService()
  4470. push.serve()
  4471. endpoint_main = push.url("/main")
  4472. act(auth, dash["push_subscribe"],
  4473. {"endpoint": endpoint_main, **subscribe_fields})
  4474. conn = db()
  4475. rows = conn.execute("select user_id, p256dh, auth from push_subscriptions"
  4476. " where endpoint = ?", (endpoint_main,)).fetchall()
  4477. admin_id = conn.execute("select id from spry_users where username = 'admin'"
  4478. ).fetchone()[0]
  4479. conn.close()
  4480. check(len(rows) == 1 and rows[0][0] == admin_id,
  4481. "a valid subscription stores against the signed-in user")
  4482. # Re-subscribing the same endpoint replaces (never duplicates).
  4483. rotated_auth = os.urandom(16)
  4484. act(auth, dash["push_subscribe"],
  4485. {"endpoint": endpoint_main, "p256dh": subscribe_fields["p256dh"],
  4486. "auth": b64url(rotated_auth)})
  4487. conn = db()
  4488. rows = conn.execute("select auth from push_subscriptions where endpoint = ?",
  4489. (endpoint_main,)).fetchall()
  4490. conn.close()
  4491. check(len(rows) == 1 and rows[0][0] == b64url(rotated_auth),
  4492. "re-subscribing an endpoint replaces its keys in place")
  4493. print("== task E: notify-me pref persistence ==")
  4494. act(auth, slot_public(auth, "/tasks", "tasks")["create"],
  4495. {"project_id": "1", "title": "Push pref task", "priority": "5"})
  4496. pref_task = task_id_of("Push pref task")
  4497. act(auth, task_state(auth, pref_task)["toggle_notify"], {})
  4498. conn = db()
  4499. check(conn.execute("select count(*) from task_notify_prefs where task_id = ?"
  4500. " and user_id = ?", (pref_task, admin_id)).fetchone()[0] == 1,
  4501. "Notify me persists a task_notify_prefs row for the user")
  4502. conn.close()
  4503. check(task_state(auth, pref_task).get("notify_me") is True,
  4504. "the task slot reflects notify_me after the toggle")
  4505. act(auth, task_state(auth, pref_task)["toggle_notify"], {})
  4506. conn = db()
  4507. check(conn.execute("select count(*) from task_notify_prefs where task_id = ?",
  4508. (pref_task,)).fetchone()[0] == 0,
  4509. "Don't notify me removes the pref row")
  4510. conn.close()
  4511. check(task_state(auth, pref_task).get("notify_me") is False,
  4512. "the task slot reflects the opt-out")
  4513. print("== task E: dispatch matrix — global blocks vs opted-in lifecycle ==")
  4514. # Two "browsers" for admin (/main, /gone) and one for another user
  4515. # (id 999, rows seeded directly — push targeting is per user).
  4516. endpoint_gone = push.url("/gone")
  4517. push.status_for["/gone"] = 410
  4518. act(auth, dash["push_subscribe"],
  4519. {"endpoint": endpoint_gone, **subscribe_fields})
  4520. other_browser = ec.generate_private_key(ec.SECP256R1())
  4521. other_public = other_browser.public_key().public_bytes(
  4522. Encoding.X962, PublicFormat.UncompressedPoint)
  4523. other_auth = os.urandom(16)
  4524. endpoint_other = push.url("/other")
  4525. conn = db()
  4526. conn.execute("insert into push_subscriptions (user_id, endpoint, p256dh, auth,"
  4527. " created_at) values (999, ?, ?, ?, strftime('%s','now'))",
  4528. (endpoint_other, b64url(other_public), b64url(other_auth)))
  4529. conn.commit()
  4530. conn.close()
  4531. # Task 1: admin opted in via the UI toggle; the other user is not.
  4532. # Baselines are captured BEFORE the enqueue: the first-start push
  4533. # fires at the allocation, the instant the dispatch lands.
  4534. act(auth, slot_public(auth, "/tasks", "tasks")["create"],
  4535. {"project_id": "1", "title": "Push lifecycle task", "priority": "5"})
  4536. task1 = task_id_of("Push lifecycle task")
  4537. act(auth, task_state(auth, task1)["toggle_notify"], {})
  4538. main_before = len(push.hits("/main"))
  4539. other_before = len(push.hits("/other"))
  4540. gone_before = len(push.hits("/gone"))
  4541. enqueue_task(auth, task1)
  4542. check(wait_for(lambda: fake.running_task == task1 or False, timeout=30,
  4543. what="task 1 assignment") is not None,
  4544. "task 1 dispatches to the fake daemon")
  4545. task1_session = fake.session_id
  4546. check(wait_for(lambda: len(push.hits("/main")) > main_before, timeout=15,
  4547. what="first-start push to the opted-in admin") is not None,
  4548. "first start notifies the opted-in user's subscription")
  4549. check(wait_for(lambda: len(push.hits("/gone")) > gone_before, timeout=15,
  4550. what="first-start push to the gone endpoint") is not None,
  4551. "first start reaches every subscription of the opted-in user")
  4552. check(quiet_window(quiet_seconds(),
  4553. lambda: len(push.hits("/other")) == other_before,
  4554. "no first-start push to the not-opted-in user"),
  4555. "first start never reaches users without the notify pref")
  4556. check(wait_task_done(auth, task1, timeout=60) is not None,
  4557. "task 1 completes")
  4558. _, done_payload = decrypt_webpush(push.hits("/main")[-1], browser,
  4559. browser_public, rotated_auth)
  4560. check(done_payload["title"] == "Push lifecycle task — completed"
  4561. and done_payload["url"] == "/tasks/%d" % task1,
  4562. "completion notifies the opted-in user (decrypted payload)")
  4563. check(len(push.hits("/other")) == other_before,
  4564. "completion stays with the opted-in user only")
  4565. check(wait_for(lambda: subscription_count(endpoint_gone) == 0, timeout=15,
  4566. what="410 endpoint pruned") is not None,
  4567. "a 410 Gone endpoint is pruned from push_subscriptions")
  4568. # Block events go to EVERY push-subscribed user globally.
  4569. main_before = len(push.hits("/main"))
  4570. other_before = len(push.hits("/other"))
  4571. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  4572. {"type": "block", "session_id": task1_session, "data": {
  4573. "question": "Global question for everyone?", "type": "question"}})
  4574. check(wait_for(lambda: len(push.hits("/main")) > main_before
  4575. and len(push.hits("/other")) > other_before, timeout=15,
  4576. what="global block push") is not None,
  4577. "a raised block notifies every push-subscribed user globally")
  4578. block_hit = push.hits("/other")[-1]
  4579. _, payload = decrypt_webpush(block_hit, other_browser, other_public, other_auth)
  4580. check(payload["title"] == "Push lifecycle task — question raised"
  4581. and payload["body"] == "Global question for everyone?"
  4582. and payload["url"] == "/tasks/%d" % task1,
  4583. "the block payload carries task title + kind, question and click URL (%s)"
  4584. % payload)
  4585. ciphertext_len = len(block_hit["body"]) - (21 + 65)
  4586. envelope = block_hit["body"]
  4587. record_size = int.from_bytes(envelope[16:20], "big")
  4588. # rs = plaintext + delimiter (1) + tag (16) == ciphertext-with-tag len.
  4589. check(record_size == ciphertext_len,
  4590. "the RFC 8291 envelope record size covers payload + delimiter + tag")
  4591. claims = verify_vapid(block_hit, vapid["public_key"], vapid["private_key"])
  4592. check(claims.get("aud") == "http://127.0.0.1:%d" % push.port
  4593. and claims.get("sub", "").startswith("mailto:")
  4594. and claims.get("exp", 0) > time.time(),
  4595. "the VAPID JWT verifies: aud = the endpoint origin, mailto sub, future exp")
  4596. check(block_hit["headers"].get("ttl") == "60"
  4597. and block_hit["headers"].get("content-type") == "application/octet-stream",
  4598. "the push POST carries TTL and the encrypted content type")
  4599. # Task 2: only the OTHER user opted in — start + failure reach them
  4600. # alone. Baseline again before the enqueue (first start fires at
  4601. # the allocation).
  4602. fake.fail_next = True
  4603. act(auth, slot_public(auth, "/tasks", "tasks")["create"],
  4604. {"project_id": "1", "title": "Push failure task", "priority": "5"})
  4605. task2 = task_id_of("Push failure task")
  4606. conn = db()
  4607. conn.execute("insert into task_notify_prefs (task_id, user_id) values (?, 999)",
  4608. (task2,))
  4609. conn.commit()
  4610. conn.close()
  4611. main_before2 = len(push.hits("/main"))
  4612. other_before2 = len(push.hits("/other"))
  4613. enqueue_task(auth, task2)
  4614. check(wait_for(lambda: fake.running_task == task2 or False, timeout=30,
  4615. what="task 2 assignment") is not None,
  4616. "task 2 dispatches (scripted to fail)")
  4617. check(wait_for(lambda: task_state(auth, task2)["status"] == "failed"
  4618. or False, timeout=30, what="task 2 failed") is not None,
  4619. "task 2 fails")
  4620. check(wait_for(lambda: len(push.hits("/other")) >= other_before2 + 2,
  4621. timeout=15, what="opted-in user's started+failed pushes"),
  4622. "the opted-in user gets first start AND failure for the task")
  4623. started_hit = [hit for hit in push.hits("/other")[other_before2:]][0]
  4624. _, started_payload = decrypt_webpush(started_hit, other_browser,
  4625. other_public, other_auth)
  4626. check(started_payload["title"] == "Push failure task — started",
  4627. "the first-start payload names task + kind (%s)" % started_payload)
  4628. _, failure_payload = decrypt_webpush(push.hits("/other")[-1], other_browser,
  4629. other_public, other_auth)
  4630. check(failure_payload["title"] == "Push failure task — failed"
  4631. and failure_payload["url"] == "/tasks/%d" % task2,
  4632. "the failure payload names the task and links its page")
  4633. check(len(push.hits("/main")) == main_before2,
  4634. "an agent failure never reaches users without the pref")
  4635. print("== task E: retry next tick (best-effort delivery) ==")
  4636. endpoint_flaky = push.url("/flaky")
  4637. act(auth, dash["push_subscribe"],
  4638. {"endpoint": endpoint_flaky, **subscribe_fields})
  4639. push.fail_next["/flaky"] = 1
  4640. flaky_before = len(push.hits("/flaky"))
  4641. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  4642. {"type": "block", "session_id": task1_session, "data": {
  4643. "question": "Flaky delivery question?", "type": "question"}})
  4644. check(wait_for(lambda: len(push.hits("/flaky")) >= flaky_before + 2,
  4645. timeout=20, what="flaky endpoint retried") is not None,
  4646. "a failed send retries on the next dispatcher tick and lands")
  4647. print("== task E: waiting-on-drafts notifies like a block event ==")
  4648. fake.suspend_on_title = "Push parked task"
  4649. act(auth, slot_public(auth, "/tasks", "tasks")["create"],
  4650. {"project_id": "1", "title": "Push parked task", "priority": "5"})
  4651. task3 = task_id_of("Push parked task")
  4652. enqueue_task(auth, task3)
  4653. check(wait_for(lambda: fake.running_task == task3 or False, timeout=30,
  4654. what="task 3 assignment") is not None,
  4655. "task 3 dispatches (scripted to suspend)")
  4656. conn = db()
  4657. conn.execute("insert into tasks (project_id, title, status, priority,"
  4658. " parent_task_id, created_at)"
  4659. " values (1, 'Push draft subtask', 'draft', 5, ?,"
  4660. " strftime('%s','now'))", (task3,))
  4661. conn.commit()
  4662. conn.close()
  4663. main_before = len(push.hits("/main"))
  4664. other_before = len(push.hits("/other"))
  4665. check(wait_for(lambda: task_state(auth, task3)["status"] == "suspended"
  4666. or False, timeout=30, what="task 3 suspended") is not None,
  4667. "task 3 parks (suspended)")
  4668. check(wait_for(lambda: len(push.hits("/main")) > main_before
  4669. and len(push.hits("/other")) > other_before, timeout=15,
  4670. what="waiting-on-drafts global push") is not None,
  4671. "the waiting-on-drafts condition notifies globally like a block event")
  4672. _, waiting_payload = decrypt_webpush(push.hits("/main")[-1], browser,
  4673. browser_public, rotated_auth)
  4674. check(waiting_payload["title"]
  4675. == "Push parked task — waiting on subtasks to be enqueued"
  4676. and "still in draft" in waiting_payload["body"]
  4677. and waiting_payload["url"] == "/tasks/%d" % task3,
  4678. "the waiting-on-drafts payload carries the clear wording + link (%s)"
  4679. % waiting_payload)
  4680. # A subtask created under the ALREADY-parked parent re-arms the push.
  4681. main_before = len(push.hits("/main"))
  4682. saddle_api("POST", "/api/saddle/subtask", fake_token,
  4683. {"parent_task_id": task3, "title": "Parked-late child"})
  4684. check(wait_for(lambda: len(push.hits("/main")) > main_before, timeout=15,
  4685. what="waiting-on-drafts re-armed push") is not None,
  4686. "a draft subtask created under a parked parent re-arms the push")
  4687. check(task_state(auth, task_id_of("Parked-late child"))["status"] == "draft",
  4688. "the parked parent's child lands in draft (no auto-enqueue)")
  4689. print("== task E: unsubscribe removes the subscription ==")
  4690. dash = slot_public(auth, "/", "dashboard")
  4691. check(dash.get("push_subscribed") is True,
  4692. "the bell shows subscribed while endpoints exist")
  4693. act(auth, dash["push_unsubscribe"], {"endpoint": endpoint_main})
  4694. check(subscription_count(endpoint_main) == 0,
  4695. "unsubscribing removes the endpoint's row")
  4696. BUNDLES_ORDER = ("pages", "dispatch", "subtasks", "lineage",
  4697. "allocation", "lifecycle", "answers", "push")
  4698. def run_bundle(name):
  4699. """Runs one shard to completion; returns a failure count."""
  4700. started = time.monotonic()
  4701. ctx = start_bundle(name)
  4702. runner = globals()["bundle_" + name]
  4703. try:
  4704. runner(ctx)
  4705. finally:
  4706. # Exit-code diagnostic: distinguishes a crashed app (negative
  4707. # returncode, e.g. -11 SIGSEGV) from a clean shutdown. When
  4708. # tearing down mid-exception, also capture the still-alive
  4709. # app's stacks before killing it (a wedged-but-alive server
  4710. # otherwise leaves no evidence).
  4711. print("(app returncode at teardown: %s)" % ctx.app.poll())
  4712. if ctx.app.poll() is None and sys.exc_info()[0] is not None:
  4713. dump_app_stacks(ctx.app.pid)
  4714. ctx.fake.kill()
  4715. try:
  4716. ctx.app.terminate()
  4717. try:
  4718. ctx.app.wait(timeout=5)
  4719. except subprocess.TimeoutExpired:
  4720. ctx.app.kill()
  4721. check(ctx.app.poll() in (0, -15),
  4722. "bundle %s app shut down cleanly (exit %s)"
  4723. % (name, ctx.app.poll()))
  4724. except OSError:
  4725. pass
  4726. print("%d passed, %d failed" % (passes, failures))
  4727. print("[bundle %s] %.1fs" % (name, time.monotonic() - started))
  4728. return failures
  4729. def run_all_parallel():
  4730. """The default meson entry: one subprocess per bundle (each with its
  4731. own app, daemons and database), verdicts aggregated into one exit
  4732. code and one combined check-count summary."""
  4733. base = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
  4734. log_dir = os.path.join(os.path.dirname(base) or "/tmp/kilo",
  4735. "webstack-shards")
  4736. os.makedirs(log_dir, exist_ok=True)
  4737. procs = {}
  4738. started = time.monotonic()
  4739. for name in BUNDLES_ORDER:
  4740. out_path = os.path.join(log_dir, "bundle-%s.log" % name)
  4741. handle = open(out_path, "w")
  4742. procs[name] = (subprocess.Popen(
  4743. [sys.executable, "-u", os.path.abspath(__file__), APP,
  4744. "--bundle", name],
  4745. stdout=handle, stderr=subprocess.STDOUT), handle, out_path)
  4746. total_pass = total_fail = 0
  4747. broken = []
  4748. for name, (proc, handle, out_path) in procs.items():
  4749. try:
  4750. proc.wait(timeout=SHARD_TIMEOUT_S)
  4751. except subprocess.TimeoutExpired:
  4752. proc.kill()
  4753. proc.wait()
  4754. handle.close()
  4755. text = open(out_path).read()
  4756. match = re.search(r"(\d+) passed, (\d+) failed", text)
  4757. took = re.search(r"\[bundle %s\] ([0-9.]+)s" % name, text)
  4758. bundle_pass = int(match.group(1)) if match else 0
  4759. bundle_fail = int(match.group(2)) if match else -1
  4760. total_pass += max(bundle_pass, 0)
  4761. total_fail += max(bundle_fail, 0)
  4762. status = "ok"
  4763. if proc.returncode != 0 or bundle_fail != 0 or match is None:
  4764. status = "BROKEN (exit %s)" % proc.returncode
  4765. broken.append(name)
  4766. print("[shard %-10s] %6ss %3d passed %3d failed %s"
  4767. % (name, took.group(1) if took else "?",
  4768. bundle_pass, bundle_fail, status))
  4769. if status != "ok":
  4770. tail = "\n".join(text.rstrip().split("\n")[-60:])
  4771. print("--- web-stack/%s tail (full log: %s) ---" % (name, out_path))
  4772. print(tail)
  4773. print("--- end web-stack/%s tail ---" % name)
  4774. print("shards wall: %.1fs (logs under %s)" % (time.monotonic() - started, log_dir))
  4775. print("ALL SHARDS: %d passed, %d failed%s"
  4776. % (total_pass, total_fail,
  4777. "" if not broken else " BROKEN: " + ", ".join(broken)))
  4778. return 1 if broken or total_fail else 0
  4779. def main():
  4780. if not os.path.exists(APP):
  4781. raise SystemExit("app binary not found at %s (build first)" % APP)
  4782. argv = sys.argv[2:]
  4783. if "--bundle" in argv:
  4784. sys.exit(1 if run_bundle(argv[argv.index("--bundle") + 1]) else 0)
  4785. if os.environ.get("WEBSTACK_SEQUENTIAL"):
  4786. # Debug path: every bundle in-process, one after another.
  4787. for name in BUNDLES_ORDER:
  4788. if run_bundle(name):
  4789. return 1
  4790. return 0
  4791. return run_all_parallel()
  4792. if __name__ == "__main__":
  4793. sys.exit(main())