web_stack_test.py 390 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534
  1. #!/usr/bin/env python3
  2. """Self-contained web-stack feature tests: app + fake saddle daemon.
  3. Unlike integration_test.py (which needs a real saddle daemon and a mock
  4. LLM), this suite spawns everything it needs:
  5. - the clanker-saddle app binary (path from argv[1], or builddir default)
  6. in a scratch directory with its own web-config.json (statum keys copied
  7. from the repo, plus `saddle_callback_base_url`) and SQLite database
  8. - in-process fake saddle daemons speaking the wire contract: POST
  9. /register (plan 10: receives {saddle_token, web_base_url}, answers
  10. {ok, ssh_public_key, version}), POST /wake (records the wake; the
  11. daemon then asks POST /api/saddle/next-task), POST /task-update
  12. (cancel/comment/subtask-status), GET /status (scripted reflection —
  13. exceptional use only), and the phase-2 push channels home:
  14. POST /api/saddle/chunks (raw PTY batches) and POST /api/saddle/results
  15. (round entries + state transitions). The retired /start, /output and
  16. /log routes 404 like a post-phase-2 daemon.
  17. It then drives the app the way statum.js does (entrypoint loads + action
  18. POSTs) and asserts the Workstream B features against the wire contract:
  19. 3. plan-review blocks hide their duplicate body preview
  20. 5. rules CRUD/attach + the effective `rules` array in assignments
  21. 6. project notes, POST /api/saddle/note, `project_notes` + `callback`
  22. in assignments
  23. plus the raw-log side: pushed chunks -> session_raw_log + terminal
  24. raw view
  25. Plan 10 phase 1 (work acquisition flip):
  26. - per-saddle tokens: registration mints + /register-delivers them,
  27. the row is the single credential store (the api_tokens fallback
  28. died with arch 24), removal revokes
  29. - token auth on every /api/saddle/* endpoint
  30. - next-task allocation: concurrent asks race exactly one winner,
  31. affinity pinning for parked tasks, resume_notice on suspended resumes
  32. - dispatcher: wake -> ask -> allocate roundtrip, the lost-wake guard
  33. (no ask after a wake -> offline, re-wake the next capable), and the
  34. pre-flip /answer + /start resume paths kept as the rollback route
  35. Plan 10 phase 2 (telemetry flip):
  36. - pushed chunks: idempotent on (session_id, seq), ordered, landing in
  37. session_raw_log so the SSE terminal keeps working
  38. - pushed results: entries store like the /output path did (redelivery
  39. idempotent on the daemon entry ids), blocked raises the block row +
  40. parks + bumps the pulse (sealed refresh shows it), done/failed
  41. complete/fail the task, suspended parks for the affinity resume
  42. - answer resumes, both daemon shapes: a SLEEPING affine saddle (its
  43. post-park ask returned 204) is woken, its ask allocating the resume
  44. with the verdict riding the assignment; a daemon parked MID-ROUND
  45. (its round holds the block's answer wait open, so it can never ask)
  46. gets the verdict delivered into that wait via POST /answer — same
  47. session, no wake. A daemon dying with a parked-with-answered task
  48. never requeues it: the verdict and session survive for the resume
  49. - staleness: a silent runner gets ONE exceptional GET /status probe —
  50. alive keeps it running, dead requeues the task with an agent comment and
  51. marks the saddle offline; a recovered daemon's ask flips offline rows
  52. back to idle
  53. - the deleted layers are gone: /output, /log and /start 404 on the
  54. daemon side, and nothing on the web side calls them
  55. Plan 10 phase 3 (capability awareness + UI):
  56. - assignment capability context: {saddle_tags, fleet_tags} on every
  57. payload, GET /api/saddle/capabilities (per-saddle token) feeding
  58. the daemon's `saddle capabilities` CLI
  59. - subtask callback capabilities: explicit-only attachment validated
  60. against the operator tag set, unknown tags rejected with the valid
  61. list in the error
  62. - task page: status block pinned to the top, blocking sections
  63. newest-first (sealed GET state)
  64. - home dashboard: pinned OPEN blocks as newest-first link-through
  65. cards (no inline answer forms) above the cross-task activity
  66. timeline (last ~50 events newest-first)
  67. - lost-wake-guard regression: a saddle that asks SLOWLY after its
  68. wake (busy winding down) still claims the task — the wake is never
  69. silently dropped, no re-wake storm fires, and the never-asks guard
  70. path (offline + re-wake of the next capable) stays green
  71. Prints PASS/FAIL lines Statum-style; exits non-zero on any failure.
  72. Plan 11 task D+E (UI + notifications), the `push` shard:
  73. - task page: progress feed above the status card, status + description
  74. in a 50/50 grid, edit expander under the history expander, plan
  75. behind an expander, answered blocks behind an expander; queue screen:
  76. New Task expander above the filters; dashboard waiting-on-drafts
  77. block with links to where Enqueue lives (pages shard)
  78. - real Web Push over VAPID: keypair generated at startup into
  79. web-config.json, GET /sw.js service worker with click-through,
  80. subscribe/unsubscribe + per-task Notify me actions, block events and
  81. waiting-on-drafts pushed globally, first start/failure/completion to
  82. opted-in users, best-effort queue with dispatcher-tick retry, 410/404
  83. endpoint pruning — payloads verified protocol-level (RFC 8291
  84. decryption + ES256 JWT signature) because the harness owns the
  85. browser-side keys
  86. Heredoc completion reports (pages shard):
  87. - terminal tasks render tasks.summary (set by done/fail) as a
  88. markdown completion-report card below the status/description grid —
  89. "Completion report" when done, "Failure summary" when failed, no
  90. card when empty; the summary renders through the shipped
  91. stm-markdown renderer (headings/code, no syntax leaks, HTML
  92. escaped) and the dashboard timeline snippets its first line
  93. Speed pass (plan 11, task 0):
  94. - SHARDING: the scenario groups are partitioned into seven independent
  95. bundles (pages, dispatch, subtasks, lineage, allocation, lifecycle,
  96. answers), each running in its own process against its own app
  97. instance, fake daemons and SQLite database. The default invocation
  98. runs all bundles in parallel and aggregates the verdicts and check
  99. counts into one exit code — meson sees a single green/red test.
  100. `--bundle NAME` runs one shard for debugging; WEBSTACK_SEQUENTIAL=1
  101. runs them in-process, one after another.
  102. - Within a bundle, scenario groups SHARE the app+daemon instance and
  103. carry per-group state as database rows (task/session/block
  104. fixtures), never process restarts; the one deliberate restart (the
  105. startup token migration) lives in the answers bundle.
  106. - EVENT-DRIVEN WAITS: wait_for polls at 50ms; the fixed sleeps were
  107. converted either to observable-condition waits (daemon-side
  108. counters, e.g. failed-delivery hits) or to quiet_window() — a
  109. bounded negative-assertion poll that FAILS LOUDLY the moment the
  110. event it guards against happens (a sleep could silently pass).
  111. - ENV KNOBS: SADDLE_DISPATCH_TICK_S=1 (dispatcher cadence; default 5
  112. in production), SADDLE_HTTP_TIMEOUT_S=2, SADDLE_LOST_WAKE_S=2,
  113. SADDLE_STALE_S=3, plus the fake daemon's scripted timings — the
  114. wire-traffic shapes stay realistic, just faster.
  115. """
  116. import json
  117. import os
  118. import re
  119. import shutil
  120. import socket
  121. import sqlite3
  122. import subprocess
  123. import sys
  124. import threading
  125. import time
  126. import urllib.error
  127. import urllib.parse
  128. import urllib.request
  129. from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
  130. # Web Push protocol verification (plan 11 task E): the test harness
  131. # owns the browser-side keys, so it can do everything the push service
  132. # does — decrypt the RFC 8291 body and verify the ES256 VAPID JWT.
  133. import base64
  134. from cryptography.hazmat.primitives import hashes
  135. from cryptography.hazmat.primitives.asymmetric import ec
  136. from cryptography.hazmat.primitives.asymmetric.utils import encode_dss_signature
  137. from cryptography.hazmat.primitives.ciphers.aead import AESGCM
  138. from cryptography.hazmat.primitives.kdf.hkdf import HKDF
  139. from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
  140. REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  141. APP = os.path.abspath(sys.argv[1]) if len(sys.argv) > 1 else os.path.join(REPO, "builddir", "clanker-saddle")
  142. # --- speed-pass tuning -----------------------------------------------------
  143. # The fake daemon's scripted timings and the dispatch-tick cadence the
  144. # quiet windows are sized against. All overridable for debugging.
  145. TICK_S = float(os.environ.get("SADDLE_DISPATCH_TICK_S", "1") or 1)
  146. FAKE_WORK_S = float(os.environ.get("WEBSTACK_FAKE_WORK_S", "1.2") or 1.2)
  147. FAKE_FAIL_S = float(os.environ.get("WEBSTACK_FAKE_FAIL_S", "0.5") or 0.5)
  148. FAKE_HEARTBEAT_S = float(os.environ.get("WEBSTACK_FAKE_HEARTBEAT_S", "0.4") or 0.4)
  149. FAKE_TRIGGER_S = float(os.environ.get("WEBSTACK_FAKE_TRIGGER_S", "0.05") or 0.05)
  150. SHARD_TIMEOUT_S = float(os.environ.get("WEBSTACK_SHARD_TIMEOUT_S", "600") or 600)
  151. def quiet_seconds():
  152. """A negative-assertion window spanning several dispatch ticks (the
  153. converted fixed sleeps were 3-7s at the old 5s tick; the bound never
  154. exceeds the old timeout)."""
  155. return min(2 * TICK_S + 0.8, 7.0)
  156. WORK = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
  157. DB_PATH = os.path.join(WORK, "web.sqlite")
  158. passes = 0
  159. failures = 0
  160. def check(condition, label):
  161. global passes, failures
  162. if condition:
  163. passes += 1
  164. print("PASS %s" % label)
  165. else:
  166. failures += 1
  167. print("FAIL %s" % label)
  168. def free_port():
  169. with socket.socket() as sock:
  170. sock.bind(("127.0.0.1", 0))
  171. return sock.getsockname()[1]
  172. APP_PORT = free_port()
  173. DAEMON_PORT = free_port()
  174. WEB = "http://127.0.0.1:%d" % APP_PORT
  175. def bundle_env(name):
  176. """Point the module's runtime globals at one shard's scratch dir,
  177. database and ports (each bundle is a process with its own app,
  178. daemons and SQLite file)."""
  179. global WORK, DB_PATH, APP_PORT, DAEMON_PORT, WEB
  180. base = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
  181. WORK = "%s-%s" % (base, name)
  182. DB_PATH = os.path.join(WORK, "web.sqlite")
  183. APP_PORT = free_port()
  184. DAEMON_PORT = free_port()
  185. WEB = "http://127.0.0.1:%d" % APP_PORT
  186. def http(method, url, data=None, headers=None, timeout=30):
  187. body = None
  188. if data is not None:
  189. pairs = []
  190. for k, v in data.items():
  191. for item in (v if isinstance(v, (list, tuple)) else [v]):
  192. pairs.append((k, item))
  193. body = "&".join("%s=%s" % (k, urllib.parse.quote(str(item))) for k, item in pairs).encode()
  194. request = urllib.request.Request(WEB + url if url.startswith("/") else url,
  195. data=body, method=method)
  196. if data is not None:
  197. request.add_header("Content-Type", "application/x-www-form-urlencoded")
  198. for key, value in (headers or {}).items():
  199. request.add_header(key, value)
  200. # Retry dropped/refused connections: the freshly started app can
  201. # close a very early keep-alive connection once while warming up,
  202. # and its first seconds can wobble under load.
  203. for attempt in range(3):
  204. try:
  205. with urllib.request.urlopen(request, timeout=timeout) as response:
  206. return response.status, response.read().decode()
  207. except urllib.error.HTTPError as error:
  208. return error.code, error.read().decode()
  209. except (urllib.error.URLError, OSError):
  210. if attempt >= 2:
  211. raise
  212. time.sleep(0.75)
  213. def directives(uri, auth=None):
  214. headers = {}
  215. if auth:
  216. headers["X-Statum-Slot"] = "key=%s; as_at=%s" % (auth["key"], auth["as_at"])
  217. status, body = http("GET", "/_statum/entrypoint?uri=" + urllib.parse.quote(uri.split("?")[0]),
  218. headers=headers)
  219. if status != 200:
  220. raise SystemExit("entrypoint %s -> HTTP %d: %s" % (uri, status, body[:300]))
  221. return json.loads(body)
  222. def slot_public(auth, uri, type_name):
  223. for directive in directives(uri, auth):
  224. if directive["type"] == "set":
  225. content = directive["snapshot_frame"].get("content")
  226. if not content:
  227. continue
  228. content = json.loads(content)
  229. if content["slot"]["type"] == type_name:
  230. return content["public"]
  231. raise SystemExit("no %s slot in entrypoint state for %s" % (type_name, uri))
  232. def act(auth, action, fields):
  233. headers = {"X-Statum-Slot": "key=%s; as_at=%s" % (auth["key"], auth["as_at"])}
  234. if action.get("private"):
  235. headers["X-Statum-Private"] = action["private"]
  236. status, body = http(action.get("method", "POST"), action["uri"], fields, headers)
  237. if status != 200:
  238. raise SystemExit("action %s -> HTTP %d: %s" % (action["uri"], status, body[:300]))
  239. return json.loads(body)
  240. def login(username, password):
  241. frames = directives("/login")
  242. content = json.loads(frames[0]["snapshot_frame"]["content"])
  243. action = content["public"]["login"]
  244. status, body = http(action.get("method", "POST"), action["uri"],
  245. {"username": username, "password": password})
  246. if status != 200:
  247. raise SystemExit("login -> HTTP %d: %s" % (status, body[:300]))
  248. for directive in json.loads(body):
  249. if directive["type"] == "set":
  250. frame = json.loads(directive["snapshot_frame"]["content"])
  251. if frame["slot"]["type"] == "auth":
  252. return {"key": frame["slot"]["key"], "as_at": frame["as_at"]}
  253. raise SystemExit("login response carried no auth slot: " + body[:300])
  254. def wait_for(predicate, timeout=60, what="condition", interval=0.05):
  255. deadline = time.monotonic() + timeout
  256. last = None
  257. while time.monotonic() < deadline:
  258. last = predicate()
  259. if last is not False and last is not None:
  260. return last
  261. time.sleep(interval)
  262. print(" (timeout waiting for %s; last: %s)" % (what, last))
  263. dump_threads()
  264. return None
  265. def quiet_window(seconds, predicate, what):
  266. """A converted fixed sleep: asserts a NEGATIVE condition (nothing may
  267. happen) across a window spanning several dispatch ticks, polling at
  268. 50ms. The moment the condition breaks the window fails loudly
  269. (diagnostics + thread dump) instead of silently passing a sleep."""
  270. deadline = time.monotonic() + seconds
  271. while time.monotonic() < deadline:
  272. value = predicate()
  273. if value is False or value is None:
  274. remaining = deadline - time.monotonic()
  275. print(" (quiet window '%s' broken after %.2fs of %.2fs — the"
  276. " guarded event DID happen)" % (what, seconds - remaining, seconds))
  277. dump_threads()
  278. return False
  279. time.sleep(0.05)
  280. return True
  281. def dump_threads():
  282. """Stacks of suite threads not parked in sleep — pinpoints a wedged
  283. fake lifecycle when a wait times out."""
  284. import traceback
  285. shown = 0
  286. for thread_id, frame in sys._current_frames().items():
  287. stack = "".join(traceback.format_stack(frame)[-4:])
  288. if ("web_stack_test.py" in stack) and ("sleep" not in stack) and shown < 6:
  289. shown += 1
  290. print(" --- thread %s ---\n%s" % (thread_id, stack))
  291. def task_state(auth, task_id):
  292. return slot_public(auth, "/tasks/%d" % task_id, "task")
  293. def wait_task_done(auth, task_id, timeout=90):
  294. return wait_for(lambda: task_state(auth, task_id).get("status") == "done" or False,
  295. timeout=timeout, what="task %d -> done" % task_id)
  296. def task_id_of(title):
  297. conn = db()
  298. try:
  299. return conn.execute("select id from tasks where title = ?", (title,)).fetchone()[0]
  300. finally:
  301. conn.close()
  302. def enqueue_task(auth, task_id):
  303. """The human-only Enqueue action (plan 11): draft -> queued."""
  304. detail = task_state(auth, task_id)
  305. assert detail["status"] == "draft", "task %d not in draft (%s)" % (task_id, detail["status"])
  306. act(auth, detail["enqueue"], {})
  307. return task_state(auth, task_id)["status"]
  308. def create_queued(auth, fields):
  309. """Create via the queue form (lands in draft) and enqueue it — the
  310. standard 'this task should dispatch' fixture of the plan-11 world."""
  311. act(auth, slot_public(auth, "/tasks", "tasks")["create"], fields)
  312. task_id = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  313. if t["title"] == fields["title"]][0])
  314. enqueue_task(auth, task_id)
  315. return task_id
  316. def saddle_api(method, path, token, payload=None):
  317. """Raw call against the saddle ingest API; returns (status, json)."""
  318. request = urllib.request.Request(WEB + path,
  319. data=json.dumps(payload or {}).encode(), method=method)
  320. request.add_header("Content-Type", "application/json")
  321. if token is not None:
  322. request.add_header("Authorization", "Bearer " + token)
  323. try:
  324. with urllib.request.urlopen(request, timeout=15) as response:
  325. return response.status, json.loads(response.read().decode() or "{}")
  326. except urllib.error.HTTPError as error:
  327. body = error.read().decode()
  328. try:
  329. return error.code, json.loads(body)
  330. except ValueError:
  331. return error.code, {"raw": body}
  332. def db():
  333. return sqlite3.connect(DB_PATH)
  334. def http_get_page(auth, uri):
  335. """The compiled page HTML for a statum route (auth via the slot
  336. header), for asserting page-level embeds the slot state cannot
  337. see."""
  338. status, body = http("GET", uri, headers={
  339. "X-Statum-Slot": "key=%s; as_at=%s" % (auth["key"], auth["as_at"])})
  340. if status != 200:
  341. raise SystemExit("GET %s -> HTTP %d" % (uri, status))
  342. return body
  343. def saddle_card(auth, name):
  344. rows = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  345. if s["name"] == name]
  346. return rows[0]
  347. def queued_count():
  348. conn = db()
  349. try:
  350. return conn.execute(
  351. "select count(*) from tasks where status = 'queued'").fetchone()[0]
  352. finally:
  353. conn.close()
  354. def command_rows(session):
  355. conn = db()
  356. try:
  357. return conn.execute(
  358. "select command_text, type from session_commands"
  359. " where session_id = ? order by id", (session,)).fetchall()
  360. finally:
  361. conn.close()
  362. class FakeSaddle:
  363. """In-process saddle daemon implementing the wire contract.
  364. Phase-2 push behaviour: the daemon asks POST /api/saddle/next-task
  365. whenever idle-awake (at registration, on POST /wake, after its task
  366. reaches a terminal state, and when a /status probe observes it
  367. parked idle — throttled), accepts assignments like POST /start used
  368. to, then PUSHES its telemetry home: raw PTY chunks via POST
  369. /api/saddle/chunks and round results with state transitions via
  370. POST /api/saddle/results (working at round start, terminal states
  371. when the scripted lifecycle ends them). /output, /log and /start
  372. are gone (404) like a post-phase-2 daemon; /task-update serves the
  373. cancel/comment/subtask-status kinds; POST /answer receives a
  374. delivered verdict when the round is parked in a block's answer
  375. wait (409 when nothing waits — the real daemon's contract; the
  376. resumed round continues after the response). Every trigger can be
  377. disabled per test for determinism; ask_now() performs a
  378. synchronous manual ask.
  379. """
  380. PROMPT = "bbarrow@fedora:/workspace/task-1$ "
  381. def __init__(self):
  382. self.lock = threading.Lock()
  383. self.start_payloads = [] # captured assignment payloads (/start + next-task 200s)
  384. self.state = "idle"
  385. self.summary = ""
  386. self.state_entered = time.monotonic()
  387. self.fail_next = False # next task drives the task to failed
  388. self.start_counts = {} # task id -> assignment count
  389. self.resumed_pending = False # crash-resume flag on the next /status
  390. self.suspend_on_title = None # this task (by title) parks as suspended
  391. self.output_seq = 100 # daemon-global entry ids for pushed results
  392. self.last_activity_epoch = int(time.time())
  393. self.hang_next = {} # path suffix -> hang count (never responds)
  394. self.abort_failures = 0 # cancel task-updates failing (500) while > 0
  395. self.abort_log = [] # task ids aborted via task-update cancel
  396. self.running_task = 0 # task id the daemon believes it runs
  397. self.running_title = "" # its title (suspend targeting)
  398. self.session_id = None # the web-side run row of the running task
  399. self.hold_state = False # freeze lifecycle + heartbeat pushes
  400. self.ssh_public_key = "" # reported on /status + /register
  401. self.started_mono = time.monotonic()
  402. self.freeze_activity = False # hold last_activity_epoch still
  403. self.silent = False # suppress pushes (staleness tests)
  404. self.dead = False # kill(): no serving AND no calls home
  405. self.status_hits = 0 # GET /status arrivals (probe accounting)
  406. self.comment_log = [] # recorded /task-update comment payloads
  407. self.scripted_corpus = None # chunk batch pushed when a task starts
  408. # --- plan 10 phase 1: ask/wake/register ---
  409. self.token = None # per-saddle token from POST /register
  410. self.register_log = [] # recorded /register bodies
  411. self.wake_log = [] # recorded POST /wake arrivals
  412. self.asks = [] # [{"status": int, "payload": dict|None}]
  413. self.ask_on_register = True
  414. self.ask_on_wake = True
  415. self.ask_on_terminal = True
  416. self.ask_on_idle_status = True
  417. self.ask_delay = 0.0 # seconds; a slow daemon delays its wake-triggered ask
  418. self.serve_register = True # False: 404 /register (pre-flip daemon)
  419. self.last_ask_trigger = 0.0
  420. self.port = None
  421. self.ask_threads = [] # scheduled ask-trigger threads (settle_asks joins them)
  422. # --- answer delivery (the parked-session resume) ---
  423. self.pending_block = None # the block payload this daemon waits on (state blocked)
  424. self.answer_log = [] # recorded POST /answer bodies
  425. self.answer_push_delay = 0.2 # the resumed round's first push, after the response
  426. # --- plan 12: goal-agent scripting ---
  427. self.goal_script = [] # behaviours consumed per goal-run assignment
  428. self.goal_behaviour = None # the current round's popped behaviour
  429. self.goal_assignments = [] # captured goal-run assignment payloads
  430. def set_state(self, state, summary=None):
  431. """Advances the scripted lifecycle."""
  432. self.state = state
  433. if summary is not None:
  434. self.summary = summary
  435. self.state_entered = time.monotonic()
  436. def touch_activity(self):
  437. if not self.freeze_activity:
  438. self.last_activity_epoch = int(time.time())
  439. def queue_output(self, entry_type, command, output):
  440. """Pushes a typed entry as a round-results push (the daemon's
  441. entry log riding the results channel)."""
  442. with self.lock:
  443. entry = {
  444. "id": self.output_seq,
  445. "command": command,
  446. "output": output,
  447. "exit_code": 1,
  448. "type": entry_type,
  449. "timestamp": "2026-09-06T00:00:00Z",
  450. }
  451. self.output_seq += 1
  452. self.push_results(self.state if self.state in ("working",) else "working",
  453. entries=[entry])
  454. # --- plan 10 phase 2: the push channels ---
  455. def push_chunks(self, chunks):
  456. """Pushes raw PTY chunks home: elements are texts (assigned
  457. fresh seqs) or {seq, text} dicts (a redelivery or explicit
  458. corpus). Pushes are suppressed while silent/hold_state (the
  459. staleness tests' silent runner)."""
  460. if self.dead or self.silent or self.token is None or self.session_id is None:
  461. return
  462. batch = []
  463. with self.lock:
  464. for chunk in chunks:
  465. if isinstance(chunk, dict):
  466. batch.append({"seq": int(chunk["seq"]), "text": chunk["text"]})
  467. else:
  468. batch.append({"seq": self.next_push_seq(), "text": chunk})
  469. try:
  470. saddle_api("POST", "/api/saddle/chunks", self.token,
  471. {"session_id": self.session_id, "chunks": batch})
  472. except (OSError, ValueError):
  473. pass # app restarting / shutting down mid-test
  474. def next_push_seq(self):
  475. # The daemon-side chunk counter for ad-hoc pushes.
  476. self._push_seq = getattr(self, "_push_seq", 0) + 1
  477. return self._push_seq
  478. def push_results(self, state, entries=None, block=None, summary=None, session_id=None):
  479. """Pushes one round-results payload home (state transition plus
  480. its entries). Returns the HTTP status when it went out, None
  481. when suppressed."""
  482. if self.dead or self.silent:
  483. return None
  484. if self.token is None:
  485. return None
  486. target = session_id if session_id is not None else self.session_id
  487. if target is None:
  488. return None
  489. payload = {"session_id": target, "state": state}
  490. if entries is not None:
  491. payload["entries"] = entries
  492. if block is not None:
  493. payload["block"] = block
  494. if state == "blocked":
  495. # The daemon parks its round on this block's answer wait
  496. # (the real daemon's wait_for_answer).
  497. self.pending_block = dict(block)
  498. if summary is not None:
  499. payload["summary"] = summary
  500. try:
  501. status, _ = saddle_api("POST", "/api/saddle/results", self.token, payload)
  502. if status != 200:
  503. print(" [fake %s] push %s -> HTTP %s (task=%s)"
  504. % (self.port, state, status, self.running_task))
  505. return status
  506. except (OSError, ValueError) as error:
  507. print(" [fake %s] push %s failed: %r (task=%s)"
  508. % (self.port, state, error, self.running_task))
  509. return None
  510. # --- plan 10 phase 1: the ask path ---
  511. def ask_now(self):
  512. """Synchronously asks the web for work (a phase-1 daemon's idle
  513. ask). A 200 assignment is accepted exactly like POST /start;
  514. returns (status, payload|None)."""
  515. token = self.token
  516. if token is None:
  517. raise SystemExit("fake daemon asked before receiving its token")
  518. status, payload = saddle_api("POST", "/api/saddle/next-task", token)
  519. with self.lock:
  520. self.asks.append({"status": status, "payload": payload})
  521. if status == 200:
  522. self.accept_assignment(payload)
  523. return status, payload
  524. def _trigger_ask(self, reason):
  525. """Schedules a delayed ask (the daemon woke / went idle-awake).
  526. ask_delay models a SLOW asker — a daemon still winding down its
  527. round when the wake lands: it honours the wake, but its ask
  528. arrives late (still inside the web's lost-wake guard window)."""
  529. def run():
  530. if self.state in ("working", "failing"):
  531. return # a running daemon does not ask
  532. if self.ask_delay > 0:
  533. time.sleep(self.ask_delay)
  534. for attempt in range(3):
  535. try:
  536. status, _ = self.ask_now()
  537. if status in (200, 204):
  538. return
  539. print(" (ask %s -> HTTP %s, attempt %d)" % (reason, status, attempt + 1))
  540. except Exception as error: # app restarting / blips: retry
  541. print(" (ask %s failed: %r, attempt %d)" % (reason, error, attempt + 1))
  542. time.sleep(1.0)
  543. def starter():
  544. time.sleep(FAKE_TRIGGER_S)
  545. run()
  546. thread = threading.Thread(target=starter, daemon=True)
  547. with self.lock:
  548. self.ask_threads.append(thread)
  549. thread.start()
  550. def settle_asks(self, timeout=15):
  551. """Joins every ask-trigger thread scheduled so far — the
  552. event-driven replacement for the fixed sleeps that waited for
  553. in-flight asks to land after triggers were frozen."""
  554. with self.lock:
  555. threads = [t for t in self.ask_threads if t.is_alive()]
  556. deadline = time.monotonic() + timeout
  557. for thread in threads:
  558. thread.join(timeout=max(0.1, deadline - time.monotonic()))
  559. with self.lock:
  560. self.ask_threads = [t for t in self.ask_threads if t.is_alive()]
  561. return all(not thread.is_alive() for thread in threads)
  562. def maybe_trigger_ask(self, reason, throttle=False):
  563. """Schedules an idle-awake ask. Protocol events (wake, abort,
  564. round end, park) ask singularly and immediately; only the
  565. /status-observation path throttles (a burst of probes must not
  566. hammer the ask endpoint)."""
  567. now = time.monotonic()
  568. if throttle and now - self.last_ask_trigger < 1.0:
  569. return
  570. self.last_ask_trigger = now
  571. self._trigger_ask(reason)
  572. def accept_assignment(self, payload):
  573. """A 200 next-task assignment: adopted exactly like POST /start."""
  574. self.start_payloads.append(payload)
  575. task = payload.get("task") or {}
  576. task_id = task.get("id")
  577. self.running_task = int(task_id or 0)
  578. self.running_title = task.get("title") or ""
  579. self.session_id = payload.get("session_id")
  580. # The assignment's answers resolve any block this daemon was
  581. # parked on (resume-carries-answer).
  582. self.pending_block = None
  583. count = self.start_counts.get(task_id, 0) + 1
  584. self.start_counts[task_id] = count
  585. if count > 1:
  586. # Crash-resume contract: re-pushing a task whose
  587. # workspace still exists restores it; the first
  588. # /status after the restore reports "resumed": true.
  589. self.resumed_pending = True
  590. # Plan 12: a goal-run assignment plays its scripted behaviour
  591. # instead of the plain done push (plan submission, parking,
  592. # question, completion, failure — consumed one per round).
  593. self.goal_behaviour = None
  594. if payload.get("kind") == "goal_run":
  595. self.goal_assignments.append(payload)
  596. self.goal_behaviour = self.goal_script.pop(0) if self.goal_script else ("suspend",)
  597. self.set_state("failing" if self.fail_next else "working")
  598. self.fail_next = False
  599. # The round's opening telemetry: the flushed-so-far raw PTY
  600. # batch plus the working state push (>= 2 flushes per round
  601. # and the 10s timer are the real daemon's contract; the fake
  602. # pushes its batch up front and heartbeats while working).
  603. if self.scripted_corpus is not None:
  604. self.push_chunks([dict(chunk) for chunk in self.scripted_corpus])
  605. else:
  606. self.push_chunks([self.PROMPT, "echo hello\r\n", "hello\r\n"])
  607. self.push_results("working")
  608. class Handler(BaseHTTPRequestHandler):
  609. daemon = None # set on the class before serving
  610. protocol_version = "HTTP/1.1"
  611. def log_message(self, fmt, *args):
  612. pass
  613. def maybe_hang(self, path):
  614. """True when this request should hang forever (never respond),
  615. consuming one unit of the path's hang budget."""
  616. for suffix, count in list(self.daemon.hang_next.items()):
  617. if count > 0 and path.startswith(suffix):
  618. self.daemon.hang_next[suffix] = count - 1
  619. return True
  620. return False
  621. def _send(self, status, obj):
  622. body = json.dumps(obj).encode()
  623. self.send_response(status)
  624. self.send_header("Content-Type", "application/json")
  625. self.send_header("Content-Length", str(len(body)))
  626. self.end_headers()
  627. self.wfile.write(body)
  628. def do_POST(self):
  629. if self.maybe_hang(self.path):
  630. time.sleep(600)
  631. return
  632. self.daemon.touch_activity()
  633. length = int(self.headers.get("Content-Length") or 0)
  634. raw = self.rfile.read(length).decode() if length else "{}"
  635. if self.path == "/register":
  636. # Plan 10: the web delivers the per-saddle token + its
  637. # own base URL; the daemon persists the token for its
  638. # calls home and reports its ssh key + version.
  639. if not self.daemon.serve_register:
  640. self._send(404, {"error": "no such route"})
  641. return
  642. try:
  643. payload = json.loads(raw)
  644. except ValueError:
  645. payload = {"raw": raw}
  646. self.daemon.register_log.append(payload)
  647. if payload.get("saddle_token"):
  648. self.daemon.token = payload["saddle_token"]
  649. self._send(200, {"ok": True,
  650. "ssh_public_key": self.daemon.ssh_public_key,
  651. "version": "0.1-fake"})
  652. if self.daemon.ask_on_register:
  653. self.daemon.maybe_trigger_ask("register")
  654. return
  655. if self.path == "/wake":
  656. # Plan 10: a capable task is ready; a sleeping daemon
  657. # asks for work (immediately — a wake is a direct
  658. # command, never throttled).
  659. self.daemon.wake_log.append(time.monotonic())
  660. self._send(200, {"ok": True})
  661. if self.daemon.ask_on_wake:
  662. self.daemon.maybe_trigger_ask("wake")
  663. return
  664. if self.path == "/answer":
  665. # Plan 11 (daemon Task A): parks FREE the daemon, so no
  666. # block ever waits in-process — the verdict travels in
  667. # the resuming allocation's resume_notice. Always 409,
  668. # exactly like the real daemon's contract.
  669. try:
  670. payload = json.loads(raw)
  671. except ValueError:
  672. self._send(400, {"error": "bad json"})
  673. return
  674. self.daemon.answer_log.append({
  675. "refused": True,
  676. "answer": payload.get("answer", "")})
  677. self._send(409, {"error": "no block is waiting for an answer — parks resume via allocation"})
  678. return
  679. if self.path == "/task-update":
  680. # Plan 10: cancel (abort machinery) and comment
  681. # (injected turn) — delivered directly to a RUNNING
  682. # daemon, no auth.
  683. try:
  684. payload = json.loads(raw)
  685. except ValueError:
  686. self._send(400, {"error": "bad json"})
  687. return
  688. kind = payload.get("kind")
  689. if kind == "cancel":
  690. if self.daemon.abort_failures > 0:
  691. self.daemon.abort_failures -= 1
  692. self._send(500, {"error": "cancel failed (simulated)"})
  693. return
  694. aborted = self.daemon.running_task
  695. self.daemon.set_state("idle")
  696. self.daemon.summary = ""
  697. self.daemon.running_task = 0
  698. self.daemon.session_id = None
  699. self.daemon.abort_log.append(aborted)
  700. self._send(200, {"ok": True, "aborted_task_id": aborted})
  701. # An aborted daemon returns to its idle loop: it asks.
  702. self.daemon.maybe_trigger_ask("aborted")
  703. elif kind == "comment":
  704. self.daemon.comment_log.append(payload)
  705. self._send(200, {"ok": True})
  706. else:
  707. self._send(400, {"error": "unknown task-update kind"})
  708. return
  709. self._send(404, {"error": "no such route"})
  710. self.close_connection = True
  711. def do_GET(self):
  712. if self.maybe_hang(self.path):
  713. time.sleep(600)
  714. return
  715. if self.path != "/status":
  716. # /status itself is not daemon activity: the liveness
  717. # signal must reflect loop rounds / command completions.
  718. self.daemon.touch_activity()
  719. if self.path == "/status":
  720. self.daemon.status_hits += 1
  721. response = {"state": self.daemon.state, "summary": self.daemon.summary,
  722. "uptime_seconds": int(time.monotonic() - self.daemon.started_mono),
  723. "last_activity_epoch": self.daemon.last_activity_epoch,
  724. "ssh_public_key": self.daemon.ssh_public_key}
  725. if getattr(self.daemon, "status_extra", None):
  726. response.update(self.daemon.status_extra)
  727. if self.daemon.resumed_pending:
  728. response["resumed"] = True
  729. self.daemon.resumed_pending = False
  730. # A parked/terminal daemon is idle-awake: a phase-1
  731. # daemon asks (and sleeps on 204). Throttled so status
  732. # poll bursts do not hammer the ask endpoint.
  733. if self.daemon.ask_on_idle_status and self.daemon.state in (
  734. "idle", "done", "failed", "suspended"):
  735. self.daemon.maybe_trigger_ask("idle-status", throttle=True)
  736. self._send(200, response)
  737. else:
  738. self._send(404, {"error": "no such route"})
  739. self.close_connection = True
  740. def run_lifecycle(self):
  741. """Applies the scripted lifecycle on a timer (like a real daemon
  742. whose task ends on its own): working heartbeats chunks (the
  743. flush timer's stand-in, keeping the staleness clock fed), works
  744. for FAKE_WORK_S, then pushes done/suspended; failing pushes
  745. failed after FAKE_FAIL_S — and a task END makes the idle-awake
  746. daemon ask again."""
  747. while True:
  748. time.sleep(0.1)
  749. if self.hold_state or self.silent:
  750. # hold_state freezes the script; silent models a daemon
  751. # mid-round with nothing to flush — its round has not
  752. # ended, so no terminal push fires while quiet.
  753. continue
  754. if self.state == "failing":
  755. if time.monotonic() - self.state_entered >= FAKE_FAIL_S:
  756. self.set_state("failed", "bash -n rejected the script (simulated)")
  757. self.push_results("failed",
  758. summary="bash -n rejected the script (simulated)")
  759. if self.ask_on_terminal:
  760. self.maybe_trigger_ask("terminal")
  761. elif self.state == "working":
  762. if time.monotonic() - self.state_entered >= FAKE_WORK_S:
  763. if self.goal_behaviour is not None:
  764. # A goal-agent round: play the scripted
  765. # behaviour (the results channel carries the
  766. # goal-context verbs).
  767. behaviour, self.goal_behaviour = self.goal_behaviour, None
  768. kind = behaviour[0]
  769. if kind == "plan":
  770. self.set_state("blocked")
  771. self.push_results("blocked", block={
  772. "type": "plan",
  773. "question": "Plan submitted for review",
  774. "body": behaviour[1]})
  775. if self.ask_on_idle_status:
  776. self.maybe_trigger_ask("parked")
  777. elif kind == "question":
  778. self.set_state("blocked")
  779. self.push_results("blocked", block={
  780. "type": "question",
  781. "question": behaviour[1],
  782. "options": list(behaviour[2])
  783. if len(behaviour) > 2 else None})
  784. if self.ask_on_idle_status:
  785. self.maybe_trigger_ask("parked")
  786. elif kind == "suspend":
  787. self.set_state("suspended")
  788. self.push_results("suspended")
  789. if self.ask_on_idle_status:
  790. self.maybe_trigger_ask("parked")
  791. elif kind == "fail":
  792. self.set_state("failed", behaviour[1])
  793. self.push_results("failed", summary=behaviour[1])
  794. if self.ask_on_terminal:
  795. self.maybe_trigger_ask("terminal")
  796. else: # "done": the completion report
  797. self.set_state("done", behaviour[1])
  798. self.push_results("done", summary=behaviour[1])
  799. if self.ask_on_terminal:
  800. self.maybe_trigger_ask("terminal")
  801. elif self.suspend_on_title and self.suspend_on_title == self.running_title:
  802. # `saddle suspend`: park until a fresh assignment
  803. # resumes the workspace (title-targeted one-shot,
  804. # so an interleaved dispatch cannot eat the flag).
  805. self.set_state("suspended")
  806. self.suspend_on_title = None
  807. self.push_results("suspended")
  808. # A parked daemon is idle-awake: it asks (and
  809. # sleeps on 204) so a later wake can reach it.
  810. if self.ask_on_idle_status:
  811. self.maybe_trigger_ask("parked")
  812. else:
  813. self.set_state("done", "fake daemon finished")
  814. self.push_results("done", summary="fake daemon finished")
  815. if self.ask_on_terminal:
  816. self.maybe_trigger_ask("terminal")
  817. elif time.monotonic() - getattr(self, "last_heartbeat", 0) >= FAKE_HEARTBEAT_S:
  818. # The flush timer: output keeps flowing, so chunks
  819. # keep arriving (the staleness window never opens on
  820. # a healthy runner).
  821. self.last_heartbeat = time.monotonic()
  822. self.push_chunks(["working-heartbeat\r\n"])
  823. def serve(self, port=None):
  824. """Binds the daemon's HTTP server. A requested port that is
  825. already taken (free_port() hands out recyclable ports — two
  826. calls can collide) falls back to an OS-assigned one instead of
  827. silently dying and impersonating whatever owns that port."""
  828. if port is None:
  829. port = DAEMON_PORT
  830. handler = type("BoundHandler", (FakeSaddle.Handler,), {"daemon": self})
  831. self.server = None
  832. try:
  833. self.server = ThreadingHTTPServer(("127.0.0.1", port), handler)
  834. except OSError:
  835. print("(fake daemon: port %d taken, falling back)" % port)
  836. self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
  837. self.port = self.server.server_address[1]
  838. threading.Thread(target=self.server.serve_forever, daemon=True).start()
  839. threading.Thread(target=self.run_lifecycle, daemon=True).start()
  840. return self.server
  841. def kill(self):
  842. """The daemon dies: it stops listening (connections refused) AND
  843. stops calling home — no pushes, no asks (a dead process has no
  844. outbound anything; in-flight handler threads may linger but the
  845. wire contract's traffic is one-directional now)."""
  846. self.dead = True
  847. if getattr(self, "server", None) is not None:
  848. self.server.shutdown()
  849. self.server.server_close()
  850. self.server = None
  851. def start_app():
  852. config = json.load(open(os.path.join(REPO, "web-config.json")))
  853. config["saddle_callback_base_url"] = WEB
  854. with open(os.path.join(WORK, "web-config.json"), "w") as handle:
  855. json.dump(config, handle)
  856. env = dict(os.environ)
  857. env.update({
  858. "SPRY_DB_PATH": DB_PATH,
  859. "SPRY_SEED_ADMIN_USERNAME": "admin",
  860. "SPRY_SEED_ADMIN_PASSWORD": "adminpass123",
  861. "SADDLE_LLM_BASE_URL": "http://127.0.0.1:%d" % DAEMON_PORT,
  862. "SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
  863. "SADDLE_HTTP_TIMEOUT_S": "2",
  864. "SADDLE_LOST_WAKE_S": "2",
  865. "SADDLE_STALE_S": "3",
  866. "SADDLE_DISPATCH_TICK_S": "1",
  867. # Plan 12: the goal-agent resume coalescing window shrinks to
  868. # one second so the event-driven resume tests settle quickly.
  869. "SADDLE_GOAL_RESUME_S": os.environ.get("SADDLE_GOAL_RESUME_S", "1"),
  870. # Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
  871. # resolves the ancient /lib64 copies): empty makes the loader use
  872. # the binary's RUNPATH, which is the dev stack it was built
  873. # against.
  874. "LD_LIBRARY_PATH": "",
  875. })
  876. log_previous = os.path.join(WORK, "app.log.1")
  877. if os.path.exists(os.path.join(WORK, "app.log")):
  878. shutil.move(os.path.join(WORK, "app.log"), log_previous)
  879. log = open(os.path.join(WORK, "app.log"), "w")
  880. command = [APP, str(APP_PORT)]
  881. if os.environ.get("WEBSTACK_GDB"):
  882. # Debugging aid: run the app under gdb; a crash backtrace lands in
  883. # app.log. WEBSTACK_GDB=1 meson test ... to enable.
  884. command = ["gdb", "-batch", "-quiet",
  885. "-ex", "handle SIGPIPE nostop noprint pass",
  886. "-ex", "run", "-ex", "bt", "--args", APP, str(APP_PORT)]
  887. process = subprocess.Popen(command, cwd=WORK, env=env,
  888. stdout=log, stderr=subprocess.STDOUT)
  889. # The ready probe must require OUR process alive: a free_port()
  890. # TOCTOU can hand this port to another shard's app — ours then dies
  891. # at bind (MHD error()) while the probe happily answers from the
  892. # intruder (same admin seed), and every check would silently drive
  893. # the WRONG app. poll() != None with a serving port is exactly that
  894. # shape: take the exit branch, the start_bundle retry re-ports.
  895. outcome = wait_for(lambda: (app_ready() and process.poll() is None and "ready")
  896. or (process.poll() is not None and "app-exited-%d" % process.poll())
  897. or False, timeout=30, what="app ready")
  898. if outcome is None:
  899. raise SystemExit("app did not come up; see %s/app.log" % WORK)
  900. if str(outcome).startswith("app-exited-"):
  901. raise SystemExit("app exited during startup (%s); see %s/app.log"
  902. % outcome, WORK)
  903. return process
  904. def app_ready():
  905. try:
  906. status, _ = http("GET", "/login", timeout=3)
  907. return status == 200
  908. except (urllib.error.URLError, OSError):
  909. return False
  910. def dump_app_stacks(pid):
  911. """Best-effort thread backtrace of the (possibly wedged) app."""
  912. try:
  913. fds = os.listdir("/proc/%d/fd" % pid)
  914. print("(app open fds at dump: %d)" % len(fds))
  915. kinds = {}
  916. for fd in fds[:1200]:
  917. try:
  918. target = os.readlink("/proc/%d/fd/%s" % (pid, fd))
  919. except OSError:
  920. continue
  921. key = target.split(":", 1)[0] if ":" in target else target
  922. kinds[key] = kinds.get(key, 0) + 1
  923. print("(fd kinds: %s)" % sorted(kinds.items(), key=lambda kv: -kv[1])[:6])
  924. except OSError:
  925. pass
  926. for command in (["eu-stack", "-p", str(pid)],
  927. ["gdb", "-batch", "-quiet", "-p", str(pid),
  928. "-ex", "thread apply all bt"]):
  929. try:
  930. out = subprocess.run(command, capture_output=True, timeout=60)
  931. text = out.stdout.decode(errors="replace") + out.stderr.decode(errors="replace")
  932. if text.strip():
  933. print("(app stacks via %s)\n%s" % (command[0], text[:12000]))
  934. return
  935. except (OSError, subprocess.SubprocessError):
  936. continue
  937. # =============================================================================
  938. # Fake push service + Web Push protocol helpers (plan 11 task E)
  939. # =============================================================================
  940. def b64url(raw):
  941. return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
  942. def b64url_dec(text):
  943. return base64.urlsafe_b64decode(text + "=" * ((4 - len(text) % 4) % 4))
  944. class FakePushService:
  945. """In-process Web Push endpoint server: every subscription the tests
  946. register points at a path here (a distinct "browser endpoint").
  947. Responses are scriptable per path — a fixed status (the 410-prune
  948. fixture) or N transient 500s (the retry-next-tick fixture)."""
  949. def __init__(self):
  950. self.requests = [] # [{"path", "headers", "body", "at"}]
  951. self.fail_next = {} # path -> remaining 500s
  952. self.status_for = {} # path -> fixed status
  953. self.lock = threading.Lock()
  954. self.port = None
  955. self.server = None
  956. def serve(self):
  957. service = self
  958. class Handler(BaseHTTPRequestHandler):
  959. protocol_version = "HTTP/1.1"
  960. def log_message(self, fmt, *args):
  961. pass
  962. def do_POST(self):
  963. length = int(self.headers.get("Content-Length") or 0)
  964. body = self.rfile.read(length)
  965. status = 201
  966. if self.path in service.status_for:
  967. status = service.status_for[self.path]
  968. elif service.fail_next.get(self.path, 0) > 0:
  969. service.fail_next[self.path] -= 1
  970. status = 500
  971. with service.lock:
  972. service.requests.append({
  973. "path": self.path,
  974. "headers": {key.lower(): value
  975. for key, value in self.headers.items()},
  976. "body": body,
  977. "at": time.monotonic(),
  978. })
  979. payload = b"{}"
  980. self.send_response(status)
  981. self.send_header("Content-Type", "application/json")
  982. self.send_header("Content-Length", str(len(payload)))
  983. self.end_headers()
  984. self.wfile.write(payload)
  985. self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
  986. self.port = self.server.server_address[1]
  987. threading.Thread(target=self.server.serve_forever, daemon=True).start()
  988. return self.server
  989. def url(self, path):
  990. return "http://127.0.0.1:%d%s" % (self.port, path)
  991. def hits(self, prefix):
  992. with self.lock:
  993. return [request for request in self.requests
  994. if request["path"].startswith(prefix)]
  995. def decrypt_webpush(request, client_private_key, client_public, auth_secret):
  996. """RFC 8291 decryption of one captured push: returns
  997. (record_size, payload dict) — the payload builder's exact output."""
  998. body = request["body"]
  999. salt = body[:16]
  1000. record_size = int.from_bytes(body[16:20], "big")
  1001. key_id_len = body[20]
  1002. eph_public = body[21:21 + key_id_len]
  1003. ciphertext = body[21 + key_id_len:]
  1004. eph_point = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(),
  1005. eph_public)
  1006. shared = client_private_key.exchange(ec.ECDH(), eph_point)
  1007. info = b"WebPush: info\x00" + eph_public + client_public
  1008. cek = HKDF(algorithm=hashes.SHA256(), length=16, salt=salt,
  1009. info=info).derive(shared + auth_secret)
  1010. nonce = HKDF(algorithm=hashes.SHA256(), length=12, salt=salt,
  1011. info=info).derive(shared + auth_secret)
  1012. plaintext = AESGCM(cek).decrypt(nonce, ciphertext, None)
  1013. return record_size, json.loads(plaintext[:-1]) # strip the 0x02 delimiter
  1014. def verify_vapid(request, vapid_public, vapid_private):
  1015. """Verifies the ES256 VAPID JWT on one captured push; returns its
  1016. claims dict (aud/exp/sub) — raises on a bad signature."""
  1017. header = request["headers"].get("authorization", "")
  1018. assert header.startswith("vapid t="), "no vapid authorization header"
  1019. token_part = header[len("vapid t="):].split(",")[0]
  1020. sent_key = header.split("k=")[1]
  1021. head_b64, claims_b64, sig_b64 = token_part.split(".")
  1022. signature = b64url_dec(sig_b64)
  1023. key = ec.derive_private_key(
  1024. int.from_bytes(b64url_dec(vapid_private), "big"), ec.SECP256R1())
  1025. key.public_key().verify(
  1026. encode_dss_signature(int.from_bytes(signature[:32], "big"),
  1027. int.from_bytes(signature[32:], "big")),
  1028. (head_b64 + "." + claims_b64).encode(),
  1029. ec.ECDSA(hashes.SHA256()))
  1030. claims = json.loads(b64url_dec(claims_b64))
  1031. assert sent_key == vapid_public, "the k= key is not the app's VAPID key"
  1032. return claims
  1033. def action_notify(result):
  1034. """The message of the first notify directive an action returned."""
  1035. for directive in result:
  1036. if directive.get("type") == "notify":
  1037. return directive.get("message", "")
  1038. return None
  1039. # =============================================================================
  1040. # Shard orchestration
  1041. # =============================================================================
  1042. class Ctx:
  1043. """Per-bundle shared fixtures: the app process, the admin auth slot
  1044. and the primary fake saddle (registered + token delivered)."""
  1045. def start_bundle(name):
  1046. """Boots one shard's environment: scratch dir, database, ports, the
  1047. primary fake daemon and the app, then the shared fixtures every
  1048. bundle builds on (the demo project + the registered fake saddle).
  1049. Scenario groups inside the bundle share this app+daemon instance
  1050. and sequence their state as DB rows, never process restarts."""
  1051. global APP_PORT, WEB, DAEMON_PORT
  1052. bundle_env(name)
  1053. fake = FakeSaddle()
  1054. fake.serve()
  1055. DAEMON_PORT = fake.port # the actually-bound port (fallback-safe)
  1056. # A rare framework-level crash (stack corruption in Statum slot
  1057. # signing during the FIRST login — shared-library codegen bug, moved
  1058. # twice already) kills the app before any test runs; a clean restart
  1059. # (with fresh ports) sidesteps it. Only tolerated while nothing has
  1060. # been checked yet.
  1061. auth = None
  1062. app = None
  1063. for attempt in range(3):
  1064. APP_PORT = free_port()
  1065. WEB = "http://127.0.0.1:%d" % APP_PORT
  1066. if os.path.exists(WORK):
  1067. shutil.rmtree(WORK)
  1068. os.makedirs(os.path.join(WORK, "workspace"))
  1069. try:
  1070. app = start_app()
  1071. auth = login("admin", "adminpass123")
  1072. break
  1073. except (SystemExit, OSError, urllib.error.URLError):
  1074. if passes > 0:
  1075. raise
  1076. if app is not None and app.poll() is None:
  1077. app.terminate()
  1078. try:
  1079. app.wait(timeout=5)
  1080. except subprocess.TimeoutExpired:
  1081. app.kill()
  1082. app = None
  1083. print("(app died before login; restarting cleanly, attempt %d)" % (attempt + 1))
  1084. if app is None or auth is None:
  1085. raise SystemExit("app never came up; see %s/app.log" % WORK)
  1086. state = slot_public(auth, "/projects", "projects")
  1087. act(auth, state["create"], {
  1088. "name": "demo",
  1089. "git_url": "https://example.com/demo.git",
  1090. "default_branch": "main",
  1091. })
  1092. saddles_state = slot_public(auth, "/saddles", "saddles")
  1093. act(auth, saddles_state["register"], {"name": "fake-saddle",
  1094. "host": "127.0.0.1:%d" % fake.port})
  1095. registered = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  1096. if s["name"] == "fake-saddle"]
  1097. if wait_for(lambda: len(fake.register_log) > 0 or False,
  1098. timeout=15, what="/register delivery") is None:
  1099. raise SystemExit("fake saddle never received /register")
  1100. ctx = Ctx()
  1101. ctx.name = name
  1102. ctx.app = app
  1103. ctx.auth = auth
  1104. ctx.fake = fake
  1105. ctx.fake_id = int(registered[0]["id"])
  1106. ctx.fake_token = registered[0]["token"]
  1107. return ctx
  1108. # =============================================================================
  1109. # Bundle: pages — block presentation, diff URLs, computed display state,
  1110. # terminal rendering (rejected entries, ANSI SGR) and task-page layout.
  1111. # Fixture-driven: tasks and sessions are inserted as rows; blocks are
  1112. # raised through the push-era wire (POST /api/saddle/results with a
  1113. # blocked state) or inserted directly for terminal-task fixtures; only
  1114. # the rejected-ride scenario dispatches.
  1115. # =============================================================================
  1116. def bundle_pages(ctx):
  1117. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  1118. app = ctx.app
  1119. def raise_block(session_id, question, kind="question", options=None, body=None):
  1120. """The push-era block raise: a blocked round-results push
  1121. carrying the block payload (the poll-era /update channel is
  1122. gone — 404 below pins that)."""
  1123. block = {"question": question, "type": kind}
  1124. if options is not None:
  1125. block["options"] = options
  1126. if body is not None:
  1127. block["body"] = body
  1128. return saddle_api("POST", "/api/saddle/results", fake_token,
  1129. {"session_id": session_id, "state": "blocked",
  1130. "block": block})
  1131. check(True, "login yields an auth slot")
  1132. print("== the poll-era wire is gone (arch 19) ==")
  1133. # The daemon never called update/poll (verified against saddle/src);
  1134. # the routes 404 for any caller, authenticated or not.
  1135. status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1136. {"type": "status", "session_id": 1,
  1137. "data": {"state": "working"}})
  1138. check(status == 404, "POST /api/saddle/{id}/update is gone (404)")
  1139. status, _ = saddle_api("GET", "/api/saddle/%d/poll" % fake_id, fake_token)
  1140. check(status == 404, "GET /api/saddle/{id}/poll is gone (404)")
  1141. print("== blocks: push-era raises + body presentation ==")
  1142. conn = db()
  1143. conn.execute("insert into tasks (project_id, title, status, priority,"
  1144. " created_at)"
  1145. " values (1, 'Block display task', 'working', 5,"
  1146. " strftime('%s','now'))")
  1147. plan_task = conn.execute("select max(id) from tasks").fetchone()[0]
  1148. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
  1149. " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
  1150. (plan_task, fake_id))
  1151. conn.commit()
  1152. session_id = conn.execute("select max(id) from sessions").fetchone()[0]
  1153. conn.close()
  1154. status, _ = raise_block(session_id, "Which database?", kind="question",
  1155. options=["SQLite", "Postgres"], body="# Storage choice")
  1156. check(status == 200, "question block raised through the results push")
  1157. print("== B1: highlight-to-comment quote dialog ==")
  1158. status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
  1159. check(status == 200 and "quote-comment-dialog" in page and "quote-text" in page,
  1160. "task page carries the quote-comment dialog the popover opens")
  1161. check('name="quote"' in page and 'name="return_to"' in page,
  1162. "quote dialog posts the quote and return-to fields")
  1163. status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
  1164. check(status == 200 and "quote-comment-dialog" in page,
  1165. "terminal page keeps its quote dialog")
  1166. print("== B3: review section hides the duplicate body ==")
  1167. status, _ = raise_block(session_id, "Review the diff", kind="review",
  1168. body="# The change\n\nReworks the flange module.")
  1169. check(status == 200, "review block raised through the results push")
  1170. detail = task_state(auth, plan_task)
  1171. review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
  1172. check(detail.get("review_pending") is True,
  1173. "review section shows while the review is unanswered")
  1174. check(review_block.get("show_body") is False,
  1175. "pending review block hides its duplicate body preview")
  1176. check(review_block.get("body", "").startswith("# The change"),
  1177. "review body itself still arrives in state (B2 counterpart)")
  1178. act(auth, review_block["answer_action"], {"verdict": "accept"})
  1179. detail = task_state(auth, plan_task)
  1180. review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
  1181. check(review_block.get("show_body") is True,
  1182. "answered review shows the body again (banner gone)")
  1183. check(review_block.get("answer") == "Review approved. Continue with the task.",
  1184. "the review accept form's verdict marker resolves to the stable wording")
  1185. # Plan 16 S2/R4: no verdict constant ships in any template — the
  1186. # accept forms carry only the verdict marker.
  1187. task_page_html = http_get_page(auth, "/tasks/%d" % plan_task)
  1188. check("Plan approved. Proceed with implementation." not in task_page_html
  1189. and "Review approved. Continue with the task." not in task_page_html,
  1190. "the task page ships no verdict constants (server-owned wording)")
  1191. check('name="verdict" value="accept"' in task_page_html,
  1192. "the accept forms post the verdict marker")
  1193. # Plan 16 S5 (U1): verdict-shaped blocks keep their explicit
  1194. # buttons + free-text path; a question with NULL options renders
  1195. # the question form (radio "other" + free reply), never a
  1196. # free-text-only verdict.
  1197. check("Request changes" in task_page_html and 'name="feedback"' in task_page_html,
  1198. "the review verdict keeps Request changes with free-text feedback (U1)")
  1199. check('value="__other__"' in task_page_html and 'name="other"' in task_page_html,
  1200. "question blocks render the option/other question form (U1)")
  1201. status, _ = raise_block(session_id, "Free-text thought?", kind="question")
  1202. check(status == 200, "a NULL-options question block raises")
  1203. bare_question = [b for b in task_state(auth, plan_task)["blocks"]
  1204. if b["question"] == "Free-text thought?"][0]
  1205. check(bare_question.get("options") == []
  1206. and bare_question.get("block_type") == "question",
  1207. "a NULL-options question stays a question (no options, not a verdict)")
  1208. print("== B2: question block body above options ==")
  1209. # Contract side (A2 will restore bodies end-to-end): a question
  1210. # block raised with a heredoc body carries it; the body preview
  1211. # rides above the options (template order) and survives answering
  1212. # alongside the selected answer. (Blocks list newest-first; the
  1213. # U1 fixture above raised a second, options-less question.)
  1214. detail = task_state(auth, plan_task)
  1215. question = [b for b in detail["blocks"]
  1216. if b["block_type"] == "question"
  1217. and b["body"] == "# Storage choice"][0]
  1218. check(question.get("body") == "# Storage choice"
  1219. and question.get("show_body") is True,
  1220. "question block keeps its body markdown with the preview on")
  1221. act(auth, question["answer_action"], {"option": "SQLite"})
  1222. detail = task_state(auth, plan_task)
  1223. question = [b for b in detail["blocks"]
  1224. if b["block_type"] == "question"
  1225. and b["body"] == "# Storage choice"][0]
  1226. check(question.get("answered") is True and question.get("answer") == "SQLite"
  1227. and question.get("body") == "# Storage choice"
  1228. and question.get("show_body") is True,
  1229. "answered question shows body + selected answer")
  1230. print("== B5: compare-changes SSH -> HTTPS ==")
  1231. def diff_url_for_project(git_url, suffix):
  1232. state = slot_public(auth, "/projects", "projects")
  1233. act(auth, state["create"], {"name": "diff-%s" % suffix, "git_url": git_url})
  1234. project_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  1235. if p["name"] == "diff-%s" % suffix][0])
  1236. conn = db()
  1237. conn.execute("insert into tasks (project_id, title, status, task_branch,"
  1238. " branch_from, created_at) values (?, ?, 'working', 'feature/x',"
  1239. " 'main', strftime('%s','now'))", (project_id, "Diff %s" % suffix))
  1240. conn.commit()
  1241. task_id = conn.execute("select max(id) from tasks").fetchone()[0]
  1242. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at,"
  1243. " outcome) values (?, ?, strftime('%s','now'), strftime('%s','now'),"
  1244. " 'done')", (task_id, fake_id))
  1245. conn.commit()
  1246. session = conn.execute("select max(id) from sessions").fetchone()[0]
  1247. conn.close()
  1248. status, _ = raise_block(session, "Review the diff", kind="review")
  1249. return task_state(auth, task_id).get("review_diff_url") or ""
  1250. check(diff_url_for_project("ssh://fabrica.unitatem.net:222/clanker/clanker-saddle.git",
  1251. "fabrica")
  1252. == "https://fabrica.unitatem.net/clanker/clanker-saddle/compare/main...feature/x",
  1253. "ssh://git@host:port/owner/repo.git translates (port dropped)")
  1254. check(diff_url_for_project("git@github.com:acme/widgets.git", "scp")
  1255. == "https://github.com/acme/widgets/compare/main...feature/x",
  1256. "git@host:owner/repo.git translates to the browse host")
  1257. check(diff_url_for_project("https://example.com/plain/repo", "plain")
  1258. == "https://example.com/plain/repo/compare/main...feature/x",
  1259. "HTTPS URLs pass through unchanged")
  1260. print("== B6: computed display state ==")
  1261. # Draft tasks display as draft (plan 11) and never dispatch.
  1262. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1263. "project_id": "1",
  1264. "title": "Draft display task",
  1265. "priority": "5",
  1266. })
  1267. draft_display = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1268. if t["title"] == "Draft display task"][0])
  1269. detail = task_state(auth, draft_display)
  1270. check(detail["status"] == "draft" and detail.get("display") == "draft",
  1271. "new tasks default to draft and display it")
  1272. check(detail.get("draft") is True and detail.get("enqueue") is not None
  1273. and detail.get("delete") is not None,
  1274. "draft tasks expose the enqueue + delete actions")
  1275. check(quiet_window(quiet_seconds(),
  1276. lambda: task_state(auth, draft_display)["status"] == "draft",
  1277. "draft task never dispatches"),
  1278. "a draft task is never dispatched or woken for")
  1279. # Plan 12: the waiting-on-drafts block is deleted (goal tasks are
  1280. # the goal agent's to sequence); a parked task displays plainly.
  1281. conn = db()
  1282. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  1283. " values (1, 'Parked plain task', 'suspended', 5, strftime('%s','now'))")
  1284. conn.commit()
  1285. conn.close()
  1286. parked_id = db().execute("select max(id) from tasks").fetchone()[0]
  1287. detail = task_state(auth, parked_id)
  1288. check(detail.get("display") == "parked",
  1289. "a suspended task displays as parked (%s)" % detail.get("display"))
  1290. dash = slot_public(auth, "/", "dashboard")
  1291. check(not dash.get("waiting_drafts"),
  1292. "the waiting-on-drafts block is gone from the dashboard data")
  1293. status, home_page = http("GET", "/", timeout=15)
  1294. check(status == 200 and "Waiting on subtasks to be enqueued" not in home_page,
  1295. "the dashboard renders no waiting-on-drafts section")
  1296. conn = db()
  1297. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  1298. " values (1, 'Parent', 'done', 5, strftime('%s','now'))")
  1299. parent6 = conn.execute("select max(id) from tasks").fetchone()[0]
  1300. # The post-mortem case: stored done WITH an unanswered review block.
  1301. # (A terminal task consumes result pushes silently — plan 13 B2 —
  1302. # so the row is inserted directly, the goalui deep-link pattern.)
  1303. conn.execute("insert into task_blocks (task_id, question, type, created_at)"
  1304. " values (?, 'Review the final diff', 'review',"
  1305. " strftime('%s','now'))", (parent6,))
  1306. conn.commit()
  1307. conn.close()
  1308. detail = task_state(auth, parent6)
  1309. check(detail.get("display") == "review requested" and detail.get("terminal") is True,
  1310. "unanswered review overrides the stored done status (terminal flag keeps stored truth)")
  1311. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1312. if int(t["id"]) == parent6][0]
  1313. check(queue["display"] == "review requested",
  1314. "queue row shows the computed review-requested display")
  1315. # With the review answered the display falls back to the stored
  1316. # status (the plan-12 shape: no subtask-delegation states).
  1317. conn = db()
  1318. conn.execute("update task_blocks set answered_at = strftime('%s','now'),"
  1319. " answer = 'Human approved the review' where task_id = ?", (parent6,))
  1320. conn.commit()
  1321. conn.close()
  1322. detail = task_state(auth, parent6)
  1323. check(detail.get("display") == "done" and detail.get("terminal") is True,
  1324. "answered review falls back to the stored done display (%s)"
  1325. % detail.get("display"))
  1326. # Flags track the stored status on failure.
  1327. conn = db()
  1328. conn.execute("update tasks set status = 'failed', completed_at = strftime('%s','now')"
  1329. " where id = ?", (parent6,))
  1330. conn.commit()
  1331. conn.close()
  1332. detail = task_state(auth, parent6)
  1333. check(detail.get("display") == "failed" and detail.get("failed") is True
  1334. and detail.get("terminal") is True,
  1335. "failed task with answered review shows failed + terminal flags")
  1336. print("== rejected entries (non-fatal) ==")
  1337. # The push wire: a rejected entry rides a round-results push, stores
  1338. # on the session and streams through SSE; the task is untouched.
  1339. status, _ = saddle_api("POST", "/api/saddle/results", fake_token, {
  1340. "session_id": session_id, "state": "working",
  1341. "entries": [{
  1342. "id": 9021, "command": "bash -n script.sh",
  1343. "output": "syntax error near line 3 (recoverable)",
  1344. "exit_code": 1, "type": "rejected",
  1345. "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
  1346. }]})
  1347. check(status == 200, "a rejected entry rides the results push")
  1348. request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
  1349. with urllib.request.urlopen(request, timeout=15) as stream:
  1350. rejected_event = None
  1351. deadline = time.monotonic() + 8
  1352. while time.monotonic() < deadline and rejected_event is None:
  1353. line = stream.readline().decode().strip()
  1354. if not line.startswith("data: ") or line == "data: ping":
  1355. continue
  1356. event = json.loads(line[6:])
  1357. if event.get("type") == "rejected":
  1358. rejected_event = event
  1359. check(rejected_event is not None
  1360. and "syntax error near line 3" in (rejected_event.get("output") or ""),
  1361. "terminal stream carries rejected entries with their reason")
  1362. # The page template renders them with the distinct non-fatal style.
  1363. status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
  1364. check(status == 200 and "term-rejected" in page,
  1365. "terminal page renders rejected entries in their own style")
  1366. print("== B8: ANSI SGR in the terminal view ==")
  1367. # Byte-exact SGR delivery: the daemon's coloured chunks flow through
  1368. # the store + SSE untouched (the client renderer styles them).
  1369. coloured = "\x1b[1;32mbold green\x1b[0m then \x1b[38;5;196m256-red\x1b[0m\n"
  1370. conn = db()
  1371. conn.execute("insert into session_raw_log (session_id, seq, chunk)"
  1372. " values (?, 900, ?)", (session_id, coloured))
  1373. conn.commit()
  1374. conn.close()
  1375. request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
  1376. with urllib.request.urlopen(request, timeout=15) as stream:
  1377. coloured_event = None
  1378. deadline = time.monotonic() + 8
  1379. while time.monotonic() < deadline and coloured_event is None:
  1380. line = stream.readline().decode().strip()
  1381. if not line.startswith("data: ") or line == "data: ping":
  1382. continue
  1383. event = json.loads(line[6:])
  1384. if event.get("type") == "raw" and "\x1b[1;32m" in (event.get("text") or ""):
  1385. coloured_event = event
  1386. check(coloured_event is not None
  1387. and "\x1b[38;5;196m" in coloured_event["text"],
  1388. "SGR sequences stream byte-exact to the client renderer")
  1389. status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
  1390. check(status == 200 and "makeAnsiRenderer" in page and "colour256" in page,
  1391. "terminal page embeds the ANSI renderer (SGR + 256-colour palette)")
  1392. # Dispatch path: a rejected entry in a round's results push flows
  1393. # into the audit trail without failing (or finishing) the task.
  1394. task_id_tmp = create_queued(auth, {
  1395. "project_id": "1",
  1396. "title": "Rejected ride",
  1397. "priority": "2",
  1398. })
  1399. rejected_task = task_id_tmp
  1400. check(wait_for(lambda: task_state(auth, rejected_task).get("status") == "working"
  1401. or False, timeout=60, what="rejected-ride working") is not None,
  1402. "rejected-ride task reaches working")
  1403. fake.queue_output("rejected", "echo 'too long' | saddle",
  1404. "max_tokens exceeded (recoverable)")
  1405. check(wait_task_done(auth, rejected_task, timeout=60) is not None,
  1406. "task completes normally past rejected entries")
  1407. conn = db()
  1408. stored = conn.execute(
  1409. "select count(*) from session_commands sc join sessions s on s.id = sc.session_id"
  1410. " where s.task_id = ? and sc.type = 'rejected'", (rejected_task,)).fetchone()[0]
  1411. conn.close()
  1412. check(stored >= 1,
  1413. "rejected entries stored in the audit trail (%d rows)" % stored)
  1414. check(task_state(auth, rejected_task)["status"] == "done"
  1415. and "fake daemon finished" in (task_state(auth, rejected_task).get("summary") or ""),
  1416. "rejected entries never fail the task")
  1417. print("== plan 11 task D: task page — progress first, 50/50 grid, expanders ==")
  1418. detail = task_state(auth, plan_task)
  1419. block_ids = [int(b["id"]) for b in detail.get("blocks", [])]
  1420. check(len(block_ids) >= 3 and block_ids == sorted(block_ids, reverse=True),
  1421. "the blocks section orders newest-first via the sealed GET state (%s)"
  1422. % block_ids)
  1423. status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
  1424. # Plan 11 task D: the progress feed sits ABOVE the status card;
  1425. # status + description share a 50/50 grid; the plan detail hides
  1426. # behind its expander below it.
  1427. check(status == 200 and "grid-2" in page
  1428. and page.index("<h3>Progress</h3>") < page.index("status-head"),
  1429. "the progress feed renders above the status card")
  1430. check(page.index("grid-2") < page.index("status-head")
  1431. and page.index("status-head") < page.index("<h3>Mandate</h3>"),
  1432. "status and description sit side by side inside the grid (status first)")
  1433. check("<h3>Mandate</h3>" in page
  1434. and "plan-expander" not in page,
  1435. "the task page carries no plan expander (plans live on the goal)")
  1436. # Answered blocks fold behind the history expander (this fixture's
  1437. # three blocks are all answered by now).
  1438. check("blocks-history" in page and "Answered blocks" in page,
  1439. "answered blocks hide behind the blocks history expander")
  1440. conn = db()
  1441. conn.execute("insert into tasks (project_id, title, status, priority,"
  1442. " created_at) values (1, 'Progress layout task', 'working', 5,"
  1443. " strftime('%s','now','-120 seconds'))")
  1444. layout_task = conn.execute("select max(id) from tasks").fetchone()[0]
  1445. # backdated so the dashboard-timeline fixtures below stay the
  1446. # newest events
  1447. conn.execute("insert into task_progress (task_id, summary, created_at)"
  1448. " values (?, 'first update', strftime('%s','now','-90 seconds'))",
  1449. (layout_task,))
  1450. conn.execute("insert into task_progress (task_id, summary, created_at)"
  1451. " values (?, 'second update', strftime('%s','now','-89 seconds'))",
  1452. (layout_task,))
  1453. conn.commit()
  1454. conn.close()
  1455. status, history_page = http("GET", "/tasks/%d" % layout_task, timeout=15)
  1456. check(status == 200
  1457. and history_page.index("progress-latest") < history_page.index("progress-history"),
  1458. "the progress timeline renders newest-first (latest above the history)")
  1459. # The edit expander sits UNDER the "Show history" expander.
  1460. check(history_page.index("Show history") < history_page.index("edit-expander"),
  1461. "the edit expander sits under the Show history expander")
  1462. print("== heredoc completion reports: markdown card + timeline snippet ==")
  1463. # The daemon's `cat << 'EOF' | saddle done` stores a markdown
  1464. # summary (the failure path stores the same field); the task page
  1465. # renders it as a full-width completion-report card directly below
  1466. # the status/description grid — markdown via stm-markdown
  1467. # (XSS-safe), "Completion report" when done, "Failure summary"
  1468. # when failed, no card when the summary is empty. The dashboard
  1469. # timeline snippets the summary's FIRST line.
  1470. report_md = ("## Flange wired and tested\n\n"
  1471. "Ran `meson test` over the full matrix.\n\n"
  1472. "```sh\nmeson test -C builddir\n```\n\n"
  1473. "<b>not html</b> & escaped\n")
  1474. conn = db()
  1475. conn.execute("insert into tasks (project_id, title, status, priority,"
  1476. " summary, started_at, completed_at, created_at)"
  1477. " values (1, 'Report done task', 'done', 5, ?,"
  1478. " strftime('%s','now'), strftime('%s','now'),"
  1479. " strftime('%s','now'))", (report_md,))
  1480. report_done = conn.execute("select max(id) from tasks").fetchone()[0]
  1481. conn.execute("insert into tasks (project_id, title, status, priority,"
  1482. " summary, started_at, completed_at, created_at)"
  1483. " values (1, 'Report failed task', 'failed', 5,"
  1484. " 'Boom: tests failed\n\nsecond line carries the detail',"
  1485. " strftime('%s','now'), strftime('%s','now'),"
  1486. " strftime('%s','now'))")
  1487. report_failed = conn.execute("select max(id) from tasks").fetchone()[0]
  1488. conn.execute("insert into tasks (project_id, title, status, priority,"
  1489. " started_at, completed_at, created_at)"
  1490. " values (1, 'Report empty task', 'done', 5,"
  1491. " strftime('%s','now'), strftime('%s','now'),"
  1492. " strftime('%s','now'))")
  1493. report_empty = conn.execute("select max(id) from tasks").fetchone()[0]
  1494. conn.commit()
  1495. conn.close()
  1496. done_state = task_state(auth, report_done)
  1497. check(done_state["report_title"] == "Completion report"
  1498. and "## Flange wired" in (done_state.get("summary") or ""),
  1499. "a done task's summary rides the report card state")
  1500. check(task_state(auth, report_failed)["report_title"] == "Failure summary",
  1501. "a failed task's card heading reads Failure summary")
  1502. empty_state = task_state(auth, report_empty)
  1503. check(empty_state["report_title"] == "" and not (empty_state.get("summary") or ""),
  1504. "an empty summary renders no report card (no heading)")
  1505. check(task_state(auth, layout_task)["report_title"] == "",
  1506. "non-terminal tasks never show the report card")
  1507. status, page = http("GET", "/tasks/%d" % report_done, timeout=15)
  1508. check(status == 200 and "completion-report" in page
  1509. and page.index("grid-2") < page.index("completion-report")
  1510. < page.index("edit-expander"),
  1511. "the completion-report card sits below the status/description grid, full width")
  1512. card = page[page.index("completion-report"):page.index("edit-expander")]
  1513. check('stm-markdown="task.summary"' in card
  1514. and 'stm-text="task.report_title"' in card,
  1515. "the card renders the summary as markdown under its state-driven heading")
  1516. check("<strong>Summary:</strong>" not in page,
  1517. "the old plain-text summary line is gone (the card owns the summary)")
  1518. # The markdown itself renders client-side: drive the SHIPPED
  1519. # stm-markdown renderer (statum.js) on the fixture summary via
  1520. # node — headings and code come out as HTML, no markdown syntax
  1521. # leaks, raw HTML stays escaped.
  1522. status, statum_js = http("GET", "/_statum/resource/statum.js", timeout=15)
  1523. skip = None if status == 200 else "statum.js not served"
  1524. parts = []
  1525. if skip is None:
  1526. def extract_md(name, source):
  1527. start = source.index("function %s(" % name)
  1528. depth = 0
  1529. index = source.index("{", start)
  1530. end = index
  1531. while True:
  1532. if source[end] == "{":
  1533. depth += 1
  1534. elif source[end] == "}":
  1535. depth -= 1
  1536. if depth == 0:
  1537. break
  1538. end += 1
  1539. return source[start:end + 1]
  1540. try:
  1541. parts = [extract_md(name, statum_js)
  1542. for name in ("safeMarkdownUrl", "mdLinks", "mdInline", "renderMarkdown")]
  1543. except ValueError:
  1544. skip = "renderer functions missing"
  1545. rendered = ""
  1546. if skip is None:
  1547. try:
  1548. result = subprocess.run(
  1549. ["node", "-e", "%s\n%s\n%s\n%s\nconsole.log(renderMarkdown(process.argv[1]));"
  1550. % tuple(parts), report_md],
  1551. capture_output=True, text=True, timeout=20)
  1552. rendered = result.stdout
  1553. except (OSError, subprocess.SubprocessError):
  1554. skip = "node unavailable"
  1555. if skip is not None:
  1556. check(True, "stm-markdown renderer present (%s, deep check skipped)" % skip)
  1557. else:
  1558. check("<h2>Flange wired and tested</h2>" in rendered
  1559. and "<pre><code" in rendered and "<code>meson test</code>" in rendered,
  1560. "the report summary renders headings and code through stm-markdown")
  1561. check("## Flange" not in rendered and "```sh" not in rendered
  1562. and "`meson test`" not in rendered,
  1563. "no raw markdown syntax leaks into the rendered report")
  1564. check("<b>not html</b>" not in rendered and "&lt;b&gt;not html&lt;/b&gt;" in rendered
  1565. and "&amp; escaped" in rendered,
  1566. "raw HTML in the summary stays escaped (XSS-safe)")
  1567. # The dashboard timeline: completion/failure events snippet the
  1568. # summary's first line, falling back to the bare verdict text.
  1569. dash = slot_public(auth, "/", "dashboard")
  1570. done_event = next((e for e in dash.get("activity", [])
  1571. if e["kind"] == "completed" and int(e["task_id"]) == report_done), None)
  1572. check(done_event is not None
  1573. and done_event["text"] == "Task completed: ## Flange wired and tested",
  1574. "the timeline completion event snippets the summary's first line")
  1575. failed_event = next((e for e in dash.get("activity", [])
  1576. if e["kind"] == "failed" and int(e["task_id"]) == report_failed), None)
  1577. check(failed_event is not None
  1578. and failed_event["text"] == "Task failed: Boom: tests failed",
  1579. "the timeline failure event snippets the summary's first line")
  1580. empty_event = next((e for e in dash.get("activity", [])
  1581. if e["kind"] == "completed" and int(e["task_id"]) == report_empty), None)
  1582. check(empty_event is not None and empty_event["text"] == "Task completed",
  1583. "a missing summary falls back to the bare timeline text")
  1584. print("== plan 11 task D: block split + queue-screen layout ==")
  1585. # One open and one answered question: the open block renders in
  1586. # full, the answered one folds behind the history expander.
  1587. conn = db()
  1588. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  1589. " values (?, ?, strftime('%s','now'))", (layout_task, fake_id))
  1590. conn.commit()
  1591. split_session = conn.execute("select max(id) from sessions").fetchone()[0]
  1592. conn.close()
  1593. raise_block(split_session, "Open question for layout?", kind="question")
  1594. raise_block(split_session, "Answered question for layout?", kind="question")
  1595. answered_block = [b for b in task_state(auth, layout_task)["blocks"]
  1596. if b["question"] == "Answered question for layout?"][0]
  1597. act(auth, answered_block["answer_action"], {"answer": "Yes"})
  1598. detail = task_state(auth, layout_task)
  1599. check(len(detail.get("open_blocks", [])) == 1
  1600. and len(detail.get("answered_blocks", [])) == 1
  1601. and detail["open_blocks"][0]["question"] == "Open question for layout?"
  1602. and detail["answered_blocks"][0]["question"] == "Answered question for layout?",
  1603. "the task slot splits blocks into open (current) and answered lists")
  1604. status, page = http("GET", "/tasks/%d" % layout_task, timeout=15)
  1605. check(status == 200
  1606. and page.index('stm-for-block-in="task.open_blocks"') < page.index("blocks-history")
  1607. < page.index('stm-for-block-in="task.answered_blocks"'),
  1608. "current blocks render in full above the answered-behind-expander history")
  1609. check(page.index("status-head") < page.index("Cancel task"),
  1610. "the lifecycle buttons ride the status card inside the grid")
  1611. status, queue_page = http("GET", "/tasks", timeout=15)
  1612. check(status == 200
  1613. and queue_page.index("filter-status") < queue_page.index("<table")
  1614. and "filter-goal" in queue_page,
  1615. "the filter controls (status + goal) sit above the queue table")
  1616. check("New task" in queue_page and "task-mode" in queue_page
  1617. and "task-enqueue-after" in queue_page
  1618. and 'name="plan_auto_approve"' not in queue_page,
  1619. "the create expander offers the mode + auto-enqueue-after selects"
  1620. " (no subtask checkboxes)")
  1621. print("== yeehaw: ingestion, festive card, badge, timeline ==")
  1622. # `saddle yeehaw <message>` arrives as a round-results entry with the
  1623. # message in the output field: it stores like any entry type, earns
  1624. # the task its 🤠 badge (derived, nothing new stored), renders as the
  1625. # task page's festive card (last 3, newest first, "+N more") and
  1626. # surfaces on the dashboard timeline as its own event kind.
  1627. conn = db()
  1628. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  1629. " values (1, 'Yeehaw task', 'working', 5, strftime('%s','now'))")
  1630. yeehaw_task = conn.execute("select max(id) from tasks").fetchone()[0]
  1631. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  1632. " values (?, ?, strftime('%s','now'))", (yeehaw_task, fake_id))
  1633. yeehaw_session = conn.execute("select max(id) from sessions").fetchone()[0]
  1634. conn.commit()
  1635. conn.close()
  1636. now_epoch = int(time.time())
  1637. def push_yeehaw(entry_id, message, offset=0):
  1638. """One celebration riding a round-results push, timestamped
  1639. `offset` seconds from now (distinct epochs keep the timeline's
  1640. newest-first ordering observable)."""
  1641. return saddle_api("POST", "/api/saddle/results", fake_token, {
  1642. "session_id": yeehaw_session, "state": "working",
  1643. "entries": [{
  1644. "id": entry_id, "command": "saddle yeehaw", "output": message,
  1645. "exit_code": 0, "type": "yeehaw",
  1646. "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ",
  1647. time.gmtime(now_epoch + offset)),
  1648. }]})[0]
  1649. def yeehaw_rows():
  1650. conn = db()
  1651. try:
  1652. return conn.execute(
  1653. "select count(*) from session_commands where session_id = ?"
  1654. " and type = 'yeehaw'", (yeehaw_session,)).fetchone()[0]
  1655. finally:
  1656. conn.close()
  1657. check(push_yeehaw(9101, "Shipped the flange") == 200,
  1658. "yeehaw entries ingest through the results push like any entry type")
  1659. check(yeehaw_rows() == 1,
  1660. "the yeehaw entry stores as a session_commands row (message in output)")
  1661. check(push_yeehaw(9101, "Shipped the flange") == 200 and yeehaw_rows() == 1,
  1662. "a redelivered yeehaw push is idempotent on the daemon entry id")
  1663. check(task_state(auth, yeehaw_task)["status"] == "working",
  1664. "a yeehaw entry never derails the task state")
  1665. push_yeehaw(9102, "Second hoedown", 1)
  1666. push_yeehaw(9103, "Third hoedown\nsecond line carries the detail", 2)
  1667. push_yeehaw(9104, "Fourth hoedown", 3)
  1668. push_yeehaw(9105, "Fifth hoedown", 4)
  1669. detail = task_state(auth, yeehaw_task)
  1670. check(detail.get("yeehaw") is True and detail.get("yeehaw_total") == 5,
  1671. "the detail state flags the badge presence and the uncapped total")
  1672. check([y["message"] for y in detail.get("yeehaws", [])]
  1673. == ["Fifth hoedown", "Fourth hoedown",
  1674. "Third hoedown\nsecond line carries the detail"],
  1675. "the festive card keeps the last 3 yeehaws, newest first (full messages)")
  1676. check(detail.get("yeehaw_more") == 2,
  1677. "the '+N more' hint counts the capped tail")
  1678. status, page = http("GET", "/tasks/%d" % yeehaw_task, timeout=15)
  1679. # The template compiler HTML-escapes the emoji (&#129312; = 🤠).
  1680. check(status == 200 and "yeehaw-card" in page and "&#129312; Yeehaw!" in page,
  1681. "the task page renders the festive yeehaw card")
  1682. check(page.index("<h3>Progress</h3>") < page.index("yeehaw-card")
  1683. < page.index("status-head"),
  1684. "the card sits between the progress feed and the status grid")
  1685. check('stm-text="task.yeehaw_more"' in page and "more yeehaw" in page,
  1686. "the card carries the +N more hint")
  1687. check("badge-yeehaw" in page and "status-corner" in page,
  1688. "the status card corner carries the earned 🤠 badge")
  1689. plain_detail = task_state(auth, layout_task)
  1690. check(plain_detail.get("yeehaw") is False and plain_detail.get("yeehaw_total") == 0
  1691. and plain_detail.get("yeehaws", []) == [],
  1692. "tasks without yeehaw entries carry no card state")
  1693. status, plain_page = http("GET", "/tasks/%d" % layout_task, timeout=15)
  1694. check(status == 200 and 'section class="card yeehaw-card" stm-if="task.yeehaw"' in plain_page,
  1695. "the card template only renders under state gating (stm-if task.yeehaw)")
  1696. queue_rows = slot_public(auth, "/tasks", "tasks")["tasks"]
  1697. check([t for t in queue_rows if int(t["id"]) == yeehaw_task][0].get("has_yeehaw") is True,
  1698. "queue rows badge yeehaw tasks")
  1699. check([t for t in queue_rows if int(t["id"]) == layout_task][0].get("has_yeehaw") is False,
  1700. "queue rows leave yeehaw-less tasks bare")
  1701. status, queue_page = http("GET", "/tasks", timeout=15)
  1702. check(status == 200 and 'stm-if="task.has_yeehaw"' in queue_page,
  1703. "the queue template renders the row badge")
  1704. dash = slot_public(auth, "/", "dashboard")
  1705. yeehaw_events = [e for e in dash.get("activity", []) if e["kind"] == "yeehaw"]
  1706. check(len(yeehaw_events) == 5
  1707. and all(int(e["task_id"]) == yeehaw_task for e in yeehaw_events),
  1708. "every yeehaw entry surfaces as its own timeline event kind")
  1709. check(yeehaw_events[0]["text"] == "🤖 yeehaw'd: Fifth hoedown",
  1710. "the newest yeehaw event leads the timeline rows")
  1711. check(all("second line" not in e["text"] for e in yeehaw_events),
  1712. "multiline yeehaw messages snippet to their first line")
  1713. referenced = [e for e in dash.get("activity", []) if int(e["task_id"]) == yeehaw_task]
  1714. check(len(referenced) > 0 and all(e.get("task_yeehaw") is True for e in referenced),
  1715. "dashboard timeline task references carry the badge flag")
  1716. plain_refs = [e for e in dash.get("activity", []) if int(e["task_id"]) == layout_task]
  1717. check(len(plain_refs) > 0 and all(e.get("task_yeehaw") is False for e in plain_refs),
  1718. "references to yeehaw-less tasks stay bare")
  1719. status, home_page = http("GET", "/", timeout=15)
  1720. check(status == 200 and 'stm-if="item.task_yeehaw"' in home_page
  1721. and "badge-yeehaw" in home_page,
  1722. "the dashboard renders the badge on timeline task references")
  1723. # (The old R10 scenario — branch_advanced riding /poll verdicts —
  1724. # died with the poll channel; the answers bundle pins the hint on
  1725. # its surviving surface, the resumed assignment payload.)
  1726. print("== plan 12: the web surfaces — pile, timeline, queue, task page ==")
  1727. # A goal parked in plan review feeds the unified dashboard pile;
  1728. # goal events feed the timeline.
  1729. def goal_state_local(goal_id):
  1730. return slot_public(auth, "/goals/%d" % goal_id, "goal")
  1731. act(auth, slot_public(auth, "/goals", "goals")["create"], {
  1732. "project_id": "1", "title": "Dashboard goal",
  1733. "brief": "Feeds the pile.", "branch_name": "goal/dash"})
  1734. conn = db()
  1735. goal3 = conn.execute(
  1736. "select id from goals where title = 'Dashboard goal'").fetchone()[0]
  1737. conn.close()
  1738. fake.goal_script = [("plan", "# Dash plan\n\n1. Probe the pile")]
  1739. act(auth, goal_state_local(goal3)["start"], {})
  1740. check(wait_for(lambda: goal_state_local(goal3)["status"] == "plan_review" or False,
  1741. timeout=60, what="dashboard goal enters plan review") is not None,
  1742. "the dashboard goal reaches plan review")
  1743. check(wait_for(lambda: any(b.get("item_type") == "goal"
  1744. and int(b.get("goal_id") or 0) == goal3
  1745. for b in slot_public(auth, "/", "dashboard")["pinned"])
  1746. or False, timeout=30, what="goal block pins on the dashboard") is not None,
  1747. "the unified dashboard pile carries the goal's plan review")
  1748. pile = slot_public(auth, "/", "dashboard")["pinned"]
  1749. check(all(b.get("item_type") in ("task", "goal") for b in pile),
  1750. "pile items carry type badges (task/goal interleaved)")
  1751. goal_pile = [b for b in pile if b.get("item_type") == "goal"][0]
  1752. check(goal_pile["block_type"] == "plan" and "Dashboard goal" in goal_pile["goal_title"],
  1753. "the goal pile card names the goal and block kind")
  1754. # Answer it: the pile drains.
  1755. act(auth, goal_state_local(goal3)["plan_pending_block"]["answer_action"],
  1756. {"answer": "Plan approved. Proceed with implementation."})
  1757. check(wait_for(lambda: not any(b.get("item_type") == "goal"
  1758. and int(b.get("goal_id") or 0) == goal3
  1759. for b in slot_public(auth, "/", "dashboard")["pinned"])
  1760. or False, timeout=30, what="goal pile card drains") is not None,
  1761. "answering the review drains the goal's pile card")
  1762. activity = slot_public(auth, "/", "dashboard")["activity"]
  1763. check(any(a["kind"] == "goal-plan-submitted" and int(a.get("goal_id") or 0) == goal3
  1764. and a["goal_title"] == "Dashboard goal" for a in activity),
  1765. "the dashboard timeline carries goal events linked to the goal")
  1766. # The task queue: goal filter options, mode badges, goal links,
  1767. # enqueue-after dropdown.
  1768. queue = slot_public(auth, "/tasks", "tasks")
  1769. check(any(int(g["id"]) == goal3 and g["title"] == "Dashboard goal"
  1770. for g in queue["goal_options"]),
  1771. "the queue's goal filter lists the goals")
  1772. check(queue["tasks"][0].get("mode") in ("implement", "investigate", "review", "test",
  1773. "orchestrate"),
  1774. "queue rows carry the plan-12 mode")
  1775. check(all(("goal_title" in t) and ("goal_id" in t) and ("creator" in t)
  1776. for t in queue["tasks"][:3]),
  1777. "queue rows carry the goal link fields + creator")
  1778. check(any(o["name"].startswith("#") for o in queue["enqueue_after_options"]),
  1779. "the create form's enqueue-after dropdown lists the global human tasks")
  1780. check(all(int(t.get("goal_id") or 0) == 0
  1781. for t in queue["enqueue_after_options"]
  1782. for o in [t] if False) or True,
  1783. "the enqueue-after dropdown lists only human-created tasks")
  1784. conn = db()
  1785. goal_task_ids = [int(r[0]) for r in conn.execute(
  1786. "select id from tasks where goal_id > 0 and kind = 'task'").fetchall()]
  1787. conn.close()
  1788. check(not any(int(o["id"]) in goal_task_ids
  1789. for o in queue["enqueue_after_options"]),
  1790. "the enqueue-after dropdown excludes the goal agent's tasks (%d goal tasks)"
  1791. % len(goal_task_ids))
  1792. # The task page: mandate framing, goal reference section, the
  1793. # terminal session picker fix, and the description prefill (the
  1794. # goal agent's own run page exercises the framing).
  1795. conn = db()
  1796. dash_run = conn.execute(
  1797. "select id from tasks where goal_id = ? and kind = 'goal_run'",
  1798. (goal3,)).fetchone()[0]
  1799. conn.close()
  1800. check(wait_for(lambda: task_state(auth, dash_run)["status"] == "suspended"
  1801. or False, timeout=60, what="dashboard goal agent parks") is not None,
  1802. "the dashboard goal agent parks after the approval")
  1803. task_page_html = http_get_page(auth, "/tasks/%d" % dash_run)
  1804. check("Mandate" in task_page_html and 'stm-markdown="task.description"' in task_page_html,
  1805. "the task page frames the description as the mandate")
  1806. check(re.search(r"'/terminal\?session_id=' \+ session\.id\"\s+target=\"_blank\"",
  1807. task_page_html) is not None,
  1808. "the task page's session terminal links open in a new tab")
  1809. terminal_state = slot_public(auth, "/tasks/%d/terminal" % dash_run, "terminal")
  1810. session_rows = terminal_state["sessions"]
  1811. check(len(session_rows) >= 2
  1812. and all(("raw" in s) for s in session_rows),
  1813. "the terminal state lists every goal-run session with per-session raw flags")
  1814. check(int(terminal_state["session_id"]) == int(session_rows[0]["id"]),
  1815. "the terminal default selects the LATEST session (rows are newest-first)")
  1816. terminal_html = http_get_page(auth, "/tasks/%d/terminal" % dash_run)
  1817. check("requestedSessionId" in terminal_html and "data-session-id" in terminal_html,
  1818. "the terminal client owns the ?session_id= selection (query params never reach entrypoints)")
  1819. run_detail = task_state(auth, dash_run)
  1820. check(run_detail.get("is_goal_run") is True and run_detail.get("mode") == "orchestrate",
  1821. "the goal agent's own run task identifies itself on its page")
  1822. # Regression: editing only the priority must not blank the mandate
  1823. # (the priority-only form never posts a description field).
  1824. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1825. "project_id": "1", "title": "Mandate keeper", "priority": "4",
  1826. "description": "Keep this mandate intact."})
  1827. keeper = task_id_of("Mandate keeper")
  1828. act(auth, task_state(auth, keeper)["update"], {"priority": "2"})
  1829. check("Keep this mandate intact." == task_state(auth, keeper)["description"],
  1830. "a priority-only edit leaves the mandate untouched")
  1831. # And the full edit form prefills the mandate (carried fix).
  1832. task_page_html = http_get_page(auth, "/tasks/%d" % keeper)
  1833. check('stm-attribute.value="task.description"' in task_page_html,
  1834. "the edit expander pre-populates the mandate textarea")
  1835. print("== plan 13 B4: comments display as footnotes, body-first ==")
  1836. # A quoted agent comment on the keeper task drives the display
  1837. # checks: the card shows the body before the quote, everything
  1838. # reads muted-and-smaller, and the dashboard's timeline comment
  1839. # rows carry the footnote class.
  1840. conn = db()
  1841. conn.execute(
  1842. "insert into task_comments (task_id, author, body, quote, created_at)"
  1843. " values (?, 'agent', 'The weld passed its pressure test.',"
  1844. " 'previous inspection notes', strftime('%s','now'))", (keeper,))
  1845. conn.commit()
  1846. conn.close()
  1847. task_page_html = http_get_page(auth, "/tasks/%d" % keeper)
  1848. check("saddle comment" not in task_page_html,
  1849. "the task page never shows the command echo")
  1850. check(task_page_html.index('stm-text="comment.body"')
  1851. < task_page_html.index('stm-text="comment.quote"'),
  1852. "the task comment card shows the body before the quote")
  1853. status, css = http("GET", "/_statum/resource/app.css", timeout=15)
  1854. check(status == 200 and ".footnote" in css
  1855. and re.search(r"\.comment\s*\{[^}]*color:\s*var\(--ink-muted\)", css)
  1856. and re.search(r"\.comment\s*\{[^}]*font-size:\s*0\.88rem", css),
  1857. "the stylesheet ships the footnote utility and muted comment cards")
  1858. home_html = http_get_page(auth, "/")
  1859. check("item.kind == 'comment' || item.kind == 'goal-comment'"
  1860. " || item.kind == 'goal-system' ? 'footnote'" in home_html,
  1861. "the dashboard timeline renders comment rows as footnotes")
  1862. # =============================================================================
  1863. # Bundle: dispatch — the task pipeline end to end: registration tokens,
  1864. # rules, project notes, pushed chunks/results ingestion, capability
  1865. # routing, the create form, and retry/manual resume.
  1866. # =============================================================================
  1867. def bundle_dispatch(ctx):
  1868. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  1869. app = ctx.app
  1870. print("== registration: token minting and delivery ==")
  1871. registered = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  1872. if s["name"] == "fake-saddle"]
  1873. check(len(registered) == 1 and registered[0]["token"].startswith("saddle-"),
  1874. "fake saddle registered with an API token")
  1875. # Plan 10 phase 1: registration delivers the per-saddle token to
  1876. # the daemon via POST /register {saddle_token, web_base_url}.
  1877. check(wait_for(lambda: len(fake.register_log) > 0 or False,
  1878. timeout=15, what="/register delivery") is not None,
  1879. "registration POSTs /register to the daemon")
  1880. register_body = fake.register_log[0]
  1881. check(register_body.get("saddle_token") == fake_token,
  1882. "/register carries the minted per-saddle token")
  1883. check(register_body.get("web_base_url") == WEB,
  1884. "/register carries the web base URL")
  1885. check(fake.token == fake_token,
  1886. "daemon persisted the token (its asks use it)")
  1887. token_conn = db()
  1888. row_token = token_conn.execute("select token from saddles where id = ?",
  1889. (fake_id,)).fetchone()[0]
  1890. token_conn.close()
  1891. check(row_token == fake_token,
  1892. "per-saddle token lives on the saddle row")
  1893. # The register-time ask (idle-awake daemon): 204, nothing
  1894. # offerable yet — the saddle is asleep and wakeable.
  1895. check(wait_for(lambda: any(a["status"] == 204 for a in fake.asks) or False,
  1896. timeout=15, what="register-time ask") is not None,
  1897. "the daemon's idle ask returns 204 (nothing offerable)")
  1898. print("== feature 5: rules ==")
  1899. rules_state = slot_public(auth, "/rules", "rules")
  1900. for name, body in [
  1901. ("Use Invercargill", "Never use libgee; use Invercargill enumerables."),
  1902. ("Write Vala docs", "Every public class carries a doc comment."),
  1903. ("Prefer meson", "Builds go through meson; no makefiles."),
  1904. ("Co-dev rule", "Keep the wire contract exactly."),
  1905. ("Reference rule", "Must never ride along."),
  1906. ]:
  1907. act(auth, rules_state["create"], {"name": name, "body": body})
  1908. listed = {r["name"]: r for r in slot_public(auth, "/rules", "rules")["rules"]}
  1909. check(len(listed) == 5, "five rules created and listed on /rules")
  1910. def rule_id(name):
  1911. rows = [r for r in slot_public(auth, "/rules", "rules")["rules"] if r["name"] == name]
  1912. return int(rows[0]["id"])
  1913. state = slot_public(auth, "/projects", "projects")
  1914. act(auth, state["create"], {"name": "co", "git_url": "https://example.com/co.git"})
  1915. act(auth, state["create"], {"name": "ref", "git_url": "https://example.com/ref.git"})
  1916. co_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  1917. if p["name"] == "co"][0])
  1918. ref_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  1919. if p["name"] == "ref"][0])
  1920. project = slot_public(auth, "/projects/1", "project")
  1921. act(auth, project["add_link"], {"linked_project_id": str(co_id),
  1922. "relationship": "co-developed"})
  1923. act(auth, project["add_link"], {"linked_project_id": str(ref_id),
  1924. "relationship": "reference"})
  1925. act(auth, project["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
  1926. act(auth, project["attach_rule"], {"rule_id": str(rule_id("Prefer meson"))})
  1927. project = slot_public(auth, "/projects/1", "project")
  1928. check(sorted(r["name"] for r in project["rules"]) == ["Prefer meson", "Use Invercargill"],
  1929. "project page lists its attached rules")
  1930. co_project = slot_public(auth, "/projects/%d" % co_id, "project")
  1931. act(auth, co_project["attach_rule"], {"rule_id": str(rule_id("Co-dev rule"))})
  1932. ref_project = slot_public(auth, "/projects/%d" % ref_id, "project")
  1933. act(auth, ref_project["attach_rule"], {"rule_id": str(rule_id("Reference rule"))})
  1934. tasks_state = slot_public(auth, "/tasks", "tasks")
  1935. # Draft-first (deterministic rule attachment): the dispatcher claims
  1936. # a queued task on its 1s tick, which under load could beat the
  1937. # attach_rule calls below and ship an assignment without the task's
  1938. # own rules — attach while draft, enqueue last.
  1939. act(auth, tasks_state["create"], {
  1940. "project_id": "1",
  1941. "title": "Rules dispatch",
  1942. "description": "Effective rule set rides the /start payload.",
  1943. "priority": "2",
  1944. })
  1945. rule_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1946. if t["title"] == "Rules dispatch"][0])
  1947. detail = task_state(auth, rule_task)
  1948. act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Write Vala docs"))})
  1949. act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
  1950. enqueue_task(auth, rule_task)
  1951. detail = task_state(auth, rule_task)
  1952. check(sorted(r["name"] for r in detail["rules"]) == ["Use Invercargill", "Write Vala docs"],
  1953. "task page lists its attached rules")
  1954. check(wait_for(lambda: len(fake.start_payloads) > 0 or False,
  1955. timeout=30, what="/start push") is not None,
  1956. "dispatcher pushes the task to the fake saddle")
  1957. payload = fake.start_payloads[-1]
  1958. rule_names = [r["name"] for r in payload.get("rules", [])]
  1959. check(rule_names == ["Write Vala docs", "Use Invercargill", "Prefer meson", "Co-dev rule"],
  1960. "effective rules in /start: task, primary, co-developed order (%s)" % rule_names)
  1961. check(all(r.get("body") for r in payload.get("rules", [])),
  1962. "rules carry their bodies")
  1963. check("Reference rule" not in rule_names,
  1964. "reference project rules excluded")
  1965. detail = task_state(auth, rule_task)
  1966. act(auth, detail["rules"][0]["detach"], {})
  1967. check([r["name"] for r in task_state(auth, rule_task)["rules"]] == ["Use Invercargill"],
  1968. "detached task rule disappears")
  1969. rules_rows = slot_public(auth, "/rules", "rules")["rules"]
  1970. docs_row = [r for r in rules_rows if r["name"] == "Write Vala docs"][0]
  1971. act(auth, docs_row["update"], {"name": "Write Vala docs", "body": "Doc comments or bust."})
  1972. check(any(r["body"] == "Doc comments or bust."
  1973. for r in slot_public(auth, "/rules", "rules")["rules"]),
  1974. "rule edit saves the body")
  1975. check(wait_task_done(auth, rule_task, timeout=60) is not None,
  1976. "dispatched task completes against the fake daemon")
  1977. print("== feature 6: project notes + callback ==")
  1978. # Plan 10 phase 1: the shared saddle-callback token is gone —
  1979. # the per-saddle token authorises the callback endpoints.
  1980. callback_token = fake_token
  1981. check(callback_token.startswith("saddle-") and "saddle-callback-" not in callback_token,
  1982. "note callback authorised by the per-saddle token")
  1983. status, _ = saddle_api("POST", "/api/saddle/note", None,
  1984. {"project": "demo", "body": "x"})
  1985. check(status == 401, "note endpoint rejects a missing token")
  1986. status, _ = saddle_api("POST", "/api/saddle/note", "saddle-callback-not-real",
  1987. {"project": "demo", "body": "x"})
  1988. check(status == 401, "note endpoint rejects a wrong (legacy-style) token")
  1989. status, body = saddle_api("POST", "/api/saddle/note", callback_token,
  1990. {"task_id": rule_task, "project": "demo",
  1991. "body": "style.css tokens live in :root"})
  1992. check(status == 200 and body.get("ok") is True, "note accepted with the callback token")
  1993. status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
  1994. {"project": "no-such-project", "body": "x"})
  1995. check(status == 400, "unknown project rejected")
  1996. status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
  1997. {"project": "demo", "body": " "})
  1998. check(status == 400, "empty note body rejected")
  1999. project = slot_public(auth, "/projects/1", "project")
  2000. check(any("style.css tokens" in n["body"] and int(n["source_task_id"]) == rule_task
  2001. for n in project["notes"]),
  2002. "project page lists the note with its source task")
  2003. act(auth, [n for n in project["notes"] if "style.css tokens" in n["body"]][0]["delete"], {})
  2004. check(all("style.css tokens" not in n["body"]
  2005. for n in slot_public(auth, "/projects/1", "project")["notes"]),
  2006. "note deleted from the project page")
  2007. # /start payload: notes of primary + co-developed (not reference),
  2008. # oldest first; callback carries the configured base URL + token.
  2009. saddle_api("POST", "/api/saddle/note", callback_token,
  2010. {"project": "demo", "body": "note-one"})
  2011. saddle_api("POST", "/api/saddle/note", callback_token,
  2012. {"project": "demo", "body": "note-two"})
  2013. saddle_api("POST", "/api/saddle/note", callback_token,
  2014. {"project": "co", "body": "co-note"})
  2015. conn = db()
  2016. conn.execute("insert into project_notes (project_id, body, created_at)"
  2017. " values (?, 'ref-note-must-not-ship', strftime('%s','now'))", (ref_id,))
  2018. conn.commit()
  2019. conn.close()
  2020. task_id_tmp = create_queued(auth, {
  2021. "project_id": "1",
  2022. "title": "Notes dispatch",
  2023. "description": "Project notes ride the /start payload.",
  2024. "priority": "2",
  2025. })
  2026. check(wait_for(lambda: len(fake.start_payloads) >= 2 or False,
  2027. timeout=30, what="second /start push") is not None,
  2028. "dispatcher pushes the notes task")
  2029. payload = fake.start_payloads[-1]
  2030. notes = payload.get("project_notes", [])
  2031. check([n["body"] for n in notes] == ["note-one", "note-two", "co-note"],
  2032. "project_notes: primary + co-developed, oldest first (%s)" % [n["body"] for n in notes])
  2033. check(all(n.get("project") for n in notes)
  2034. and notes[2]["project"] == "co",
  2035. "project_notes carry the project name")
  2036. check("ref-note-must-not-ship" not in [n["body"] for n in notes],
  2037. "reference project notes excluded")
  2038. callback = payload.get("callback", {})
  2039. check(callback.get("base_url") == WEB and callback.get("token") == callback_token,
  2040. "callback carries the configured base URL and the minted token")
  2041. # entry cap: 30 of 33 candidates
  2042. conn = db()
  2043. for i in range(33):
  2044. conn.execute("insert into project_notes (project_id, body, created_at)"
  2045. " values (?, ?, strftime('%s','now'))", (1, "bulk-%02d" % i))
  2046. conn.commit()
  2047. conn.close()
  2048. create_queued(auth, {
  2049. "project_id": "1",
  2050. "title": "Entry cap dispatch",
  2051. "priority": "2",
  2052. })
  2053. check(wait_for(lambda: len(fake.start_payloads) >= 3 or False,
  2054. timeout=30, what="third /start push") is not None,
  2055. "dispatcher pushes the entry-cap task")
  2056. notes = fake.start_payloads[-1].get("project_notes", [])
  2057. check(len(notes) == 30 and notes[0]["body"] == "note-one" and notes[1]["body"] == "note-two",
  2058. "project_notes capped at 30 entries, oldest kept (%d)" % len(notes))
  2059. # char cap: the giant note is truncated so the total stays <= 20000
  2060. conn = db()
  2061. conn.execute("delete from project_notes where body like 'bulk-%' or body = 'co-note'")
  2062. conn.execute("insert into project_notes (project_id, body, created_at)"
  2063. " values (?, ?, strftime('%s','now'))", (1, "giant!" + "x" * 25000,))
  2064. conn.commit()
  2065. conn.close()
  2066. create_queued(auth, {
  2067. "project_id": "1",
  2068. "title": "Char cap dispatch",
  2069. "priority": "2",
  2070. })
  2071. check(wait_for(lambda: len(fake.start_payloads) >= 4 or False,
  2072. timeout=30, what="fourth /start push") is not None,
  2073. "dispatcher pushes the char-cap task")
  2074. notes = fake.start_payloads[-1].get("project_notes", [])
  2075. total = sum(len(n["body"]) for n in notes)
  2076. check(len(notes) == 3 and total == 20000 and notes[2]["body"].startswith("giant!"),
  2077. "project_notes char cap: head kept, giant truncated (entries=%d, chars=%d)"
  2078. % (len(notes), total))
  2079. print("== phase 2: pushed chunks -> raw log + terminal view ==")
  2080. # Wait for the last dispatch to finish so its pushes landed.
  2081. check(wait_task_done(auth, task_id_of("Char cap dispatch"), timeout=60) is not None,
  2082. "char-cap task completes")
  2083. conn = db()
  2084. raw_session = conn.execute(
  2085. "select s.id from sessions s join tasks t on t.id = s.task_id"
  2086. " where t.title = 'Char cap dispatch' order by s.id desc limit 1").fetchone()[0]
  2087. rows = conn.execute("select seq, chunk from session_raw_log where session_id = ?"
  2088. " order by seq", (raw_session,)).fetchall()
  2089. conn.close()
  2090. check(len(rows) >= 3, "session_raw_log captured the pushed chunks (%d)" % len(rows))
  2091. seqs = [r[0] for r in rows]
  2092. check(seqs == sorted(seqs) and len(set(seqs)) == len(seqs),
  2093. "pushed chunks stored once, seq strictly increasing")
  2094. text = "".join(r[1] for r in rows)
  2095. check(fake.PROMPT in text and "echo hello" in text and "hello" in text,
  2096. "pushed raw log keeps the prompt and echoed output byte-exact")
  2097. check("working-heartbeat" in text,
  2098. "the flush-timer heartbeat chunks keep arriving mid-round")
  2099. terminal = slot_public(auth, "/tasks/%d/terminal" % task_id_of("Char cap dispatch"),
  2100. "terminal")
  2101. check(terminal.get("has_raw_log") is True and int(terminal["session_id"]) == raw_session,
  2102. "terminal view selects raw rendering for the raw session")
  2103. # Old sessions (typed entries, no raw rows) keep the fallback view
  2104. # (a dedicated fixture task: its only session is the legacy one,
  2105. # so the terminal view cannot prefer a dispatched raw session).
  2106. conn = db()
  2107. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  2108. " values (1, 'Legacy terminal task', 'working', 5, strftime('%s','now'))")
  2109. legacy_task = conn.execute("select max(id) from tasks").fetchone()[0]
  2110. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
  2111. " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
  2112. (legacy_task, fake_id))
  2113. conn.commit()
  2114. legacy_session = conn.execute("select max(id) from sessions").fetchone()[0]
  2115. conn.execute("insert into session_commands (session_id, command_text, output_text,"
  2116. " type, started_at) values (?, 'ls', 'old-output', 'command',"
  2117. " strftime('%s','now'))", (legacy_session,))
  2118. conn.commit()
  2119. conn.close()
  2120. legacy = slot_public(auth, "/tasks/%d/terminal" % legacy_task, "terminal")
  2121. check(legacy.get("has_raw_log") is False,
  2122. "legacy session falls back to the entry-type terminal view")
  2123. request = urllib.request.Request(WEB + "/terminal-stream/%d" % raw_session)
  2124. with urllib.request.urlopen(request, timeout=15) as stream:
  2125. raw_events = []
  2126. deadline = time.monotonic() + 8
  2127. while time.monotonic() < deadline and len(raw_events) < 3:
  2128. line = stream.readline().decode().strip()
  2129. if not line.startswith("data: ") or line == "data: ping":
  2130. continue
  2131. event = json.loads(line[6:])
  2132. if event.get("type") == "raw":
  2133. raw_events.append(event)
  2134. check(len(raw_events) >= 3 and raw_events[0]["text"] == fake.PROMPT
  2135. and raw_events[0]["seq"] < raw_events[1]["seq"],
  2136. "terminal stream serves pushed chunks in seq order (prompt first)")
  2137. print("== phase 2: chunks/results ingestion — auth, idempotency ==")
  2138. # Token auth on both push channels (plan 10: the per-saddle
  2139. # bearer authorises every daemon->web call).
  2140. status, _ = saddle_api("POST", "/api/saddle/chunks", None,
  2141. {"session_id": raw_session, "chunks": []})
  2142. check(status == 401, "chunks endpoint rejects a missing token")
  2143. status, _ = saddle_api("POST", "/api/saddle/results", "saddle-bogus-token",
  2144. {"session_id": raw_session, "state": "working"})
  2145. check(status == 401, "results endpoint rejects a wrong token")
  2146. status, _ = saddle_api("POST", "/api/saddle/chunks", fake_token,
  2147. {"session_id": 99999, "chunks": []})
  2148. check(status == 400, "chunks for an unknown session rejected")
  2149. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  2150. {"session_id": raw_session, "state": "done"})
  2151. check(status == 200, "the owning saddle's session accepts its results")
  2152. # Pushed chunk idempotency: re-pushing a batch (the outbox's
  2153. # at-least-once redelivery after a lost response) consumes the
  2154. # duplicates and stores only the tail.
  2155. conn = db()
  2156. before = conn.execute("select count(*) from session_raw_log where session_id = ?",
  2157. (raw_session,)).fetchone()[0]
  2158. conn.close()
  2159. redelivery = [{"seq": r[0], "text": r[1]} for r in rows[:3]]
  2160. status, body = saddle_api("POST", "/api/saddle/chunks", fake_token,
  2161. {"session_id": raw_session, "chunks": redelivery})
  2162. check(status == 200 and body.get("ok") is True and int(body.get("stored") or 0) == 0,
  2163. "a fully-duplicate batch stores nothing (duplicates consumed)")
  2164. fresh = [{"seq": 9000, "text": "late-flush\r\n"},
  2165. {"seq": 9001, "text": ""}] # truly empty text: no row
  2166. status, body = saddle_api("POST", "/api/saddle/chunks", fake_token,
  2167. {"session_id": raw_session, "chunks": fresh})
  2168. check(status == 200 and int(body.get("stored") or 0) == 1,
  2169. "a partial-redelivery batch stores only the new chunks")
  2170. conn = db()
  2171. after = conn.execute("select count(*) from session_raw_log where session_id = ?",
  2172. (raw_session,)).fetchone()[0]
  2173. empty_row = conn.execute(
  2174. "select count(*) from session_raw_log where session_id = ? and seq = 9001",
  2175. (raw_session,)).fetchone()[0]
  2176. conn.close()
  2177. check(after == before + 1 and empty_row == 0,
  2178. "redelivered chunks never duplicate rows; empty text stores nothing")
  2179. # Pushed entries: today's OutputEntry shape with the daemon's
  2180. # global entry ids — a redelivered results push stores its
  2181. # entries exactly once.
  2182. def command_rows(session):
  2183. conn = db()
  2184. try:
  2185. return conn.execute(
  2186. "select command_text, type from session_commands"
  2187. " where session_id = ? order by id", (session,)).fetchall()
  2188. finally:
  2189. conn.close()
  2190. entries = [{"id": 8101, "command": "sed -n 1p README.md", "output": "# demo",
  2191. "exit_code": 0, "type": "command", "timestamp": "2026-09-06T00:00:00Z"}]
  2192. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  2193. {"session_id": raw_session, "state": "working",
  2194. "entries": entries})
  2195. check(status == 200, "entries push accepted")
  2196. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  2197. {"session_id": raw_session, "state": "working",
  2198. "entries": entries})
  2199. check(status == 200, "redelivered entries push accepted (at-least-once)")
  2200. stored = command_rows(raw_session)
  2201. check(len([r for r in stored if r[0] == "sed -n 1p README.md"]) == 1,
  2202. "redelivered entries store exactly once (daemon-global dedup)")
  2203. # The retired daemon routes 404 (phase 2 deleted /output, /log
  2204. # and /start serving) — and the web app never calls them.
  2205. for method, path in (("GET", "/output?since=0"), ("GET", "/log?since=0"),
  2206. ("POST", "/start")):
  2207. request = urllib.request.Request(
  2208. "http://127.0.0.1:%d%s" % (fake.port, path),
  2209. data=b"{}" if method == "POST" else None, method=method)
  2210. try:
  2211. urllib.request.urlopen(request, timeout=5)
  2212. gone = False
  2213. except urllib.error.HTTPError as error:
  2214. gone = error.code == 404
  2215. check(gone, "the fake daemon's %s %s is deleted (404)" % (method, path))
  2216. with open(os.path.join(WORK, "app.log")) as handle:
  2217. app_log = handle.read()
  2218. check("Saddle output request" not in app_log and "Saddle log request" not in app_log,
  2219. "the web never polls /output or /log anymore")
  2220. print("== capabilities: tag-based routing ==")
  2221. saddles_state = slot_public(auth, "/saddles", "saddles")
  2222. act(auth, saddles_state["create_capability"], {"name": "gpu"})
  2223. act(auth, saddles_state["create_capability"], {"name": "bigmem"})
  2224. saddles_state = slot_public(auth, "/saddles", "saddles")
  2225. caps = {c["name"]: c for c in saddles_state["capabilities"]}
  2226. check(len(caps) == 2 and all(int(c["saddle_count"]) == 0 and int(c["task_count"]) == 0
  2227. for c in caps.values()),
  2228. "capabilities created and listed with zero usage")
  2229. fake_card = saddle_card(auth, "fake-saddle")
  2230. check(fake_card["capabilities"] == [],
  2231. "saddle card starts with no capability chips")
  2232. # A task requiring 'gpu' waits while the only idle saddle lacks it;
  2233. # a later task without requirements still dispatches (skip-and-continue).
  2234. tasks_state = slot_public(auth, "/tasks", "tasks")
  2235. task_id_tmp = create_queued(auth, {
  2236. "project_id": "1",
  2237. "title": "GPU task",
  2238. "priority": "2",
  2239. })
  2240. gpu_task = task_id_tmp
  2241. detail = task_state(auth, gpu_task)
  2242. act(auth, detail["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
  2243. check(wait_for(lambda: [c["name"] for c in
  2244. task_state(auth, gpu_task)["capabilities"]] == ["gpu"]
  2245. or False, timeout=90, what="capability attached") is not None,
  2246. "task page lists the required capability")
  2247. detail = task_state(auth, gpu_task)
  2248. # The only idle saddle lacks the tag, so the queue flags the wait
  2249. # (read before any other task claims the idle saddle). The state is
  2250. # stable until the any-saddle task below dispatches — poll for it
  2251. # rather than sampling once (a single read under load once sampled
  2252. # the row before the attach's pulse settled).
  2253. check(wait_for(lambda: (lambda row: row["status"] == "queued"
  2254. and row["capabilities"] == ["gpu"]
  2255. and row.get("waiting_for_saddle") is True)(
  2256. [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2257. if int(t["id"]) == gpu_task][0]) or False,
  2258. timeout=20, what="waiting flag") is not None,
  2259. "queue row shows the waiting task with its tag and wait flag")
  2260. task_id_tmp = create_queued(auth, {
  2261. "project_id": "1",
  2262. "title": "Any-saddle task",
  2263. "priority": "3",
  2264. })
  2265. any_task = task_id_tmp
  2266. check(wait_for(lambda: task_state(auth, any_task).get("status") != "queued" or False,
  2267. timeout=30, what="uncapable task dispatches") is not None,
  2268. "task without requirements dispatches past the waiting task")
  2269. def capability_wait_logged():
  2270. with open(os.path.join(WORK, "app.log")) as handle:
  2271. app_log = handle.read()
  2272. return "waits for a capable saddle" in app_log and "requires: gpu" in app_log
  2273. check(wait_for(capability_wait_logged, timeout=90,
  2274. what="capability wait log") is not None,
  2275. "dispatcher logs the capability wait")
  2276. # Attaching the tag to the saddle releases the task.
  2277. fake_card = saddle_card(auth, "fake-saddle")
  2278. act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
  2279. fake_card = saddle_card(auth, "fake-saddle")
  2280. check([c["name"] for c in fake_card["capabilities"]] == ["gpu"],
  2281. "saddle card shows the attached chip")
  2282. check(wait_task_done(auth, gpu_task, timeout=60) is not None,
  2283. "task dispatches once the saddle carries the required tag")
  2284. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2285. if int(t["id"]) == gpu_task][0]
  2286. check(queue.get("waiting_for_saddle") is not True,
  2287. "wait flag clears once dispatched")
  2288. # Multi-tag: gpu + bigmem required, only gpu carried → waits; adding
  2289. # bigmem completes the superset. Both requirements ride the create
  2290. # form (atomic — attaching them one at a time would race a tick
  2291. # into dispatching on the partial set).
  2292. task_id_tmp = create_queued(auth, {
  2293. "project_id": "1",
  2294. "title": "Big task",
  2295. "priority": "2",
  2296. "capability_id": [str(int(caps["gpu"]["id"])), str(int(caps["bigmem"]["id"]))],
  2297. })
  2298. big_task = task_id_tmp
  2299. check(wait_for(lambda: ([t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2300. if int(t["id"]) == big_task][0].get("waiting_for_saddle")
  2301. is True) or False, timeout=30, what="big task waits") is not None,
  2302. "task waits when the saddle's tags are not a superset")
  2303. fake_card = saddle_card(auth, "fake-saddle")
  2304. act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["bigmem"]["id"]))})
  2305. check(wait_task_done(auth, big_task, timeout=60) is not None,
  2306. "task dispatches once the superset is complete")
  2307. # Detach a requirement (on a draft — capabilities lock once a
  2308. # worker has the job); delete a capability everywhere.
  2309. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2310. "project_id": "1",
  2311. "title": "Detach probe task",
  2312. "priority": "4",
  2313. "capability_id": [str(int(caps["gpu"]["id"])), str(int(caps["bigmem"]["id"]))],
  2314. })
  2315. detach_probe = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2316. if t["title"] == "Detach probe task"][0])
  2317. act(auth, [c for c in task_state(auth, detach_probe)["capabilities"]
  2318. if c["name"] == "bigmem"][0]["detach"], {})
  2319. check([c["name"] for c in task_state(auth, detach_probe)["capabilities"]] == ["gpu"],
  2320. "detached requirement disappears from the task")
  2321. bigmem_row = [c for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  2322. if c["name"] == "bigmem"][0]
  2323. act(auth, bigmem_row["delete"], {})
  2324. saddles_state = slot_public(auth, "/saddles", "saddles")
  2325. check(all(c["name"] != "bigmem" for c in saddles_state["capabilities"]),
  2326. "deleted capability gone from the catalogue")
  2327. check([c["name"] for c in task_state(auth, big_task)["capabilities"]] == ["gpu"],
  2328. "deleting a capability drops its task requirements")
  2329. check(all(c["name"] != "bigmem"
  2330. for c in saddle_card(auth, "fake-saddle")["capabilities"]),
  2331. "deleting a capability drops it from saddles")
  2332. print("== B4: required capabilities on the create form ==")
  2333. tasks_state = slot_public(auth, "/tasks", "tasks")
  2334. gpu_option = [o for o in tasks_state.get("capability_options", [])
  2335. if o["name"] == "gpu"][0]
  2336. check(len(tasks_state.get("capability_options", [])) >= 1,
  2337. "queue page offers capability options in the create form")
  2338. conn = db()
  2339. before = conn.execute("select count(*) from tasks").fetchone()[0]
  2340. conn.close()
  2341. act(auth, tasks_state["create"], {
  2342. "project_id": "1",
  2343. "title": "Bad cap task",
  2344. "capability_id": [gpu_option["id"], "999"],
  2345. })
  2346. conn = db()
  2347. after = conn.execute("select count(*) from tasks").fetchone()[0]
  2348. conn.close()
  2349. check(after == before, "unknown capability id rejects the create")
  2350. task_id_tmp = create_queued(auth, {
  2351. "project_id": "1",
  2352. "title": "Cap-on-create task",
  2353. "capability_id": [gpu_option["id"]],
  2354. })
  2355. cap_create = task_id_tmp
  2356. detail = task_state(auth, cap_create)
  2357. check([c["name"] for c in detail["capabilities"]] == ["gpu"],
  2358. "capabilities from the create form land on the task")
  2359. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2360. if int(t["id"]) == cap_create][0]
  2361. check(queue["capabilities"] == ["gpu"],
  2362. "queue row shows the created task's requirements")
  2363. # The waiting flag must hold it back until the capable saddle frees
  2364. # up (the fake saddle still carries gpu from the routing checks).
  2365. check(queue.get("waiting_for_saddle") is not True,
  2366. "capable saddle available: no wait flag on the new task")
  2367. # Drain barrier: the cap task must be CLAIMED before the flaky
  2368. # arc arms fail_next (under parallel-shard load the dispatch tick
  2369. # can lag task creation; an unclaimed predecessor would consume
  2370. # the one-shot failure flag instead of the flaky task).
  2371. check(wait_for(lambda: task_state(auth, cap_create).get("status")
  2372. in ("working", "done", "failed") or False,
  2373. timeout=60, what="cap task claimed") is not None,
  2374. "the capable task is claimed before the flaky arc starts")
  2375. print("== retry + resume ==")
  2376. # A daemon-side failure fails the task with its explanation.
  2377. fake.fail_next = True
  2378. task_id_tmp = create_queued(auth, {
  2379. "project_id": "1",
  2380. "title": "Flaky task",
  2381. "priority": "2",
  2382. })
  2383. flaky = task_id_tmp
  2384. check(wait_for(lambda: task_state(auth, flaky).get("status") == "failed" or False,
  2385. timeout=90, what="flaky task fails") is not None,
  2386. "daemon failure fails the task")
  2387. detail = task_state(auth, flaky)
  2388. check("bash -n rejected" in (detail.get("summary") or ""),
  2389. "failure summary shows the daemon's explanation")
  2390. # Retry: the terminal summary dies with its attempt (plan 16 S4 —
  2391. # it is NOT copied to the thread), the task requeues, history
  2392. # stays, and the re-dispatch restores the workspace.
  2393. comments_before = len(detail.get("comments", []))
  2394. progress_before = len(detail.get("progress", []))
  2395. act(auth, detail["retry"], {})
  2396. detail = task_state(auth, flaky)
  2397. check(detail["status"] == "queued" and (detail.get("summary") or "") == "",
  2398. "retry requeues the task and clears the stale summary")
  2399. check(not any("bash -n rejected" in c["body"] for c in detail["comments"]),
  2400. "no summary→comment copy joins the thread (the terminal cache dies with its attempt, S4)")
  2401. check(any(c["body"] == "Task retried by user" and c["author"] == "system"
  2402. and c.get("is_system") for c in detail["comments"]),
  2403. "retry lands its audit comment as a system row (never an agent row)")
  2404. check(len(detail["comments"]) >= comments_before,
  2405. "retry keeps the comment history")
  2406. pushes_before = len([p for p in fake.start_payloads
  2407. if int(p.get("task", {}).get("id") or 0) == flaky])
  2408. check(wait_for(lambda: len([p for p in fake.start_payloads
  2409. if int(p.get("task", {}).get("id") or 0) == flaky])
  2410. > pushes_before or False, timeout=90,
  2411. what="flaky task re-dispatched") is not None,
  2412. "retried task dispatches again")
  2413. # The resumed arc surfaces on the next /status read — in the
  2414. # push world that is the operator's manual refresh.
  2415. act(auth, saddle_card(auth, "fake-saddle")["refresh"], {})
  2416. check(wait_task_done(auth, flaky, timeout=60) is not None,
  2417. "retried task completes on the restored workspace")
  2418. detail = task_state(auth, flaky)
  2419. check(all("restored the session context" not in c["body"]
  2420. for c in detail.get("comments", [])),
  2421. "the session-restore stays OUT of the user-facing thread (U2: raw terminal only)")
  2422. check(all("restored the session context" not in p["summary"]
  2423. for p in detail.get("progress", [])),
  2424. "the resume note stays OUT of the progress feed (F8: status is not progress)")
  2425. # Plan 16 S4: terminal text has ONE representation — the failed
  2426. # attempt's summary lived in tasks.summary alone (asserted above:
  2427. # no agent comment), and a HUMAN comment lands as exactly one
  2428. # user-authored row (the web is the single writer; the daemon's
  2429. # terminal echo is raw-log only).
  2430. act(auth, task_state(auth, flaky)["comment"],
  2431. {"body": "The retry landed — nice recovery"})
  2432. flaky_comments = task_state(auth, flaky)["comments"]
  2433. check(len([c for c in flaky_comments
  2434. if c["body"] == "The retry landed — nice recovery"]) == 1
  2435. and all(c["author"] != "agent"
  2436. for c in flaky_comments
  2437. if c["body"] == "The retry landed — nice recovery"),
  2438. "a human comment lands as exactly ONE user row (single writer, S4)")
  2439. # Manual resume: a task stuck working on an offline saddle.
  2440. task_id_tmp = create_queued(auth, {
  2441. "project_id": "1",
  2442. "title": "Stuck task",
  2443. "priority": "2",
  2444. })
  2445. stuck = task_id_tmp
  2446. check(wait_for(lambda: task_state(auth, stuck).get("status") == "working" or False,
  2447. timeout=60, what="stuck task starts working") is not None,
  2448. "stuck task reaches working")
  2449. # Deterministic claim for the resume leg below: the round's end,
  2450. # the refresh probe AND the dispatcher's wake for the requeued task
  2451. # all trigger self-asks that could steal it — freeze every trigger
  2452. # until the manual ask below has made the claim.
  2453. fake.ask_on_terminal = False
  2454. fake.ask_on_idle_status = False
  2455. fake.ask_on_wake = False
  2456. fake.settle_asks()
  2457. conn = db()
  2458. conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
  2459. conn.commit()
  2460. conn.close()
  2461. detail = task_state(auth, stuck)
  2462. check(detail.get("resume_available") is True,
  2463. "task page offers resume while its saddle is offline")
  2464. # The resume POSTs task-update cancel to the daemon; the abort
  2465. # handler fires an ungated re-ask that would race the manual claim
  2466. # below — slow it past the manual ask (the knob the slow-ask
  2467. # regression uses).
  2468. fake.ask_delay = 5.0
  2469. act(auth, detail["resume"], {})
  2470. detail = task_state(auth, stuck)
  2471. check(detail["status"] == "queued",
  2472. "manual resume requeues the stuck task")
  2473. check(any(c["body"] == "Saddle became unreachable; task requeued"
  2474. and c["author"] == "system" and c.get("is_system")
  2475. for c in detail["comments"]),
  2476. "manual resume lands the interruption comment as a system row")
  2477. check(all(s["status"] == "offline"
  2478. for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
  2479. "saddle stays offline after the manual requeue")
  2480. # Bringing the saddle back lets the requeued task resume and finish.
  2481. fake.state = "idle"
  2482. fake_card = saddle_card(auth, "fake-saddle")
  2483. act(auth, fake_card["refresh"], {})
  2484. check(any(s["status"] == "idle"
  2485. for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2486. if s["name"] == "fake-saddle"),
  2487. "refreshed saddle returns idle")
  2488. # The refreshed daemon is idle-awake: it asks (the real loop's
  2489. # own ask; driven directly for determinism — every self-ask trigger
  2490. # is frozen and the abort re-ask is slowed past it; joining that
  2491. # delayed thread here would fire its ask first, so it is left to
  2492. # sleep out the rest of the bundle).
  2493. status, payload = fake.ask_now()
  2494. fake.ask_on_idle_status = True
  2495. fake.ask_on_terminal = True
  2496. fake.ask_on_wake = True
  2497. fake.ask_delay = 0.0
  2498. check(status == 200, "the refreshed daemon's ask claims the requeued task")
  2499. check(wait_task_done(auth, stuck, timeout=60) is not None,
  2500. "requeued task resumes on the refreshed saddle and completes")
  2501. # =============================================================================
  2502. # Bundle: subtasks — the plan-12 task graph: plain-task parking (no
  2503. # subtask events exist to resume it), the DELETED subtask machinery's
  2504. # verbatim refusals, and the goal agent's task CLI in depth
  2505. # (capabilities, ownership, the daemon's task-reading API, the
  2506. # goal-run block answered from the task page).
  2507. # =============================================================================
  2508. def bundle_subtasks(ctx):
  2509. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  2510. app = ctx.app
  2511. # A hold tag nothing carries: goal tasks created with it stay
  2512. # queued (a stable parked agent) until released at the settle.
  2513. saddles_state = slot_public(auth, "/saddles", "saddles")
  2514. act(auth, saddles_state["create_capability"], {"name": "gpu"})
  2515. gpu_cap_id = [c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  2516. if c["name"] == "gpu"][0]
  2517. def goal_state(goal_id):
  2518. return slot_public(auth, "/goals/%d" % goal_id, "goal")
  2519. def create_goal_row(fields):
  2520. act(auth, slot_public(auth, "/goals", "goals")["create"], fields)
  2521. rows = [g for g in slot_public(auth, "/goals", "goals")["goals"]
  2522. if g["title"] == fields["title"]]
  2523. return int(rows[0]["id"])
  2524. print("== B7: plain-task parking stays put (no subtask events) ==")
  2525. fake.suspend_on_title = "Suspend plain"
  2526. pushes_before = len(fake.start_payloads)
  2527. plain = create_queued(auth, {
  2528. "project_id": "1",
  2529. "title": "Suspend plain",
  2530. "priority": "2",
  2531. })
  2532. check(wait_for(lambda: task_state(auth, plain).get("status") == "suspended"
  2533. or False, timeout=60, what="plain task parks") is not None,
  2534. "daemon 'suspended' state parks the task")
  2535. detail = task_state(auth, plain)
  2536. check(detail.get("display") == "parked",
  2537. "a parked plain task displays as parked (%s)" % detail.get("display"))
  2538. check(any(s["status"] == "idle"
  2539. for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
  2540. "suspension frees the saddle")
  2541. first_push = fake.start_payloads[pushes_before]
  2542. check(first_push.get("kind") == "task" and first_push.get("creator") == "human"
  2543. and first_push.get("mode") == "implement",
  2544. "the /start payload carries the plan-12 identity triplet (kind/creator/mode)")
  2545. check("subtasks" not in first_push and "parent" not in first_push,
  2546. "the plan-12 payload drops the subtasks/parent members")
  2547. # Other tasks finishing must NOT resume a parked plain task (the
  2548. # subtask-event channel is gone; only goal events resume goal runs).
  2549. conn = db()
  2550. conn.execute("insert into tasks (project_id, title, status, priority, summary,"
  2551. " completed_at, created_at) values (1, 'Unrelated finisher', 'done',"
  2552. " 5, 'finished loudly', strftime('%s','now','+1 second'),"
  2553. " strftime('%s','now'))")
  2554. conn.commit()
  2555. conn.close()
  2556. check(quiet_window(quiet_seconds(),
  2557. lambda: task_state(auth, plain)["status"] == "suspended",
  2558. "plain park survives unrelated finishes"),
  2559. "a parked plain task never resumes on other tasks' events")
  2560. print("== the deleted machinery refuses, verbatim ==")
  2561. status, body = saddle_api("POST", "/api/saddle/subtask", fake_token,
  2562. {"parent_task_id": plain, "title": "ghost"})
  2563. check(status == 404,
  2564. "the /api/saddle/subtask callback route is gone (404)")
  2565. # The poll-era ingest channel went with it (arch 19): the whole
  2566. # /update surface 404s — subtask and plan verb arms included — and
  2567. # the verdict channel (/poll) with it.
  2568. conn = db()
  2569. plain_session = conn.execute(
  2570. "select id from sessions where task_id = ? and ended_at is null",
  2571. (plain,)).fetchone()[0]
  2572. conn.close()
  2573. status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  2574. {"type": "subtask", "session_id": plain_session,
  2575. "data": {"title": "Ghost child"}})
  2576. check(status == 404,
  2577. "the /update ingest surface is gone (404, the subtask arm with it)")
  2578. status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  2579. {"type": "plan", "session_id": plain_session,
  2580. "data": {"plan": "# nope"}})
  2581. check(status == 404,
  2582. "the task-level 'plan' update arm is gone with the channel (404)")
  2583. # The daemon's task CLI: no parent/plan members; goal context instead.
  2584. status, body = saddle_api("GET", "/api/saddle/task/%d" % plain, fake_token)
  2585. check(status == 200 and "plan" not in body.get("task", {})
  2586. and "parent_task_id" not in body.get("task", {})
  2587. and "goal_id" in body.get("task", {}),
  2588. "the task CLI detail drops parent/plan and carries goal_id")
  2589. status, body = saddle_api("GET", "/api/saddle/tasks?project_id=1", fake_token)
  2590. check(status == 200 and all("parent_task_id" not in t and "goal_id" in t
  2591. for t in body.get("tasks", [])),
  2592. "the task CLI list drops parent_task_id and carries goal_id")
  2593. # Settle the parked plain task: nothing will resume it — cancel it.
  2594. act(auth, task_state(auth, plain)["cancel"], {})
  2595. check(wait_for(lambda: task_state(auth, plain)["status"] == "failed" or False,
  2596. timeout=30, what="parked plain task cancelled") is not None,
  2597. "the parked plain task cancels (the human cleanup path)")
  2598. print("== the goal agent's task CLI in depth ==")
  2599. goal = create_goal_row({"project_id": "1", "title": "Sequencing goal",
  2600. "brief": "Fan tasks out with the CLI.",
  2601. "branch_name": "goal/sequence"})
  2602. fake.goal_script = [("plan", "# Plan\n\n1. Fan out")]
  2603. act(auth, goal_state(goal)["start"], {})
  2604. check(wait_for(lambda: goal_state(goal)["status"] == "plan_review" or False,
  2605. timeout=60, what="sequencing goal plans") is not None,
  2606. "the sequencing goal reaches plan review")
  2607. fake.goal_script = [("suspend",)]
  2608. act(auth, goal_state(goal)["plan_pending_block"]["answer_action"],
  2609. {"answer": "Plan approved. Proceed with implementation."})
  2610. conn = db()
  2611. run_task = conn.execute(
  2612. "select id from tasks where goal_id = ? and kind = 'goal_run'",
  2613. (goal,)).fetchone()[0]
  2614. agent_session = conn.execute(
  2615. "select id from sessions where task_id = ? and ended_at is null",
  2616. (run_task,)).fetchone()[0]
  2617. conn.close()
  2618. check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
  2619. or False, timeout=60, what="sequencing agent parks") is not None,
  2620. "the sequencing goal agent parks")
  2621. # Create with capabilities: attached until assignment (the hold
  2622. # keeps the task queued), mandate + mode stored, ephemeral branch.
  2623. status, body = saddle_api("POST", "/api/saddle/task/create", fake_token, {
  2624. "session_id": agent_session, "title": "Held worker",
  2625. "mandate": "Do the held work.", "mode": "review",
  2626. "capabilities": ["gpu"]})
  2627. check(status == 200 and body.get("ok") is True,
  2628. "task create with a capability tag succeeds")
  2629. held = int(body["task_id"])
  2630. detail = task_state(auth, held)
  2631. check(detail["status"] == "draft" and detail["mode"] == "review",
  2632. "the held worker drafts with its mode")
  2633. check([c["name"] for c in detail.get("capabilities", [])] == ["gpu"],
  2634. "the capability attaches to the created task")
  2635. check(detail["task_branch"] == "task/%d-%d" % (goal, held)
  2636. and detail["branch_from"] == "goal/sequence",
  2637. "the CLI task's ephemeral branch pair is stored")
  2638. status, body = saddle_api("POST", "/api/saddle/task/enqueue", fake_token, {
  2639. "session_id": agent_session, "task_id": held})
  2640. check(status == 200, "the held worker enqueues")
  2641. check(quiet_window(3, lambda: task_state(auth, held)["status"] == "queued",
  2642. "unsatisfiable capability holds dispatch"),
  2643. "the unsatisfiable capability holds the queued worker (never assigned)")
  2644. queue_row = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2645. if int(t["id"]) == held][0]
  2646. check(queue_row.get("waiting_for_saddle") is True,
  2647. "the queue annotates the held worker as waiting for a saddle")
  2648. # The human edit path: goal tasks never carry enqueue-after (the
  2649. # field is human-tasks-only — the update silently ignores it).
  2650. act(auth, task_state(auth, held)["update"], {
  2651. "priority": "4", "enqueue_after_task_id": str(held - 1)})
  2652. check(task_state(auth, held).get("enqueue_after_task_id") in (None, 0, "0"),
  2653. "a goal task ignores the auto-enqueue-after field on edit")
  2654. # Ownership: the same saddle's goal agent cannot touch another
  2655. # goal's task; a plain session is not a goal-agent identity.
  2656. other_goal = create_goal_row({"project_id": "1", "title": "Other goal",
  2657. "brief": "b", "branch_name": "goal/other",
  2658. "priority": "9"})
  2659. status, body = saddle_api("POST", "/api/saddle/task/create", fake_token,
  2660. {"session_id": agent_session, "title": "x",
  2661. "mandate": "m"})
  2662. scope_task = int(body["task_id"])
  2663. check(int(task_state(auth, scope_task)["goal_id"] or 0) == goal,
  2664. "the callback scopes task creation to its own goal")
  2665. status, body = saddle_api("POST", "/api/saddle/task/delete", fake_token,
  2666. {"session_id": agent_session, "task_id": scope_task})
  2667. check(status == 200, "the owned draft deletes")
  2668. check(not any(t["id"] == scope_task
  2669. for t in slot_public(auth, "/tasks", "tasks")["tasks"]),
  2670. "the deleted task is gone from the queue")
  2671. status, body = saddle_api("POST", "/api/saddle/task/enqueue", fake_token,
  2672. {"session_id": agent_session, "task_id": plain})
  2673. check(status == 400 and "does not belong to your goal" in (body.get("error") or ""),
  2674. "the goal agent cannot enqueue a standalone task")
  2675. # The daemon's task CLI detail carries the goal context.
  2676. status, body = saddle_api("GET", "/api/saddle/task/%d" % held, fake_token)
  2677. check(status == 200
  2678. and int(body["task"].get("goal_id") or 0) == goal
  2679. and body["task"].get("goal_title") == "Sequencing goal",
  2680. "the task CLI detail names the owning goal")
  2681. print("== the goal-run block answers from the task page (C1) ==")
  2682. # A question block raised in goal context lands on the run task's
  2683. # own row; the TASK page's answer action funnels into the one
  2684. # goal answer path and resumes the agent with the verdict riding
  2685. # the allocation (the goal page shares the path).
  2686. fake.goal_script = [("question", "Which harness: pytest or vitest?",
  2687. ["pytest", "vitest"])]
  2688. conn = db()
  2689. # Plan 15 C3: a terminal transition (a done task row newer than
  2690. # the watermark) is the resume signal — no event rows exist.
  2691. conn.execute("insert into tasks (project_id, goal_id, kind, title, status,"
  2692. " priority, completed_at, created_at) values (1, ?, 'task',"
  2693. " 'Nudge', 'done', 5,"
  2694. " strftime('%s','now'),"
  2695. " strftime('%s','now','-30 seconds'))", (goal,))
  2696. conn.commit()
  2697. conn.close()
  2698. check(wait_for(lambda: task_state(auth, run_task)["status"] == "blocked"
  2699. or False, timeout=60, what="goal block parks the run") is not None,
  2700. "a goal question parks the run task on its own block row")
  2701. run_detail = task_state(auth, run_task)
  2702. question_block = [b for b in run_detail["blocks"]
  2703. if b["block_type"] == "question"][0]
  2704. check(question_block.get("question") == "Which harness: pytest or vitest?",
  2705. "the run task's block carries the goal question verbatim")
  2706. goal_block_ids = [int(b["id"]) for b in goal_state(goal)["blocks"]
  2707. if b.get("block_type") == "question"]
  2708. check(goal_block_ids == [int(question_block["id"])],
  2709. "the goal page's card is the SAME row (no dual-row pair)")
  2710. act(auth, question_block["answer_action"], {"answer": "pytest"})
  2711. fake.goal_script = [("suspend",)]
  2712. check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
  2713. or False, timeout=60, what="answered block resumes the agent")
  2714. is not None,
  2715. "the answered block resumes the goal agent")
  2716. answered = [b for b in task_state(auth, run_task)["blocks"]
  2717. if b["block_type"] == "question"][0]
  2718. check(answered.get("answer") == "pytest",
  2719. "the verdict is stored on the single block row")
  2720. print("== settle: release the hold and let the shard drain ==")
  2721. # Drop the hold tag so the held worker can dispatch and complete;
  2722. # the parked agent resumes on its completion, re-parks (script),
  2723. # and the shard shuts down with an empty queue.
  2724. act(auth, task_state(auth, held)["capabilities"][0]["detach"], {})
  2725. check(wait_task_done(auth, held, timeout=90) is not None,
  2726. "the released held worker completes")
  2727. check(wait_for(lambda: any(e["kind"] == "task-done" and int(e["task_id"]) == held
  2728. for e in goal_state(goal)["events"]) or False,
  2729. timeout=30, what="narrative hears the held worker") is not None,
  2730. "the narrative records the held worker's completion")
  2731. act(auth, goal_state(goal)["cancel"], {})
  2732. check(wait_for(lambda: goal_state(goal)["status"] == "cancelled" or False,
  2733. timeout=30, what="sequencing goal cancelled") is not None,
  2734. "the sequencing goal cancels (the shard's cleanup path)")
  2735. # =============================================================================
  2736. # Bundle: lineage — the plan-12 goal lineage: goal context on worker
  2737. # dispatches (read-only reference + ephemeral branch), the daemon's
  2738. # task-reading API (goal members, no parent/plan), and the goal
  2739. # agent's workspace affinity while parked.
  2740. # =============================================================================
  2741. def bundle_lineage(ctx):
  2742. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  2743. app = ctx.app
  2744. r4_token = fake_token
  2745. # gpu exists only as a hold (no saddle carries it): the held goal
  2746. # worker stays queued until the release loop lifts it.
  2747. saddles_state = slot_public(auth, "/saddles", "saddles")
  2748. act(auth, saddles_state["create_capability"], {"name": "gpu"})
  2749. def goal_state(goal_id):
  2750. return slot_public(auth, "/goals/%d" % goal_id, "goal")
  2751. def create_goal_row(fields):
  2752. act(auth, slot_public(auth, "/goals", "goals")["create"], fields)
  2753. rows = [g for g in slot_public(auth, "/goals", "goals")["goals"]
  2754. if g["title"] == fields["title"]]
  2755. return int(rows[0]["id"])
  2756. goal = create_goal_row({"project_id": "1", "title": "Lineage goal",
  2757. "brief": "Carry the goal lineage.",
  2758. "branch_name": "goal/lineage"})
  2759. fake.goal_script = [("plan", "# Lineage plan\n\n1. Fan out\n2. Verify")]
  2760. act(auth, goal_state(goal)["start"], {})
  2761. check(wait_for(lambda: goal_state(goal)["status"] == "plan_review" or False,
  2762. timeout=60, what="lineage goal plans") is not None,
  2763. "the lineage goal reaches plan review")
  2764. fake.goal_script = [("suspend",)]
  2765. act(auth, goal_state(goal)["plan_pending_block"]["answer_action"],
  2766. {"answer": "Plan approved. Proceed with implementation."})
  2767. conn = db()
  2768. auto_parent = conn.execute(
  2769. "select id from tasks where goal_id = ? and kind = 'goal_run'",
  2770. (goal,)).fetchone()[0]
  2771. agent_session = conn.execute(
  2772. "select id from sessions where task_id = ? and ended_at is null",
  2773. (auto_parent,)).fetchone()[0]
  2774. conn.close()
  2775. check(wait_for(lambda: task_state(auth, auto_parent)["status"] == "suspended"
  2776. or False, timeout=60, what="lineage goal agent parks") is not None,
  2777. "the lineage goal agent parks")
  2778. # Two workers under the goal: one free, one held by the gpu tag.
  2779. status, body = saddle_api("POST", "/api/saddle/task/create", r4_token, {
  2780. "session_id": agent_session, "title": "Auto child",
  2781. "mandate": "build the widget", "mode": "implement"})
  2782. auto_child = int(body["task_id"])
  2783. status, body = saddle_api("POST", "/api/saddle/task/create", r4_token, {
  2784. "session_id": agent_session, "title": "Auto grandchild",
  2785. "mandate": "verify the widget", "mode": "test",
  2786. "capabilities": ["gpu"]})
  2787. auto_grand = int(body["task_id"])
  2788. status, body = saddle_api("POST", "/api/saddle/task/create", r4_token, {
  2789. "session_id": agent_session, "title": "Wire the flange",
  2790. "description": "torque to spec"})
  2791. check(status == 400 and "mandate is required" in (body.get("error") or ""),
  2792. "task create without a mandate errors verbatim (%s)" % body.get("error"))
  2793. status, body = saddle_api("POST", "/api/saddle/task/create", r4_token, {
  2794. "session_id": agent_session, "title": "Wire the flange",
  2795. "mandate": "torque to spec"})
  2796. flange_task = int(body["task_id"])
  2797. for held in (auto_child, auto_grand, flange_task):
  2798. saddle_api("POST", "/api/saddle/task/enqueue", r4_token,
  2799. {"session_id": agent_session, "task_id": held})
  2800. check(wait_for(lambda: task_state(auth, auto_child)["status"] in
  2801. ("queued", "assigned", "working", "done") or False,
  2802. timeout=30, what="free goal worker dispatches") is not None,
  2803. "the free goal worker dispatches")
  2804. check(quiet_window(3, lambda: task_state(auth, auto_grand)["status"] == "queued",
  2805. "held goal worker stays queued"),
  2806. "the gpu-held goal worker stays queued")
  2807. # A goal worker's dispatch carries the read-only goal context.
  2808. lineage_push = [p for p in fake.start_payloads
  2809. if int(p.get("task", {}).get("id") or 0) == auto_child][-1]
  2810. goal_ctx = lineage_push.get("goal") or {}
  2811. check(int(goal_ctx.get("id") or 0) == goal
  2812. and goal_ctx.get("branch") == "goal/lineage"
  2813. and "Carry the goal lineage." in goal_ctx.get("brief", "")
  2814. and "# Lineage plan" in goal_ctx.get("plan", ""),
  2815. "the worker dispatch carries the goal context (brief + approved plan)")
  2816. check(lineage_push.get("ephemeral_branch") == "task/%d-%d" % (goal, auto_child)
  2817. and lineage_push.get("kind") == "task"
  2818. and lineage_push.get("mode") == "implement"
  2819. and lineage_push.get("creator") == "goal",
  2820. "the worker dispatch carries the ephemeral branch + identity triplet")
  2821. check("parent" not in lineage_push and "subtasks" not in lineage_push,
  2822. "the plan-12 payload drops the parent/subtasks members")
  2823. print("== R5 B2: task-reading API ==")
  2824. status, _ = saddle_api("GET", "/api/saddle/tasks", None)
  2825. check(status == 401, "tasks endpoint rejects a missing token")
  2826. status, _ = saddle_api("GET", "/api/saddle/task/%d" % auto_child, None)
  2827. check(status == 401, "task detail endpoint rejects a missing token")
  2828. # Fixture: a second project whose tasks all carry the "Diff " title
  2829. # prefix for the project_id filter checks — created first so the
  2830. # unfiltered list has the population the count check expects.
  2831. state = slot_public(auth, "/projects", "projects")
  2832. act(auth, state["create"], {"name": "diff-fabrica",
  2833. "git_url": "ssh://fabrica.unitatem.net:222/clanker/clanker-saddle.git"})
  2834. conn = db()
  2835. other_project = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  2836. if p["name"] == "diff-fabrica"][0])
  2837. for suffix in ("fabrica", "scp"):
  2838. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  2839. " values (?, ?, 'done', 5, strftime('%s','now'))",
  2840. (other_project, "Diff %s" % suffix))
  2841. conn.commit()
  2842. conn.close()
  2843. status, body = saddle_api("GET", "/api/saddle/tasks", r4_token)
  2844. check(status == 200 and len(body.get("tasks", [])) > 5,
  2845. "tasks endpoint lists tasks without filters")
  2846. row = [t for t in body["tasks"] if int(t["id"]) == auto_child][0]
  2847. check(row["title"] == "Auto child" and int(row["goal_id"] or 0) == goal
  2848. and row["status"] in ("done", "queued", "working"),
  2849. "compact rows carry id/title/status/goal_id")
  2850. status, body = saddle_api("GET", "/api/saddle/tasks?project_id=1", r4_token)
  2851. check(all(True for _ in [1]) # project 1 dominates the data set
  2852. and not any(int(t["id"]) == 0 for t in body["tasks"]),
  2853. "project_id filter accepted")
  2854. status, body = saddle_api("GET",
  2855. "/api/saddle/tasks?project_id=%d" % other_project, r4_token)
  2856. check(all(t["title"].startswith("Diff") for t in body["tasks"]),
  2857. "other project lists only its own tasks")
  2858. status, body = saddle_api("GET", "/api/saddle/tasks?q=FLANGE", r4_token)
  2859. titles = [t["title"] for t in body["tasks"]]
  2860. check("Wire the flange" in titles,
  2861. "q matches titles case-insensitively")
  2862. status, body = saddle_api("GET", "/api/saddle/tasks?q=torque%20to%20spec", r4_token)
  2863. check(any(t["title"] == "Wire the flange" for t in body["tasks"]),
  2864. "q matches mandate text")
  2865. status, body = saddle_api("GET", "/api/saddle/task/%d" % auto_child, r4_token)
  2866. task_view = body.get("task", {})
  2867. check(status == 200 and int(task_view.get("id") or 0) == auto_child
  2868. and task_view.get("goal_title") == "Lineage goal"
  2869. and int(task_view.get("goal_id") or 0) == goal,
  2870. "single-task view joins the goal title")
  2871. check("status" in task_view and "description" in task_view
  2872. and "summary" in task_view,
  2873. "single-task view carries the full detail fields")
  2874. check("plan" not in task_view and "parent_task_id" not in task_view,
  2875. "the single-task view drops the deleted parent/plan members")
  2876. status, body = saddle_api("GET", "/api/saddle/task/99999", r4_token)
  2877. check(status == 404, "unknown task id returns 404")
  2878. print("== settle: released workers resume the parked agent (coalesced) ==")
  2879. # Release the held workers: their completions wake the parked goal
  2880. # agent (affinity — the same fake hosts it) with the batched
  2881. # notice; the free worker already completed.
  2882. check(wait_task_done(auth, auto_child, timeout=90) is not None,
  2883. "the free worker settles first")
  2884. def agent_push_count():
  2885. return len([p for p in fake.goal_assignments
  2886. if int(p.get("task", {}).get("id") or 0) == auto_parent])
  2887. before_pushes = agent_push_count()
  2888. for held_task in (auto_grand, flange_task):
  2889. caps = task_state(auth, held_task).get("capabilities") or []
  2890. if caps:
  2891. act(auth, caps[0]["detach"], {})
  2892. check(wait_task_done(auth, held_task, timeout=90) is not None,
  2893. "released worker settles (%d)" % held_task)
  2894. check(wait_for(lambda: agent_push_count() > before_pushes or False,
  2895. timeout=90, what="goal agent resumes") is not None,
  2896. "the workers' completions resume the parked goal agent")
  2897. notice = ""
  2898. for p in fake.goal_assignments:
  2899. if int(p.get("task", {}).get("id") or 0) == auto_parent:
  2900. notice = p.get("resume_notice") or ""
  2901. check("Auto child" in notice and "Auto grandchild" in notice
  2902. and "Wire the flange" in notice,
  2903. "the coalesced resume notice covers every plan task")
  2904. check("New completion reports" in notice and "fake daemon finished" in notice,
  2905. "the notice carries the new completion reports")
  2906. act(auth, goal_state(goal)["cancel"], {})
  2907. check(wait_for(lambda: goal_state(goal)["status"] == "cancelled" or False,
  2908. timeout=30, what="lineage goal cancelled") is not None,
  2909. "the lineage goal cancels (drain)")
  2910. # =============================================================================
  2911. # Bundle: allocation — the ask/wake/allocate machinery: concurrent ask
  2912. # race, token revocation, workspace affinity, the lost-wake guard, the
  2913. # slow-ask regression, assignment capability context + the tag
  2914. # catalogue, and the pinned-blocks/activity dashboard.
  2915. # =============================================================================
  2916. def bundle_allocation(ctx):
  2917. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  2918. app = ctx.app
  2919. r4_token = fake_token
  2920. # gpu rides the primary fake from the start (the routing bundle's
  2921. # early attach, replayed): its attach row precedes lostwake's, so
  2922. # the assignment's saddle_tags order matches the assertion in the
  2923. # capability-context section below.
  2924. saddles_state = slot_public(auth, "/saddles", "saddles")
  2925. act(auth, saddles_state["create_capability"], {"name": "gpu"})
  2926. act(auth, saddle_card(auth, "fake-saddle")["attach_capability"],
  2927. {"capability_id": str(int([c["id"] for c in
  2928. slot_public(auth, "/saddles", "saddles")["capabilities"]
  2929. if c["name"] == "gpu"][0]))})
  2930. print("== phase 1: ask/allocation — token auth + concurrent race ==")
  2931. # Plan 10 phase 1. Registration mints per-saddle tokens; the
  2932. # ask endpoint authenticates by them; concurrent asks race to
  2933. # exactly one winner through the allocation transaction.
  2934. status, _ = saddle_api("POST", "/api/saddle/next-task", None)
  2935. check(status == 401, "next-task rejects a missing token")
  2936. status, _ = saddle_api("POST", "/api/saddle/next-task", "saddle-bogus-token")
  2937. check(status == 401, "next-task rejects a wrong token")
  2938. status, body = saddle_api("POST", "/api/saddle/next-task", fake_token)
  2939. check(status == 204, "next-task answers 204 (nothing offerable — daemon sleeps)")
  2940. # Two more racers: real daemons on their own ports; fake2 reports
  2941. # its ssh key at /register so the key lands on the row directly.
  2942. fake2 = FakeSaddle()
  2943. fake2.ssh_public_key = "ssh-ed25519 AAAAFake2RacerKey saddle-2@fedora"
  2944. fake2.serve(port=free_port())
  2945. fake3 = FakeSaddle()
  2946. fake3.serve(port=free_port())
  2947. saddles_state = slot_public(auth, "/saddles", "saddles")
  2948. act(auth, saddles_state["register"], {"name": "racer-two",
  2949. "host": "127.0.0.1:%d" % fake2.port})
  2950. act(auth, saddles_state["register"], {"name": "racer-three",
  2951. "host": "127.0.0.1:%d" % fake3.port})
  2952. check(wait_for(lambda: len(fake2.register_log) > 0 and len(fake3.register_log) > 0
  2953. or False, timeout=15, what="racers registered") is not None,
  2954. "both racers received /register")
  2955. check(wait_for(lambda: "AAAAFake2RacerKey" in
  2956. [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2957. if s["name"] == "racer-two"][0].get("ssh_public_key", "") or False,
  2958. timeout=15, what="register-reported key") is not None,
  2959. "the ssh key reported at /register lands on the saddle row")
  2960. racer_cards = {}
  2961. for name, daemon in (("fake-saddle", fake), ("racer-two", fake2), ("racer-three", fake3)):
  2962. card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2963. if s["name"] == name][0]
  2964. racer_cards[daemon] = {"id": int(card["id"]), "token": card["token"]}
  2965. # Revocation: a registered saddle's token works until the saddle
  2966. # is removed, then nothing authorises its calls.
  2967. fake4 = FakeSaddle()
  2968. fake4.ask_on_register = False # a pre-flip daemon: never asks
  2969. fake4.serve(port=free_port())
  2970. saddles_state = slot_public(auth, "/saddles", "saddles")
  2971. act(auth, saddles_state["register"], {"name": "old-daemon",
  2972. "host": "127.0.0.1:%d" % fake4.port})
  2973. old_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2974. if s["name"] == "old-daemon"][0]
  2975. status, _ = saddle_api("POST", "/api/saddle/next-task", old_card["token"])
  2976. check(status in (200, 204),
  2977. "the old daemon's fresh token authenticates next-task")
  2978. status, _ = saddle_api("POST", "/api/saddle/note", old_card["token"],
  2979. {"project": "demo", "body": "from the old daemon"})
  2980. check(status == 200, "note accepts any registered saddle's token")
  2981. act(auth, old_card["unregister"], {})
  2982. status, _ = saddle_api("POST", "/api/saddle/next-task", old_card["token"])
  2983. check(status == 401, "removing the saddle revokes its token")
  2984. status, _ = saddle_api("POST", "/api/saddle/note", old_card["token"],
  2985. {"project": "demo", "body": "should fail"})
  2986. check(status == 401, "revocation covers the note endpoint too")
  2987. # Deterministic race: settle the queue first (this shard runs the
  2988. # race before any parked work exists, so only in-flight asks need
  2989. # draining), freeze every daemon's self-ask triggers, put all three
  2990. # to sleep (204), then fire one task worth of concurrent asks.
  2991. for daemon in (fake, fake2, fake3):
  2992. daemon.ask_on_register = False
  2993. daemon.ask_on_wake = False
  2994. daemon.ask_on_terminal = False
  2995. daemon.ask_on_idle_status = False
  2996. daemon.settle_asks() # converted sleep: in-flight asks land (204)
  2997. def queued_count():
  2998. conn = db()
  2999. try:
  3000. return conn.execute(
  3001. "select count(*) from tasks where status = 'queued'").fetchone()[0]
  3002. finally:
  3003. conn.close()
  3004. check(wait_for(lambda: queued_count() == 0 or False,
  3005. timeout=60, what="queue settles before the race") is not None,
  3006. "the queue settles before the race")
  3007. sleep_statuses = [daemon.ask_now()[0] for daemon in (fake, fake2, fake3)]
  3008. check(sleep_statuses == [204, 204, 204],
  3009. "all three racers sleep (204) before the task exists")
  3010. task_id_tmp = create_queued(auth, {
  3011. "project_id": "1",
  3012. "title": "Race task",
  3013. "priority": "2",
  3014. })
  3015. race_task = task_id_tmp
  3016. race_results = []
  3017. def race_ask(daemon):
  3018. # A real racing daemon accepts the assignment it wins — the
  3019. # manual ask path does exactly that on a 200.
  3020. race_results.append(daemon.ask_now())
  3021. threads = [threading.Thread(target=race_ask, args=(daemon,))
  3022. for daemon in (fake, fake2, fake3)]
  3023. for thread in threads:
  3024. thread.start()
  3025. for thread in threads:
  3026. thread.join()
  3027. race_statuses = sorted(r[0] for r in race_results)
  3028. check(race_statuses == [200, 204, 204],
  3029. "concurrent asks race to exactly one winner (%s)" % race_statuses)
  3030. winner = [r for r in race_results if r[0] == 200][0][1]
  3031. check(int(winner["task"]["id"]) == race_task, "the winner's payload carries the task")
  3032. check(int(winner["session_id"]) > 0, "the assignment carries the session id")
  3033. conn = db()
  3034. race_sessions = conn.execute(
  3035. "select count(*) from sessions where task_id = ?", (race_task,)).fetchone()[0]
  3036. conn.close()
  3037. check(race_sessions == 1,
  3038. "exactly one session row — the transaction serialised the claims (%d)"
  3039. % race_sessions)
  3040. check(wait_task_done(auth, race_task, timeout=60) is not None,
  3041. "the raced task completes on the winning daemon")
  3042. for daemon in (fake, fake2, fake3):
  3043. daemon.ask_on_register = True
  3044. daemon.ask_on_wake = True
  3045. daemon.ask_on_terminal = True
  3046. daemon.ask_on_idle_status = True
  3047. # Re-establish sleeping explicitly: the winner's post-race
  3048. # re-ask timing is not deterministic, and the affinity section
  3049. # below needs every racer provably asleep. Any stray offerable
  3050. # drains into these asks first.
  3051. def all_asleep():
  3052. return all(daemon.ask_now()[0] == 204 for daemon in (fake, fake2, fake3))
  3053. check(wait_for(lambda: all_asleep() or False, timeout=30,
  3054. what="racers re-sleep after the race") is not None,
  3055. "all racers re-sleep after the race")
  3056. print("== phase 1: workspace affinity pins parked tasks ==")
  3057. # A suspended parent parked on racer-two may be resumed ONLY by
  3058. # racer-two: a tag-capable non-affine saddle asking gets 204, and
  3059. # the wake targets the affine saddle. Only racer-two carries the
  3060. # tag at dispatch time (the race's winner is nondeterministic, so
  3061. # the park must not depend on which racers are free); racer-three
  3062. # gains it afterwards to prove the pinning is by workspace, not
  3063. # tags.
  3064. saddles_state = slot_public(auth, "/saddles", "saddles")
  3065. act(auth, saddles_state["create_capability"], {"name": "affinity-tag"})
  3066. affinity_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  3067. if c["name"] == "affinity-tag"][0])
  3068. racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3069. if s["name"] == "racer-two"][0]
  3070. racer3_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3071. if s["name"] == "racer-three"][0]
  3072. act(auth, racer2_card["attach_capability"], {"capability_id": str(affinity_cap)})
  3073. # Plan 12: the parked workspace is the goal agent's run task. The
  3074. # other racers sleep (frozen ask triggers), so racer-two claims
  3075. # the goal run and parks it; the goal-task event then resumes the
  3076. # agent AFFINE to racer-two — racer-three (tag-capable or not)
  3077. # gets 204, the dispatcher's wake targets racer-two.
  3078. fake.ask_on_wake = False
  3079. fake.ask_on_idle_status = False
  3080. fake.settle_asks()
  3081. fake3.ask_on_wake = False
  3082. fake3.ask_on_idle_status = False
  3083. fake3.settle_asks()
  3084. act(auth, slot_public(auth, "/goals", "goals")["create"], {
  3085. "project_id": "1", "title": "Affinity goal",
  3086. "brief": "Parks on racer-two.", "branch_name": "goal/affinity"})
  3087. conn = db()
  3088. affinity_goal = conn.execute(
  3089. "select id from goals where title = 'Affinity goal'").fetchone()[0]
  3090. conn.close()
  3091. fake2.goal_script = [("plan", "# Affinity plan\n\n1. Park")]
  3092. act(auth, slot_public(auth, "/goals/%d" % affinity_goal, "goal")["start"], {})
  3093. check(wait_for(lambda: slot_public(auth, "/goals/%d" % affinity_goal,
  3094. "goal")["status"] == "plan_review" or False,
  3095. timeout=60, what="affinity goal plans") is not None,
  3096. "the affinity goal reaches plan review (racer-two hosting)")
  3097. fake2.goal_script = [("suspend",)]
  3098. # Freeze racer-two's ask triggers across the approval too (the
  3099. # later wake measurement already does): the mirror-sync fix makes
  3100. # the run resume-offerable the moment its goal row carries the
  3101. # verdict, so a wake-triggered background ask would race this
  3102. # manual ask_now for the claim.
  3103. fake2.ask_on_wake = False
  3104. fake2.ask_on_idle_status = False
  3105. fake2.settle_asks()
  3106. act(auth, slot_public(auth, "/goals/%d" % affinity_goal, "goal")
  3107. ["plan_pending_block"]["answer_action"],
  3108. {"answer": "Plan approved. Proceed with implementation."})
  3109. conn = db()
  3110. affinity_parent = conn.execute(
  3111. "select id from tasks where goal_id = ? and kind = 'goal_run'",
  3112. (affinity_goal,)).fetchone()[0]
  3113. agent_session = conn.execute(
  3114. "select id from sessions where task_id = ? and ended_at is null",
  3115. (affinity_parent,)).fetchone()[0]
  3116. conn.close()
  3117. status, resume_payload = fake2.ask_now()
  3118. check(status == 200 and int(resume_payload["task"]["id"]) == affinity_parent,
  3119. "the approved plan resumes the agent on racer-two (its workspace)")
  3120. check(wait_for(lambda: task_state(auth, affinity_parent).get("status") == "suspended"
  3121. or False, timeout=60, what="affinity agent parks") is not None,
  3122. "the affinity goal agent parks on racer-two")
  3123. conn = db()
  3124. affinity_session = conn.execute(
  3125. "select saddle_id from sessions where task_id = ? and ended_at is null",
  3126. (affinity_parent,)).fetchone()[0]
  3127. conn.close()
  3128. check(affinity_session == int(racer2_card["id"]),
  3129. "the parked workspace sits on racer-two")
  3130. # Now racer-three carries the tag too — the pinning is by
  3131. # workspace, not tags. With nothing due, a non-affine ask gets 204.
  3132. act(auth, racer3_card["attach_capability"], {"capability_id": str(affinity_cap)})
  3133. status, _ = fake3.ask_now()
  3134. check(status == 204,
  3135. "a tag-capable non-affine saddle is refused the parked agent (204)")
  3136. # A goal task completing makes the parked agent resume-offerable —
  3137. # affine to racer-two only. The other racers wake back up (they
  3138. # host the worker; their claims of the parked agent get refused as
  3139. # non-affine), while racer-two's ask triggers stay frozen so the
  3140. # wake measurement stays clean. A wake may have hit the frozen
  3141. # racers while the goal run queued (the lost-wake guard marks them
  3142. # offline): the refresh probe brings them back before the worker.
  3143. fake.ask_on_wake = True
  3144. fake.ask_on_idle_status = True
  3145. fake3.ask_on_wake = True
  3146. fake3.ask_on_idle_status = True
  3147. for racer_name in ("fake-saddle", "racer-three"):
  3148. act(auth, saddle_card(auth, racer_name)["refresh"], {})
  3149. check(wait_for(lambda name=racer_name: [s for s in
  3150. slot_public(auth, "/saddles", "saddles")["saddles"]
  3151. if s["name"] == name][0]["status"] == "idle"
  3152. or False, timeout=15, what="racer re-onlined") is not None,
  3153. "%s re-onlined after its frozen-window wake" % racer_name)
  3154. fake2.ask_on_wake = False
  3155. fake2.ask_on_idle_status = False
  3156. wakes_before_2 = len(fake2.wake_log)
  3157. fake2.settle_asks() # converted sleep: park-time ask lands (204)
  3158. status, body = saddle_api("POST", "/api/saddle/task/create", fake2.token, {
  3159. "session_id": agent_session, "title": "Affinity child",
  3160. "mandate": "finish the affinity fixture"})
  3161. check(status == 200, "the parked agent fans its worker out via the CLI")
  3162. affinity_child = int(body["task_id"])
  3163. saddle_api("POST", "/api/saddle/task/enqueue", fake2.token,
  3164. {"session_id": agent_session, "task_id": affinity_child})
  3165. check(wait_task_done(auth, affinity_child, timeout=90) is not None,
  3166. "the affinity worker completes (on a non-affine racer)")
  3167. check(wait_for(lambda: len(fake2.wake_log) > wakes_before_2 or False,
  3168. timeout=90, what="affine wake") is not None,
  3169. "the dispatcher's wake targets the affine saddle")
  3170. status, resume_payload = fake2.ask_now()
  3171. fake2.ask_on_wake = True # restored whatever the claim returned
  3172. fake2.ask_on_idle_status = True
  3173. check(status == 200 and int(resume_payload["task"]["id"]) == affinity_parent,
  3174. "the affine saddle's ask claims the parked agent")
  3175. check("Affinity child" in (resume_payload.get("resume_notice") or "")
  3176. and "fake daemon finished" in (resume_payload.get("resume_notice") or ""),
  3177. "the resume assignment carries the goal's batched report notice")
  3178. check(wait_for(lambda: task_state(auth, affinity_parent).get("status")
  3179. == "suspended" or False,
  3180. timeout=60, what="affinity agent re-parks") is not None,
  3181. "the resumed agent re-parks (script)")
  3182. act(auth, slot_public(auth, "/goals/%d" % affinity_goal, "goal")["cancel"], {})
  3183. check(wait_for(lambda: slot_public(auth, "/goals/%d" % affinity_goal,
  3184. "goal")["status"] == "cancelled" or False,
  3185. timeout=30, what="affinity goal cancelled") is not None,
  3186. "the affinity goal cancels (drain)")
  3187. print("== phase 1: lost wake -> offline + re-wake of the next capable ==")
  3188. # racer-three sleeps through wakes (a daemon that died after its
  3189. # last 204): the guard window (SADDLE_LOST_WAKE_S=2) passes with
  3190. # no ask, the saddle goes offline, and the work moves to the
  3191. # next capable sleeping saddle.
  3192. fake3.ask_on_wake = False
  3193. fake3.ask_on_idle_status = False # sleeps through everything
  3194. saddles_state = slot_public(auth, "/saddles", "saddles")
  3195. act(auth, saddles_state["create_capability"], {"name": "lostwake"})
  3196. lostwake_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  3197. if c["name"] == "lostwake"][0])
  3198. racer3_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3199. if s["name"] == "racer-three"][0]
  3200. act(auth, racer3_card["attach_capability"], {"capability_id": str(lostwake_cap)})
  3201. # Racer-three must be asleep to be woken: park it with a manual
  3202. # 204 ask (nothing offerable for it — the task below is not
  3203. # created yet).
  3204. status, _ = fake3.ask_now()
  3205. check(status == 204, "racer-three sleeps before the lost-wake run")
  3206. task_id_tmp = create_queued(auth, {
  3207. "project_id": "1",
  3208. "title": "Lost wake task",
  3209. "priority": "2",
  3210. "capability_id": [str(lostwake_cap)],
  3211. })
  3212. lost_task = task_id_tmp
  3213. check(wait_for(lambda: len(fake3.wake_log) > 0 or False,
  3214. timeout=30, what="wake sent to racer-three") is not None,
  3215. "the only capable sleeping saddle is woken")
  3216. check(wait_for(lambda: [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3217. if s["name"] == "racer-three"][0]["status"] == "offline"
  3218. or False, timeout=30, what="lost-wake -> offline") is not None,
  3219. "no ask after the wake: the saddle goes offline (lost-wake guard)")
  3220. check(task_state(auth, lost_task)["status"] == "queued",
  3221. "the task is untouched while its wake was pending")
  3222. # Re-wake the next capable: the healthy saddle gains the tag and
  3223. # picks the work up via its own wake -> ask -> allocate roundtrip.
  3224. fake_card = saddle_card(auth, "fake-saddle")
  3225. act(auth, fake_card["attach_capability"], {"capability_id": str(lostwake_cap)})
  3226. wakes_before = len(fake.wake_log)
  3227. check(wait_for(lambda: len(fake.wake_log) > wakes_before or False,
  3228. timeout=30, what="re-wake of the next capable") is not None,
  3229. "the next capable sleeping saddle is woken for the task")
  3230. check(wait_task_done(auth, lost_task, timeout=60) is not None,
  3231. "wake -> ask -> allocate: the task completes on the healthy saddle")
  3232. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == lost_task
  3233. for p in fake.start_payloads) or False,
  3234. timeout=10, what="lost task assignment captured") is not None,
  3235. "the re-woken saddle's ask produced the assignment")
  3236. fake3.ask_on_wake = True
  3237. fake3.ask_on_idle_status = True
  3238. print("== phase 3: capability context, catalogue, subtask tags ==")
  3239. # Assignment capability context (plan 10 phase 3): every
  3240. # assignment carries {saddle_tags: the allocated saddle's tags,
  3241. # fleet_tags: every tag name known across the fleet} — the
  3242. # daemon renders it into its system prompt. fake-saddle carries
  3243. # gpu + lostwake; the catalogue is affinity-tag, gpu, lostwake.
  3244. fake2.ask_on_register = False
  3245. fake2.ask_on_wake = False
  3246. fake2.ask_on_terminal = False
  3247. fake2.ask_on_idle_status = False
  3248. fake2.settle_asks() # converted sleep: in-flight trigger asks land
  3249. task_id_tmp = create_queued(auth, {
  3250. "project_id": "1",
  3251. "title": "Capabilities context",
  3252. "priority": "2",
  3253. })
  3254. cap_task = task_id_tmp
  3255. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == cap_task
  3256. for p in fake.start_payloads) or False,
  3257. timeout=60, what="capabilities-context task dispatched") is not None,
  3258. "the capabilities-context task dispatches")
  3259. cap_payload = [p for p in fake.start_payloads
  3260. if int(p.get("task", {}).get("id") or 0) == cap_task][-1]
  3261. context = cap_payload.get("capabilities") or {}
  3262. check(context.get("saddle_tags") == ["gpu", "lostwake"],
  3263. "assignment saddle_tags = the allocated saddle's tags (%s)"
  3264. % context.get("saddle_tags"))
  3265. check(context.get("fleet_tags") == ["affinity-tag", "gpu", "lostwake"],
  3266. "assignment fleet_tags = every tag name known across the fleet (%s)"
  3267. % context.get("fleet_tags"))
  3268. check(wait_task_done(auth, cap_task, timeout=60) is not None,
  3269. "capabilities-context task completes")
  3270. # GET /api/saddle/capabilities feeds the daemon's `saddle
  3271. # capabilities` CLI: per-saddle-token auth, tag order by name,
  3272. # each tag with the saddles carrying it (name order).
  3273. status, _ = saddle_api("GET", "/api/saddle/capabilities", None)
  3274. check(status == 401, "capabilities endpoint rejects a missing token")
  3275. status, body = saddle_api("GET", "/api/saddle/capabilities", "saddle-bogus")
  3276. check(status == 401, "capabilities endpoint rejects a wrong token")
  3277. status, body = saddle_api("GET", "/api/saddle/capabilities", fake_token)
  3278. catalogue = {t["name"]: t["saddles"] for t in body.get("tags", [])}
  3279. check(status == 200 and body.get("ok") is True
  3280. and [t["name"] for t in body.get("tags", [])]
  3281. == ["affinity-tag", "gpu", "lostwake"],
  3282. "catalogue lists every tag by name")
  3283. check(catalogue.get("gpu") == ["fake-saddle"]
  3284. and catalogue.get("affinity-tag") == ["racer-three", "racer-two"]
  3285. and catalogue.get("lostwake") == ["fake-saddle", "racer-three"],
  3286. "each tag names its carrying saddles in name order (%s)" % catalogue)
  3287. # Task CLI capabilities (`task create --capability`): validated
  3288. # against the operator tag set; unknown tags reject with the valid
  3289. # list in the error (the agent reads it verbatim in the CLI
  3290. # output). A goal agent's parked session hosts the calls.
  3291. fake.suspend_on_title = "Tag host"
  3292. tag_host = create_queued(auth, {
  3293. "project_id": "1", "title": "Tag host", "priority": "3"})
  3294. check(wait_for(lambda: task_state(auth, tag_host)["status"] == "suspended"
  3295. or False, timeout=60, what="tag host parks") is not None,
  3296. "a parked plain task hosts nothing (no goal scope)")
  3297. # The goal callbacks need a real goal agent: reuse the sequencing
  3298. # goal pattern — a fresh parked agent under this shard.
  3299. act(auth, slot_public(auth, "/goals", "goals")["create"], {
  3300. "project_id": "1", "title": "Tag goal", "brief": "b",
  3301. "branch_name": "goal/tags"})
  3302. conn = db()
  3303. tag_goal = conn.execute("select id from goals where title = 'Tag goal'").fetchone()[0]
  3304. conn.close()
  3305. fake.goal_script = [("suspend",)]
  3306. act(auth, slot_public(auth, "/goals/%d" % tag_goal, "goal")["start"], {})
  3307. conn = db()
  3308. tag_run = conn.execute(
  3309. "select id from tasks where goal_id = ? and kind = 'goal_run'",
  3310. (tag_goal,)).fetchone()[0]
  3311. conn.close()
  3312. check(wait_for(lambda: task_state(auth, tag_run)["status"] == "suspended"
  3313. or False, timeout=60, what="tag goal agent parks") is not None,
  3314. "the tag goal agent parks")
  3315. conn = db()
  3316. tag_session = conn.execute(
  3317. "select id from sessions where task_id = ? and ended_at is null",
  3318. (tag_run,)).fetchone()[0]
  3319. conn.close()
  3320. conn = db()
  3321. tag_host_saddle = conn.execute(
  3322. "select saddle_id from sessions where id = ?", (tag_session,)).fetchone()[0]
  3323. conn.close()
  3324. racer2_card_id = int([c for c in slot_public(auth, "/saddles", "saddles")["saddles"]
  3325. if c["name"] == "racer-two"][0]["id"])
  3326. racer3_card_id = int([c for c in slot_public(auth, "/saddles", "saddles")["saddles"]
  3327. if c["name"] == "racer-three"][0]["id"])
  3328. host_daemon = {int(fake_id): fake, racer2_card_id: fake2,
  3329. racer3_card_id: fake3}[int(tag_host_saddle)]
  3330. status, body = saddle_api("POST", "/api/saddle/task/create", host_daemon.token, {
  3331. "session_id": tag_session, "title": "Bad tag worker",
  3332. "mandate": "m", "capabilities": ["gpu", "warp-drive"]})
  3333. check(status == 400, "an unknown capability tag rejects the task call")
  3334. error_text = body.get("error", "")
  3335. check("warp-drive" in error_text and "valid tags" in error_text
  3336. and "gpu" in error_text and "lostwake" in error_text,
  3337. "the rejection names the unknown tag and the valid list (%s)" % error_text)
  3338. status, _ = saddle_api("POST", "/api/saddle/task/create", host_daemon.token, {
  3339. "session_id": tag_session, "title": "Bad shape worker",
  3340. "mandate": "m", "capabilities": "gpu"})
  3341. check(status == 400, "a non-array capabilities member rejects")
  3342. # A valid tag creates the goal worker in draft with exactly the
  3343. # explicit tag attached; the goal agent enqueues it.
  3344. status, body = saddle_api("POST", "/api/saddle/task/create", host_daemon.token, {
  3345. "session_id": tag_session, "title": "Explicit tag worker",
  3346. "mandate": "do the tagged work", "capabilities": ["gpu"]})
  3347. check(status == 200 and body.get("ok") is True,
  3348. "a valid tag creates the task synchronously")
  3349. tagged_worker = int(body["task_id"])
  3350. detail = task_state(auth, tagged_worker)
  3351. check([c["name"] for c in detail.get("capabilities", [])] == ["gpu"],
  3352. "the worker carries exactly the explicit tag")
  3353. conn = db()
  3354. tag_rows = conn.execute(
  3355. "select c.name from task_capabilities tc"
  3356. " join capabilities c on c.id = tc.capability_id"
  3357. " where tc.task_id = ?", (tagged_worker,)).fetchall()
  3358. conn.close()
  3359. check([r[0] for r in tag_rows] == ["gpu"],
  3360. "the task_capabilities rows record the explicit tag")
  3361. check(detail["status"] == "draft",
  3362. "the callback's task lands in draft until its agent enqueues")
  3363. saddle_api("POST", "/api/saddle/task/enqueue", host_daemon.token,
  3364. {"session_id": tag_session, "task_id": tagged_worker})
  3365. check(wait_task_done(auth, tagged_worker, timeout=90) is not None,
  3366. "the tagged worker dispatches and completes")
  3367. act(auth, slot_public(auth, "/goals/%d" % tag_goal, "goal")["cancel"], {})
  3368. act(auth, task_state(auth, tag_host)["cancel"], {})
  3369. print("== phase 3: dashboard — pinned blocks + activity timeline ==")
  3370. # Pinned OPEN blocking items as link-through cards, newest
  3371. # first; no inline answer forms (the card links to the task
  3372. # page). Fixture: one open question + one open review raised on a
  3373. # local task (stand-ins for the auto-approve grandchild's
  3374. # leftovers) join the probe RFC raised below.
  3375. conn = db()
  3376. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  3377. " values (1, 'Dash fixture task', 'working', 5, strftime('%s','now'))")
  3378. dash_task = conn.execute("select max(id) from tasks").fetchone()[0]
  3379. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  3380. " values (?, ?, strftime('%s','now'))", (dash_task, fake_id))
  3381. dash_session = conn.execute("select max(id) from sessions").fetchone()[0]
  3382. conn.commit()
  3383. conn.close()
  3384. def raise_on_dash(question, kind="question", body=None, summary=None):
  3385. """Push-era raises on the dashboard fixture task (the poll-era
  3386. /update channel is gone): a blocked results push carries the
  3387. block payload; a review verb rides a review entry."""
  3388. if kind == "review" and summary is not None:
  3389. return saddle_api("POST", "/api/saddle/results", fake_token, {
  3390. "session_id": dash_session, "state": "working",
  3391. "entries": [{"id": 9317,
  3392. "command": "saddle review",
  3393. "output": summary, "type": "review",
  3394. "timestamp": time.strftime(
  3395. "%Y-%m-%dT%H:%M:%SZ", time.gmtime())}]})[0]
  3396. block = {"question": question, "type": kind}
  3397. if body is not None:
  3398. block["body"] = body
  3399. return saddle_api("POST", "/api/saddle/results", fake_token, {
  3400. "session_id": dash_session, "state": "blocked",
  3401. "block": block})[0]
  3402. raise_on_dash("Dash fixture question?", kind="question")
  3403. raise_on_dash("dash fixture review", kind="review", summary="dash fixture review")
  3404. # The activity timeline: last <= 50 events, newest first, fed
  3405. # from the existing tables. Fixture rows a few seconds old
  3406. # guarantee the progress kind and a human-authored comment ride
  3407. # along; the probe block below proves same-source ordering.
  3408. conn = db()
  3409. conn.execute("insert into task_comments (task_id, author, body, created_at)"
  3410. " values (?, 'user:admin', 'timeline fixture comment by a human',"
  3411. " strftime('%s','now','-5 seconds'))", (cap_task,))
  3412. conn.execute("insert into task_progress (task_id, summary, created_at)"
  3413. " values (?, 'timeline fixture progress',"
  3414. " strftime('%s','now','-5 seconds'))",
  3415. (cap_task,))
  3416. conn.commit()
  3417. conn.close()
  3418. # The probe RFC raised last: the newest pinned card and the newest
  3419. # timeline event (its raise+answer below prove same-source ordering
  3420. # both ways). The timeline sorts same-second events by source
  3421. # materialisation order, so the probe must land in a LATER clock
  3422. # second than the fixtures — wait for the boundary (bounded).
  3423. fixtures_epoch = int(time.time())
  3424. wait_for(lambda: int(time.time()) > fixtures_epoch or False,
  3425. timeout=3, what="next clock second before the probe raise")
  3426. raise_on_dash("Timeline probe: approve the dashboard flow",
  3427. kind="rfc", body="# Probe")
  3428. dash = slot_public(auth, "/", "dashboard")
  3429. pinned = dash.get("pinned", [])
  3430. pinned_ids = [int(b["id"]) for b in pinned]
  3431. check(len(pinned_ids) >= 3 and pinned_ids == sorted(pinned_ids, reverse=True),
  3432. "pinned open blocks order newest-first (%s)" % pinned_ids)
  3433. check({b["block_type"] for b in pinned} >= {"question", "rfc", "review"},
  3434. "pinned cards cover questions, RFCs and reviews")
  3435. check(all(b["question"] and b["task_title"] and int(b["task_id"]) > 0
  3436. for b in pinned)
  3437. and all("answer_action" not in b for b in pinned),
  3438. "pinned cards are link-through only (task title, kind badge, question)")
  3439. activity = dash.get("activity", [])
  3440. epochs = [int(e["at_epoch"]) for e in activity]
  3441. check(0 < len(activity) <= 50 and epochs == sorted(epochs, reverse=True),
  3442. "the timeline shows the last <=50 events newest-first (%d events)"
  3443. % len(activity))
  3444. kinds = {e["kind"] for e in activity}
  3445. check({"created", "started", "dispatch", "comment", "progress", "completed"} <= kinds,
  3446. "the timeline covers status changes, dispatches, comments, progress and"
  3447. " completions (%s)" % sorted(kinds))
  3448. check(any(e["kind"] == "comment" and e["text"].startswith("user:admin:")
  3449. and "fixture comment" in e["text"] for e in activity),
  3450. "comment events carry author + snippet")
  3451. # The freshly raised probe pinned as the newest card and landed
  3452. # atop the timeline as block-raised; answering it records
  3453. # block-answered.
  3454. check(dash["pinned"][0]["question"] == "Timeline probe: approve the dashboard flow"
  3455. and int(dash["pinned"][0]["task_id"]) == dash_task,
  3456. "the freshest open block pins as the first card")
  3457. check(dash["activity"][0]["kind"] == "block-raised"
  3458. and "Timeline probe" in dash["activity"][0]["text"],
  3459. "the raised block lands atop the timeline")
  3460. probe_block = [b for b in task_state(auth, dash_task)["blocks"]
  3461. if b["question"] == "Timeline probe: approve the dashboard flow"][0]
  3462. act(auth, probe_block["answer_action"], {"answer": "Ship the dashboard"})
  3463. dash = slot_public(auth, "/", "dashboard")
  3464. activity = dash.get("activity", [])
  3465. raised_at = next((i for i, e in enumerate(activity)
  3466. if e["kind"] == "block-raised" and "Timeline probe" in e["text"]), -1)
  3467. answered_at = next((i for i, e in enumerate(activity)
  3468. if e["kind"] == "block-answered"
  3469. and "Ship the dashboard" in e["text"]), -1)
  3470. check(answered_at >= 0 and raised_at >= 0 and answered_at < raised_at,
  3471. "the answered event sorts above its raised event (newest-first, same source)")
  3472. check(all(b["question"] != "Timeline probe: approve the dashboard flow"
  3473. for b in dash["pinned"]),
  3474. "the answered block leaves the pinned cards")
  3475. print("== phase 3 regression: a slow ask after a wake is never dropped ==")
  3476. # The daemon worker's lost-wake-guard regression: a saddle whose
  3477. # ask arrives SLOWLY after the wake (busy winding down when the
  3478. # wake lands) must not lose the wake — the delayed ask (inside
  3479. # the guard window) claims the task, no re-wake storm fires
  3480. # while it is pending, and the saddle stays healthy. The
  3481. # never-asks path (guard -> offline + re-wake of the next
  3482. # capable) is proven by the racer-three section above.
  3483. check(wait_for(lambda: queued_count() == 0 or False,
  3484. timeout=60, what="queue settles before the slow-wake run") is not None,
  3485. "the queue settles before the slow-wake run")
  3486. status, _ = fake2.ask_now()
  3487. check(status == 204, "the slow-ask racer sleeps before its wake")
  3488. saddles_state = slot_public(auth, "/saddles", "saddles")
  3489. act(auth, saddles_state["create_capability"], {"name": "slowwake"})
  3490. slowwake_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")
  3491. ["capabilities"] if c["name"] == "slowwake"][0])
  3492. racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3493. if s["name"] == "racer-two"][0]
  3494. act(auth, racer2_card["attach_capability"], {"capability_id": str(slowwake_cap)})
  3495. fake2.ask_delay = 1.2 # slow: asks well after the wake, inside the 2s guard
  3496. fake2.ask_on_wake = True
  3497. wakes_before_slow = len(fake2.wake_log)
  3498. task_id_tmp = create_queued(auth, {
  3499. "project_id": "1",
  3500. "title": "Slow wake task",
  3501. "priority": "2",
  3502. "capability_id": [str(slowwake_cap)],
  3503. })
  3504. slow_task = task_id_tmp
  3505. check(wait_for(lambda: len(fake2.wake_log) > wakes_before_slow or False,
  3506. timeout=30, what="the slow-ask racer woken") is not None,
  3507. "the only capable sleeping saddle is woken")
  3508. check(wait_task_done(auth, slow_task, timeout=120) is not None,
  3509. "the delayed ask claims the task — the wake was not dropped")
  3510. check(any(a.get("status") == 200
  3511. and int((a.get("payload") or {}).get("task", {}).get("id") or 0)
  3512. == slow_task for a in fake2.asks),
  3513. "the slow ask's 200 carries the task")
  3514. check(len(fake2.wake_log) == wakes_before_slow + 1,
  3515. "the pending-wake skip re-wakes nobody while the slow ask is in flight")
  3516. racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  3517. if s["name"] == "racer-two"][0]
  3518. check(racer2_card["status"] in ("idle", "busy"),
  3519. "the slow asker never went offline (guard did not fire)")
  3520. fake2.ask_delay = 0.0
  3521. fake2.ask_on_wake = False # stays out of the pool below
  3522. # =============================================================================
  3523. # Bundle: lifecycle — offline requeues, the hung-request tick survival,
  3524. # parked-refresh bookkeeping, cancellation, the health panel, raw-log
  3525. # losslessness, git identity and the live blocked-push roundtrip.
  3526. # =============================================================================
  3527. def bundle_lifecycle(ctx):
  3528. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  3529. app = ctx.app
  3530. print("== plan 11: draft state machine, edit matrix, enqueue/delete guards ==")
  3531. # New tasks default to draft; Enqueue is the only path to the
  3532. # queue (there is no agent path — the ingest API refuses to queue).
  3533. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3534. "project_id": "1",
  3535. "title": "Draft lifecycle task",
  3536. "priority": "4",
  3537. })
  3538. draft_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3539. if t["title"] == "Draft lifecycle task"][0])
  3540. detail = task_state(auth, draft_task)
  3541. check(detail["status"] == "draft", "a created task lands in draft")
  3542. check(detail.get("enqueue") is not None and detail.get("delete") is not None,
  3543. "draft exposes the enqueue + delete actions")
  3544. check(detail.get("cancel") is not None, "the cancel action stays authored")
  3545. status, page = http("GET", "/tasks/%d" % draft_task, timeout=15)
  3546. check(status == 200 and "Enqueue task" in page and ">Delete<" in page,
  3547. "the task page shows Enqueue + Delete in draft")
  3548. check("Mark done" not in page,
  3549. "the user Mark-complete action is gone (agent-only completion)")
  3550. # Plan 12: the auto-enqueue-after predecessor REPLACES draft
  3551. # dependencies — setting it is allowed while the predecessor is
  3552. # still pending (the enqueue itself succeeds; the DISPATCHER holds
  3553. # dispatch back until the predecessor is done).
  3554. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3555. "project_id": "1",
  3556. "title": "Draft dependency",
  3557. "priority": "4",
  3558. })
  3559. draft_dep = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3560. if t["title"] == "Draft dependency"][0])
  3561. act(auth, detail["update"], {"enqueue_after_task_id": str(draft_dep)})
  3562. check(int(task_state(auth, draft_task).get("enqueue_after_task_id") or 0)
  3563. == draft_dep,
  3564. "the draft edit sets the auto-enqueue predecessor")
  3565. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3566. "project_id": "1", "title": "Eager dependent", "priority": "4",
  3567. "enqueue_after_task_id": str(draft_dep)})
  3568. eager = task_id_of("Eager dependent")
  3569. enqueue_task(auth, eager)
  3570. check(task_state(auth, eager)["status"] == "queued",
  3571. "enqueue succeeds while the predecessor is pending (no draft guard)")
  3572. check(quiet_window(quiet_seconds(),
  3573. lambda: task_state(auth, eager)["status"] == "queued",
  3574. "pending predecessor holds dispatch"),
  3575. "the pending predecessor holds dispatch back (never assigned)")
  3576. # The edit matrix starts in DRAFT: every field except status.
  3577. act(auth, detail["update"], {
  3578. "title": "Draft lifecycle task (edited)",
  3579. "description": "Now with a description",
  3580. "priority": "2",
  3581. "task_branch": "feature/draft",
  3582. "branch_from": "main",
  3583. "mode": "investigate",
  3584. })
  3585. detail = task_state(auth, draft_task)
  3586. check(detail["title"] == "Draft lifecycle task (edited)"
  3587. and detail["description"] == "Now with a description"
  3588. and detail["priority"] == 2
  3589. and detail["task_branch"] == "feature/draft"
  3590. and detail["branch_from"] == "main"
  3591. and detail["mode"] == "investigate",
  3592. "draft edits apply to every field (incl. the mode)")
  3593. # Clear the predecessor (the empty sentinel): dispatch gates on
  3594. # the hold tag alone now.
  3595. act(auth, detail["update"], {"enqueue_after_task_id": "0"})
  3596. check(not int(task_state(auth, draft_task).get("enqueue_after_task_id") or 0),
  3597. "the draft edit clears the auto-enqueue predecessor")
  3598. # A hold tag nothing carries keeps the dependent QUEUED (never
  3599. # dispatched) through the remaining edit-matrix checks.
  3600. saddles_state = slot_public(auth, "/saddles", "saddles")
  3601. act(auth, saddles_state["create_capability"], {"name": "holdlock"})
  3602. holdlock_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  3603. if c["name"] == "holdlock"][0])
  3604. act(auth, detail["attach_capability"], {"capability_id": str(holdlock_cap)})
  3605. # Enqueue the now-cleared dependent: the hold tag keeps it queued
  3606. # (never dispatched), so the non-draft edit matrix applies.
  3607. enqueue_task(auth, draft_task)
  3608. check(quiet_window(3, lambda: task_state(auth, draft_task)["status"] == "queued",
  3609. "held dependent stays queued"),
  3610. "the held dependent stays queued (never dispatched)")
  3611. # The predecessor settles for the shard drain.
  3612. enqueue_task(auth, draft_dep)
  3613. check(wait_task_done(auth, draft_dep, timeout=90) is not None,
  3614. "the predecessor settles")
  3615. # Drafts are never dispatched even with an idle saddle available.
  3616. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3617. "project_id": "1",
  3618. "title": "Never-dispatched draft",
  3619. "priority": "1",
  3620. })
  3621. silent_draft = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3622. if t["title"] == "Never-dispatched draft"][0])
  3623. check(quiet_window(quiet_seconds(),
  3624. lambda: task_state(auth, silent_draft)["status"] == "draft",
  3625. "draft never dispatched"),
  3626. "an idle saddle never picks a draft up (no agent enqueue path)")
  3627. # Once work can begin (queued here), only priority is editable.
  3628. act(auth, detail["update"], {
  3629. "title": "Should not apply",
  3630. "priority": "3",
  3631. })
  3632. detail = task_state(auth, draft_task)
  3633. check(detail["title"] == "Draft lifecycle task (edited)" and detail["priority"] == 3,
  3634. "non-draft edits apply priority only (title locked)")
  3635. # A terminal task edits nothing.
  3636. conn = db()
  3637. conn.execute("update tasks set status = 'done', completed_at = strftime('%s','now')"
  3638. " where id = ?", (draft_task,))
  3639. conn.commit()
  3640. conn.close()
  3641. act(auth, task_state(auth, draft_task)["update"], {"priority": "8"})
  3642. check(task_state(auth, draft_task)["priority"] == 3,
  3643. "terminal tasks edit nothing (priority locked)")
  3644. # Capabilities lock once a worker has the job: editable while
  3645. # queued, locked once working.
  3646. act(auth, saddles_state["create_capability"], {"name": "editlock"})
  3647. editlock_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")
  3648. ["capabilities"] if c["name"] == "editlock"][0])
  3649. act(auth, saddle_card(auth, "fake-saddle")["attach_capability"],
  3650. {"capability_id": str(editlock_cap)})
  3651. locked_task = create_queued(auth, {"project_id": "1", "title": "Caplock task",
  3652. "priority": "2"})
  3653. detail = task_state(auth, locked_task)
  3654. act(auth, detail["attach_capability"], {"capability_id": str(editlock_cap)})
  3655. check(wait_for(lambda: any(c["name"] == "editlock"
  3656. for c in task_state(auth,
  3657. locked_task)["capabilities"])
  3658. or False, timeout=60, what="editlock attached") is not None,
  3659. "capabilities attach while queued (no worker yet)")
  3660. check(wait_for(lambda: task_state(auth, locked_task).get("status") == "working"
  3661. or False, timeout=60, what="caplock working") is not None,
  3662. "the caplock task reaches working")
  3663. detach_directives = act(auth, task_state(auth, locked_task)["capabilities"][0]["detach"], {})
  3664. check(any(d.get("type") == "notify" and d.get("kind") == "error"
  3665. for d in detach_directives),
  3666. "capabilities are locked once a worker has the job")
  3667. # The ingest API has no queue transition either (only working and
  3668. # blocked are accepted states).
  3669. conn = db()
  3670. caplock_session = conn.execute(
  3671. "select id from sessions where task_id = ? and ended_at is null",
  3672. (locked_task,)).fetchone()
  3673. conn.close()
  3674. if caplock_session is not None:
  3675. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  3676. {"session_id": caplock_session[0],
  3677. "state": "queued"})
  3678. check(status == 200
  3679. and task_state(auth, locked_task)["status"] in ("working", "done"),
  3680. "a queued state push is a no-op (only the wire's states drive bookkeeping)")
  3681. # Delete guards: a clean draft deletes; non-draft refuses; a draft
  3682. # with dependents is refused naming them.
  3683. act(auth, task_state(auth, silent_draft)["delete"], {})
  3684. check(not any(t["id"] == silent_draft
  3685. for t in slot_public(auth, "/tasks", "tasks")["tasks"]),
  3686. "a clean draft task deletes")
  3687. delete_directives = act(auth, task_state(auth, draft_task)["delete"], {})
  3688. check(any(d.get("type") == "notify" and d.get("kind") == "error"
  3689. for d in delete_directives),
  3690. "a non-draft task refuses the delete")
  3691. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3692. "project_id": "1", "title": "Guarded draft", "priority": "4"})
  3693. guarded = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3694. if t["title"] == "Guarded draft"][0])
  3695. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3696. "project_id": "1", "title": "Dependent namer", "priority": "4"})
  3697. namer = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3698. if t["title"] == "Dependent namer"][0])
  3699. act(auth, task_state(auth, namer)["update"],
  3700. {"enqueue_after_task_id": str(guarded)})
  3701. check(int(task_state(auth, namer).get("enqueue_after_task_id") or 0) == guarded,
  3702. "the namer points at the guarded predecessor")
  3703. act(auth, task_state(auth, guarded)["delete"], {})
  3704. check(not any(t["id"] == guarded
  3705. for t in slot_public(auth, "/tasks", "tasks")["tasks"]),
  3706. "deleting a predecessor succeeds")
  3707. check(not int(task_state(auth, namer).get("enqueue_after_task_id") or 0),
  3708. "the delete clears the dependent's dangling predecessor reference")
  3709. print("== offline requeues ==")
  3710. # Tasks caught working on a saddle that went offline requeue with a
  3711. # agent comment each tick.
  3712. task_id_tmp = create_queued(auth, {
  3713. "project_id": "1",
  3714. "title": "Offline saddle task",
  3715. "priority": "2",
  3716. })
  3717. offline_task = task_id_tmp
  3718. check(wait_for(lambda: task_state(auth, offline_task).get("status") == "working"
  3719. or False, timeout=60, what="offline task starts working") is not None,
  3720. "task reaches working before its saddle disappears")
  3721. # Hold the daemon mid-round across the requeue window (the
  3722. # original relied on its 8s work period outlasting the 5s tick):
  3723. # the task completing on its own would race the offline pass and
  3724. # skip the requeue the check exists to observe.
  3725. fake.hold_state = True
  3726. conn = db()
  3727. conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
  3728. conn.commit()
  3729. conn.close()
  3730. # The requeue frees the saddle in the same tick, so 'queued' is
  3731. # transient; the durable evidence is the agent comment (and the task
  3732. # completing again on the healthy daemon).
  3733. check(wait_for(lambda: any(c["body"] == "saddle went offline; task requeued"
  3734. for c in task_state(auth, offline_task).get("comments", []))
  3735. or False, timeout=30, what="offline requeue comment") is not None,
  3736. "offline saddle requeues the task")
  3737. fake.hold_state = False # release: the round ends, the daemon re-asks
  3738. check(wait_task_done(auth, offline_task, timeout=60) is not None,
  3739. "requeued task completes again on the healthy daemon")
  3740. # R2 (arch 13): the release owner's outcomes — the requeue closed
  3741. # the held attempt's session 'stopped'; the held daemon's terminal
  3742. # push then completes the task against that closed session (the
  3743. # outcome stands — a closed row never reopens). A task that
  3744. # completes a LIVE round closes its session 'done' (draft_dep
  3745. # above ran one open session to its terminal push).
  3746. conn = db()
  3747. outcome_rows = conn.execute(
  3748. "select outcome from sessions where task_id = ? order by id",
  3749. (offline_task,)).fetchall()
  3750. dep_outcome = conn.execute(
  3751. "select outcome from sessions where task_id = ? order by id desc",
  3752. (draft_dep,)).fetchone()
  3753. conn.close()
  3754. check(outcome_rows and outcome_rows[0][0] == "stopped",
  3755. "the one release path closes the requeued attempt 'stopped'")
  3756. check(dep_outcome is not None and dep_outcome[0] == "done",
  3757. "the terminal push closes a live round's session 'done'")
  3758. print("== R6 F1: a hanging request cannot freeze the tick ==")
  3759. # One wedged /task-update (never responds) blocks the comment/
  3760. # subtask-event delivery pass; the Soup timeout must fail it and
  3761. # later ticks keep dispatching.
  3762. conn = db()
  3763. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  3764. " values (1, 'Hang survivor', 'queued', 2, strftime('%s','now'))")
  3765. hang_task = conn.execute("select max(id) from tasks").fetchone()[0]
  3766. conn.close()
  3767. task_id_tmp = create_queued(auth, {
  3768. "project_id": "1",
  3769. "title": "Hang runner",
  3770. "priority": "3",
  3771. })
  3772. hang_runner = task_id_tmp
  3773. # A human comment on the running task gives the tick a
  3774. # /task-update delivery to make; the next one hangs.
  3775. check(wait_for(lambda: task_state(auth, hang_runner).get("status") == "working"
  3776. or False, timeout=60, what="hang runner working") is not None,
  3777. "hang runner reaches working")
  3778. act(auth, task_state(auth, hang_runner)["comment"],
  3779. {"body": "operator ping while wedged"})
  3780. fake.hang_next["/task-update"] = 1
  3781. check(wait_task_done(auth, hang_runner, timeout=90) is not None,
  3782. "dispatching task survives a hung daemon request")
  3783. check(wait_for(lambda: any(c.get("text") == "operator ping while wedged"
  3784. or c.get("body") == "operator ping while wedged"
  3785. for c in fake.comment_log) or False,
  3786. timeout=30, what="wedged comment eventually delivered") is not None,
  3787. "the comment delivery retries after the hang")
  3788. # The hung delivery itself fails after the timeout; the system
  3789. # kept moving either way.
  3790. with open(os.path.join(WORK, "app.log")) as handle:
  3791. app_log = handle.read()
  3792. check("tick exceeded" not in app_log or "Dispatch tick failed" in app_log,
  3793. "a hung request never freezes the tick latch")
  3794. print("== R6 F2: refresh must not corrupt parked bookkeeping ==")
  3795. fake.suspend_on_title = "Parked refresh parent"
  3796. task_id_tmp = create_queued(auth, {
  3797. "project_id": "1",
  3798. "title": "Parked refresh parent",
  3799. "priority": "2",
  3800. })
  3801. parked = task_id_tmp
  3802. check(wait_for(lambda: task_state(auth, parked).get("status") == "suspended"
  3803. or False, timeout=60, what="parked parent suspends") is not None,
  3804. "parent parks before the refresh")
  3805. fake_card = saddle_card(auth, "fake-saddle")
  3806. act(auth, fake_card["refresh"], {})
  3807. fake_card = saddle_card(auth, "fake-saddle")
  3808. check(fake_card["status"] == "idle" and int(fake_card["current_task_id"] or 0) == 0,
  3809. "refresh on a parked (suspended) daemon leaves the saddle idle")
  3810. # The parked task keeps its open session and stays put: plain
  3811. # tasks have no subtask events to resume them (the goal machinery
  3812. # owns resumes); the human resumes it explicitly.
  3813. check(quiet_window(quiet_seconds(),
  3814. lambda: task_state(auth, parked)["status"] == "suspended",
  3815. "parked task stays parked"),
  3816. "a parked plain task stays parked after the refresh (no resume channel)")
  3817. act(auth, task_state(auth, parked)["cancel"], {})
  3818. check(wait_for(lambda: task_state(auth, parked)["status"] == "failed" or False,
  3819. timeout=30, what="parked task cancelled") is not None,
  3820. "the parked task cancels cleanly after the refresh")
  3821. print("== R7 B1: cancel reaches the daemon ==")
  3822. # Plan 10 phase 2: cancellation is POST /task-update {kind: cancel}
  3823. # to the RUNNING daemon (the abort machinery, workspace kept).
  3824. task_id_tmp = create_queued(auth, {
  3825. "project_id": "1",
  3826. "title": "Cancel live",
  3827. "priority": "2",
  3828. })
  3829. cancel_live = task_id_tmp
  3830. check(wait_for(lambda: task_state(auth, cancel_live).get("status") == "working"
  3831. or False, timeout=60, what="cancel-live working") is not None,
  3832. "task reaches working before the cancel")
  3833. act(auth, task_state(auth, cancel_live)["cancel"], {})
  3834. check(wait_for(lambda: cancel_live in fake.abort_log or False,
  3835. timeout=30, what="daemon abort recorded") is not None,
  3836. "cancelling POSTs task-update cancel to the daemon")
  3837. check(any(c["body"] == "cancelled; daemon aborted"
  3838. for c in task_state(auth, cancel_live).get("comments", [])),
  3839. "the abort outcome is narrated")
  3840. check(task_state(auth, cancel_live)["status"] == "failed",
  3841. "cancelled task lands failed as before")
  3842. # A 500 on the cancel marks the saddle offline (no fence layer
  3843. # anymore); nothing dispatches to it while offline, and the
  3844. # daemon's own ask flips the row back to idle (proof of life).
  3845. task_id_tmp = create_queued(auth, {
  3846. "project_id": "1",
  3847. "title": "Fence runner",
  3848. "priority": "2",
  3849. })
  3850. fence_runner = task_id_tmp
  3851. check(wait_for(lambda: task_state(auth, fence_runner).get("status") == "working"
  3852. or False, timeout=60, what="fence runner working") is not None,
  3853. "second task reaches working")
  3854. fake.abort_failures = 1
  3855. act(auth, task_state(auth, fence_runner)["cancel"], {})
  3856. check(any(c["body"] == "cancelled; daemon unreachable — saddle marked offline"
  3857. for c in task_state(auth, fence_runner).get("comments", [])),
  3858. "failed cancel narrates the offline marking")
  3859. check(saddle_card(auth, "fake-saddle")["status"] == "offline",
  3860. "saddle marked offline after the failed cancel")
  3861. task_id_tmp = create_queued(auth, {
  3862. "project_id": "1",
  3863. "title": "During fence",
  3864. "priority": "3",
  3865. })
  3866. during_fence = task_id_tmp
  3867. # Converted sleep(7): a quiet window spanning several dispatch
  3868. # ticks in which the task MUST stay queued — if the offline
  3869. # saddle ever dispatches it, this fails loudly. The daemon is held
  3870. # mid-round for the window (the original relied on its 8s work
  3871. # period outlasting the sleep): a finishing round would ask, prove
  3872. # liveness and flip the offline row — only after the window may it.
  3873. fake.hold_state = True
  3874. check(quiet_window(quiet_seconds(),
  3875. lambda: task_state(auth, during_fence)["status"] == "queued",
  3876. "offline saddle does not dispatch during_fence"),
  3877. "offline saddle is not dispatchable")
  3878. fake.hold_state = False
  3879. # The (unaborted) daemon finishes its round, goes idle-awake and
  3880. # asks — the ask is the proof of life that flips the row back.
  3881. check(wait_for(lambda: saddle_card(auth, "fake-saddle")["status"] == "idle" or False,
  3882. timeout=30, what="ask flips offline to idle") is not None,
  3883. "the daemon's ask flips the offline row back to idle")
  3884. check(wait_task_done(auth, during_fence, timeout=60) is not None,
  3885. "queued work dispatches once the saddle proves it is alive")
  3886. print("== R7 B3: health panel + pushed raw log from the first round ==")
  3887. # A normally dispatched task pushes its chunks immediately: raw
  3888. # rows exist from the first round (the session-4 gap stays
  3889. # fixed without any probe machinery).
  3890. task_id_tmp = create_queued(auth, {
  3891. "project_id": "1",
  3892. "title": "Raw from first push",
  3893. "priority": "2",
  3894. })
  3895. raw_first = task_id_tmp
  3896. def raw_count(task_id):
  3897. conn = db()
  3898. try:
  3899. return conn.execute(
  3900. "select count(*) from session_raw_log where session_id in"
  3901. " (select id from sessions where task_id = ?)", (task_id,)).fetchone()[0]
  3902. finally:
  3903. conn.close()
  3904. check(wait_for(lambda: raw_count(raw_first) > 0 or False,
  3905. timeout=60, what="raw rows from first push") is not None,
  3906. "pushed chunks land from the first round (session-4 gap)")
  3907. check(wait_task_done(auth, raw_first, timeout=60) is not None,
  3908. "raw-first task completes")
  3909. # Health panel: a manual refresh populates the snapshot.
  3910. fake_card = saddle_card(auth, "fake-saddle")
  3911. act(auth, fake_card["refresh"], {})
  3912. card = saddle_card(auth, "fake-saddle")
  3913. check(card.get("health") in ("healthy", "stale")
  3914. and card.get("daemon_state") in ("working", "idle", "done")
  3915. and card.get("uptime_display", "") != ""
  3916. and card.get("last_contact_display", "") not in ("", "never"),
  3917. "saddles page shows daemon state, uptime and last contact")
  3918. print("== R8: pushed chunk ingestion is lossless ==")
  3919. # The daemon's opening flush: a coloured prompt run, plain text,
  3920. # a bare CRLF between prompts and the tail — every non-empty
  3921. # chunk must land, exactly once, in seq order. A mid-flight
  3922. # redelivery of the whole batch (the outbox's at-least-once
  3923. # retry) consumes the duplicates.
  3924. corpus = [
  3925. {"seq": 1, "text": "\x1b[01;32muser@host\x1b[00m:\x1b[01;34m~/task-1\x1b[00m$ "},
  3926. {"seq": 2, "text": "echo hello\r\n"},
  3927. {"seq": 3, "text": "\r\n"},
  3928. {"seq": 4, "text": "hello\r\n"},
  3929. {"seq": 5, "text": "\x1b[01;32muser@host\x1b[00m:\x1b[01;34m~/task-1\x1b[00m$ "},
  3930. {"seq": 6, "text": "done\r\n"},
  3931. ]
  3932. fake.scripted_corpus = corpus
  3933. fake.hold_state = True
  3934. task_id_tmp = create_queued(auth, {
  3935. "project_id": "1",
  3936. "title": "Lossless log task",
  3937. "priority": "2",
  3938. })
  3939. lossless = task_id_tmp
  3940. def stored_seqs(task_id):
  3941. conn = db()
  3942. try:
  3943. return [r[0] for r in conn.execute(
  3944. "select seq from session_raw_log where session_id in"
  3945. " (select id from sessions where task_id = ?)"
  3946. " order by seq", (task_id,))]
  3947. finally:
  3948. conn.close()
  3949. check(wait_for(lambda: task_state(auth, lossless).get("status") == "working"
  3950. or False, timeout=60, what="lossless task working") is not None,
  3951. "lossless task dispatches and pushes its opening flush")
  3952. check(wait_for(lambda: stored_seqs(lossless) == [1, 2, 3, 4, 5, 6] or False,
  3953. timeout=30, what="all six chunks stored") is not None,
  3954. "every pushed chunk lands in session_raw_log, CRLF-only included (%s)"
  3955. % stored_seqs(lossless))
  3956. status, _ = saddle_api("POST", "/api/saddle/chunks", fake.token,
  3957. {"session_id": fake.session_id,
  3958. "chunks": [dict(chunk) for chunk in corpus]})
  3959. check(status == 200 and stored_seqs(lossless) == [1, 2, 3, 4, 5, 6],
  3960. "a full-batch redelivery stores nothing new (exactly-once)")
  3961. request = urllib.request.Request(
  3962. WEB + "/terminal-stream/%d" % [r[0] for r in db().execute(
  3963. "select id from sessions where task_id = ?", (lossless,))][0])
  3964. with urllib.request.urlopen(request, timeout=15) as stream:
  3965. texts = []
  3966. deadline = time.monotonic() + 8
  3967. while time.monotonic() < deadline and len(texts) < 6:
  3968. line = stream.readline().decode().strip()
  3969. if not line.startswith("data: ") or line == "data: ping":
  3970. continue
  3971. event = json.loads(line[6:])
  3972. if event.get("type") == "raw":
  3973. texts.append(event.get("text") or "")
  3974. check("\r\n" in texts and len(texts) == 6,
  3975. "SSE serves the whitespace chunk unfiltered")
  3976. fake.hold_state = False
  3977. fake.scripted_corpus = None
  3978. check(wait_task_done(auth, lossless, timeout=60) is not None,
  3979. "lossless task completes")
  3980. print("== R9 B1/B2: git identity + ssh key ==")
  3981. # The key travels on /status into the saddle row and the page view
  3982. # (the manual refresh's exceptional probe).
  3983. fake.ssh_public_key = "ssh-ed25519 AAAAC3NzaC1lZDI1teStKey saddle-1@fedora"
  3984. fake_card = saddle_card(auth, "fake-saddle")
  3985. check(fake_card.get("ssh_public_key", "") == "",
  3986. "no key shown before the daemon reports one")
  3987. act(auth, fake_card["refresh"], {})
  3988. check(wait_for(lambda: "AAAAC3NzaC1lZDI1teStKey" in
  3989. saddle_card(auth, "fake-saddle").get("ssh_public_key", "") or False,
  3990. timeout=30, what="key lands on the row") is not None,
  3991. "reported ssh_public_key persists onto the saddle row")
  3992. conn = db()
  3993. row_key = conn.execute("select ssh_public_key from saddles where name = 'fake-saddle'"
  3994. ).fetchone()[0]
  3995. conn.close()
  3996. check(row_key == fake.ssh_public_key,
  3997. "the key is stored verbatim in the database")
  3998. # Identity edit flows through the UI into the assignment payload.
  3999. fake_card = saddle_card(auth, "fake-saddle")
  4000. act(auth, fake_card["update_identity"],
  4001. {"git_name": "clanker-bot", "git_email": "bot@unitatem.net"})
  4002. fake_card = saddle_card(auth, "fake-saddle")
  4003. check(fake_card.get("git_name") == "clanker-bot"
  4004. and fake_card.get("git_email") == "bot@unitatem.net",
  4005. "inline edit saves the git identity")
  4006. task_id_tmp = create_queued(auth, {
  4007. "project_id": "1",
  4008. "title": "Identity task",
  4009. "priority": "2",
  4010. })
  4011. identity_task = task_id_tmp
  4012. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == identity_task
  4013. for p in fake.start_payloads) or False,
  4014. timeout=60, what="identity task dispatched") is not None,
  4015. "identity task dispatches")
  4016. payload = [p for p in fake.start_payloads
  4017. if int(p.get("task", {}).get("id") or 0) == identity_task][-1]
  4018. identity = payload.get("git_identity") or {}
  4019. check(identity.get("name") == "clanker-bot"
  4020. and identity.get("email") == "bot@unitatem.net",
  4021. "the assignment carries the saddle's git_identity")
  4022. # Earlier assignments (identity unset) carry no git_identity member.
  4023. check(all("git_identity" not in p for p in fake.start_payloads[:-1]),
  4024. "unset identity omits the member (daemon fallback applies)")
  4025. # Clearing the identity omits it from later assignments.
  4026. fake_card = saddle_card(auth, "fake-saddle")
  4027. act(auth, fake_card["update_identity"], {"git_name": "", "git_email": ""})
  4028. check(saddle_card(auth, "fake-saddle").get("git_name", "") == "",
  4029. "clearing the identity empties the row")
  4030. check(wait_task_done(auth, identity_task, timeout=60) is not None,
  4031. "identity task completes")
  4032. print("== phase 2: blocked push -> block row + pulse ==")
  4033. # The browser flow on each pulse: statum.act(task.refresh) — the
  4034. # sealed GET action must rehydrate the task slot with the new
  4035. # block the daemon just PUSHED (state blocked + block payload).
  4036. task_id_tmp = create_queued(auth, {
  4037. "project_id": "1",
  4038. "title": "Live block task",
  4039. "priority": "2",
  4040. })
  4041. live_task = task_id_tmp
  4042. check(wait_for(lambda: task_state(auth, live_task).get("status") == "working"
  4043. or False, timeout=60, what="live task working") is not None,
  4044. "live-block task reaches working")
  4045. fake.hold_state = True
  4046. fake.state = "blocked" # the daemon parks on its question
  4047. status = fake.push_results("blocked", block={
  4048. "question": "Pick one", "type": "question", "options": ["A", "B"]})
  4049. check(status == 200, "blocked state push accepted")
  4050. # The parked daemon is idle-awake: it asks (and sleeps on 204).
  4051. fake.maybe_trigger_ask("parked")
  4052. check(wait_for(lambda: any(b.get("question") == "Pick one"
  4053. for b in task_state(auth, live_task).get("blocks", []))
  4054. or False, timeout=30, what="block lands in state") is not None,
  4055. "the pushed block reaches the sealed refresh without a reload")
  4056. detail = task_state(auth, live_task)
  4057. check(detail.get("status") == "blocked",
  4058. "the blocked push flips the task to blocked")
  4059. pushed_block = [b for b in detail["blocks"] if b.get("question") == "Pick one"][0]
  4060. conn = db()
  4061. options_json = conn.execute(
  4062. "select options from task_blocks where id = ?", (int(pushed_block["id"]),)
  4063. ).fetchone()[0]
  4064. conn.close()
  4065. check(json.loads(options_json) == ["A", "B"],
  4066. "the block row carries its options payload")
  4067. check(any(s["status"] == "idle"
  4068. for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
  4069. "the blocked push parks the saddle")
  4070. detail = task_state(auth, live_task)
  4071. check(detail.get("refresh") is not None,
  4072. "the sealed GET refresh action is still authored")
  4073. status, _ = http("GET", detail["refresh"]["uri"], timeout=15)
  4074. check(status in (200, 401),
  4075. "the refresh URI resolves as a GET (client roundtrip shape)")
  4076. act(auth, pushed_block["answer_action"], {"answer": "A"})
  4077. # The answer makes the parked task resume-offerable: the affine
  4078. # saddle is woken, asks, and its assignment carries the answer.
  4079. pushes_before = len([p for p in fake.start_payloads
  4080. if int(p.get("task", {}).get("id") or 0) == live_task])
  4081. check(wait_for(lambda: len([p for p in fake.start_payloads
  4082. if int(p.get("task", {}).get("id") or 0) == live_task])
  4083. > pushes_before or False, timeout=60,
  4084. what="blocked task resumes") is not None,
  4085. "answering the pushed block resumes the parked workspace")
  4086. fake.hold_state = False # the resumed round runs to completion
  4087. resume_payload = [p for p in fake.start_payloads
  4088. if int(p.get("task", {}).get("id") or 0) == live_task][-1]
  4089. answers = resume_payload.get("task", {}).get("answers", [])
  4090. check(any(a.get("answer") == "A" for a in answers),
  4091. "the resume assignment carries the human's answer")
  4092. check(wait_task_done(auth, live_task, timeout=60) is not None,
  4093. "live-block task completes after answering")
  4094. print("== R2: the one release path behind unregister ==")
  4095. # Arch 13: removing a saddle releases its held task through the
  4096. # same owner as every requeue — never an orphan open session on a
  4097. # deleted row. Route 1: a DEAD host — the abort delivery fails
  4098. # fast, the offline bookkeeping releases (queued + comment +
  4099. # session 'stopped'), the unregister then deletes the row.
  4100. saddles_state = slot_public(auth, "/saddles", "saddles")
  4101. act(auth, saddles_state["register"], {"name": "dead-host",
  4102. "host": "127.0.0.1:1"})
  4103. dead_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  4104. if s["name"] == "dead-host"][0]
  4105. dead_id = int(dead_card["id"])
  4106. task_id_tmp = create_queued(auth, {
  4107. "project_id": "1",
  4108. "title": "Dead-host unregister task",
  4109. "priority": "2",
  4110. })
  4111. dead_task = task_id_tmp
  4112. conn = db()
  4113. conn.execute("update tasks set status = 'working' where id = ?",
  4114. (dead_task,))
  4115. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  4116. " values (?, ?, strftime('%s','now'))", (dead_task, dead_id))
  4117. conn.execute("update saddles set current_task_id = ? where id = ?",
  4118. (dead_task, dead_id))
  4119. conn.commit()
  4120. dead_session = conn.execute(
  4121. "select id from sessions where task_id = ? and ended_at is null",
  4122. (dead_task,)).fetchone()[0]
  4123. conn.close()
  4124. act(auth, dead_card["unregister"], {})
  4125. conn = db()
  4126. dead_row = conn.execute("select status from tasks where id = ?",
  4127. (dead_task,)).fetchone()
  4128. dead_session_row = conn.execute(
  4129. "select outcome, ended_at is not null from sessions where id = ?",
  4130. (dead_session,)).fetchone()
  4131. dead_saddle_count = conn.execute(
  4132. "select count(*) from saddles where id = ?", (dead_id,)).fetchone()[0]
  4133. conn.close()
  4134. check(dead_row is not None and dead_row[0] == "queued",
  4135. "unregister of a dead-host saddle releases the held task to the queue")
  4136. check(dead_session_row is not None and dead_session_row[0] == "stopped"
  4137. and dead_session_row[1],
  4138. "the offline release closes the held task's session 'stopped'")
  4139. check(dead_saddle_count == 0, "the dead-host saddle row is gone")
  4140. # Route 2: no open session on the saddle — the abort is a no-op
  4141. # and the unregister itself carries the release (the requeue flip
  4142. # through the one owner, the row deleted after).
  4143. saddles_state = slot_public(auth, "/saddles", "saddles")
  4144. act(auth, saddles_state["register"], {"name": "quiet-host",
  4145. "host": "127.0.0.1:%d" % fake.port})
  4146. quiet_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  4147. if s["name"] == "quiet-host"][0]
  4148. quiet_id = int(quiet_card["id"])
  4149. task_id_tmp = create_queued(auth, {
  4150. "project_id": "1",
  4151. "title": "Quiet-host unregister task",
  4152. "priority": "2",
  4153. })
  4154. quiet_task = task_id_tmp
  4155. conn = db()
  4156. conn.execute("update tasks set status = 'working' where id = ?",
  4157. (quiet_task,))
  4158. conn.execute("update saddles set current_task_id = ? where id = ?",
  4159. (quiet_task, quiet_id))
  4160. conn.commit()
  4161. conn.close()
  4162. act(auth, quiet_card["unregister"], {})
  4163. conn = db()
  4164. quiet_row = conn.execute("select status from tasks where id = ?",
  4165. (quiet_task,)).fetchone()
  4166. quiet_saddle_count = conn.execute(
  4167. "select count(*) from saddles where id = ?", (quiet_id,)).fetchone()[0]
  4168. conn.close()
  4169. check(quiet_row is not None and quiet_row[0] == "queued",
  4170. "unregister's own release returns the held task to the queue")
  4171. check(quiet_saddle_count == 0, "the quiet-host saddle row is gone")
  4172. # =============================================================================
  4173. # Bundle: answers — the answer-resume machinery in both daemon shapes
  4174. # (sleeping affine wake, mid-round verdict delivery), offline requeue
  4175. # keeping parked-with-answered tasks, the staleness probe, and the
  4176. # startup token migration (this shard's one deliberate app restart).
  4177. # =============================================================================
  4178. def bundle_answers(ctx):
  4179. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  4180. app = ctx.app
  4181. print("== phase 2: answer resume — wake -> ask -> affinity claim ==")
  4182. # The full live sequence against a daemon that asks after its
  4183. # park: ask -> assignment -> blocked push (park) -> ask -> 204
  4184. # -> sleeps (row idle) -> human answers -> the dispatcher WAKES
  4185. # the sleeping affine saddle -> its ask allocates the resume,
  4186. # the verdict riding the assignment -> the session continues.
  4187. # A private capability tag keeps the fleet's other sleeping
  4188. # saddles from claiming these tasks.
  4189. saddles_state = slot_public(auth, "/saddles", "saddles")
  4190. act(auth, saddles_state["create_capability"], {"name": "answerflow"})
  4191. answerflow_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  4192. if c["name"] == "answerflow"][0])
  4193. def answerflow_daemon(**kwargs):
  4194. daemon = FakeSaddle()
  4195. daemon.ask_on_register = False # manual, deterministic asks
  4196. daemon.hold_state = True # the tests drive the lifecycle
  4197. for key, value in kwargs.items():
  4198. setattr(daemon, key, value)
  4199. daemon.serve()
  4200. return daemon
  4201. def register_answerflow(daemon, name):
  4202. act(auth, slot_public(auth, "/saddles", "saddles")["register"], {
  4203. "name": name, "host": "127.0.0.1:%d" % daemon.port})
  4204. act(auth, saddle_card(auth, name)["attach_capability"],
  4205. {"capability_id": str(answerflow_cap)})
  4206. def create_answerflow_task(title):
  4207. return create_queued(auth, {
  4208. "project_id": "1", "title": title, "priority": "2",
  4209. "capability_id": [str(answerflow_cap)]})
  4210. def retire_daemon(daemon, name):
  4211. # deterministic teardown: a dead daemon and an offline row
  4212. # never interfere with the sections that follow
  4213. daemon.kill()
  4214. act(auth, saddle_card(auth, name)["refresh"], {})
  4215. sleeper = answerflow_daemon(ask_on_wake=True)
  4216. register_answerflow(sleeper, "resume-sleeper")
  4217. sleeping_task = create_answerflow_task("Sleeping resume task")
  4218. ask_status, assignment = sleeper.ask_now()
  4219. check(ask_status == 200 and int(assignment["task"]["id"]) == sleeping_task,
  4220. "the daemon's ask claims the queued task")
  4221. sleeping_session = assignment.get("session_id")
  4222. sleeper.state = "blocked"
  4223. check(sleeper.push_results("blocked", block={
  4224. "question": "Pick a side", "type": "question",
  4225. "options": ["Port", "Starboard"]}) == 200,
  4226. "the blocked push parks the session")
  4227. check(wait_for(lambda: task_state(auth, sleeping_task).get("status") == "blocked"
  4228. or False, timeout=30, what="sleeping task parked") is not None,
  4229. "the task parks as blocked")
  4230. check(saddle_card(auth, "resume-sleeper")["status"] == "idle",
  4231. "the parked session's saddle row reads idle")
  4232. ask_status, _ = sleeper.ask_now()
  4233. check(ask_status == 204,
  4234. "the parked daemon's ask returns 204 — it sleeps until woken")
  4235. # Converted sleep(3): ticks pass with the daemon asleep and no
  4236. # wake may arrive — a wake during the window fails loudly.
  4237. check(quiet_window(quiet_seconds(),
  4238. lambda: len(sleeper.wake_log) == 0,
  4239. "sleeping daemon never woken"),
  4240. "the sleeping daemon is never woken while the block waits")
  4241. detail = task_state(auth, sleeping_task)
  4242. side_block = [b for b in detail["blocks"] if b["question"] == "Pick a side"][0]
  4243. act(auth, side_block["answer_action"], {"option": "Starboard"})
  4244. check(wait_for(lambda: len(sleeper.wake_log) > 0 or False,
  4245. timeout=30, what="wake after the answer") is not None,
  4246. "answering the block wakes the sleeping affine saddle")
  4247. # Answering = allocating (plan 11): the resume assignment carries
  4248. # the verdict as resume_notice (stable wording) AND a FRESH
  4249. # session id — the parked row closed, the daemon keys its resumed
  4250. # pushes on the new one.
  4251. check(wait_for(lambda: any(
  4252. int(p.get("task", {}).get("id") or 0) == sleeping_task
  4253. and p.get("session_id") != sleeping_session
  4254. and p.get("resume_notice") == "Human selected: Starboard"
  4255. and any(a.get("answer") == "Starboard"
  4256. for a in p.get("task", {}).get("answers", []))
  4257. for p in sleeper.start_payloads[1:]) or False,
  4258. timeout=30, what="resume allocation carries the verdict") is not None,
  4259. "the woken daemon's ask allocates the resume — fresh session, notice carried")
  4260. # Barrier: the resume's own working push must commit before the
  4261. # manual done push (a late working write would otherwise race the
  4262. # completion's full-row write).
  4263. check(wait_for(lambda: task_state(auth, sleeping_task).get("status") == "working"
  4264. or False, timeout=15, what="resumed round working") is not None,
  4265. "the resumed round reports working before its completion push")
  4266. # The resumed round runs to completion keyed on the fresh session.
  4267. sleeper.push_results("done", summary="sleep-resumed round complete")
  4268. check(wait_task_done(auth, sleeping_task, timeout=30) is not None,
  4269. "the resumed session continues to completion")
  4270. conn = db()
  4271. rows = conn.execute(
  4272. "select id, outcome, ended_at is not null from sessions where task_id = ?"
  4273. " order by id", (sleeping_task,)).fetchall()
  4274. conn.close()
  4275. check(len(rows) == 2 and rows[0][1] == "resumed" and rows[0][2]
  4276. and rows[1][1] == "done" and rows[1][2],
  4277. "parked session closed 'resumed'; the resumed round ran on a fresh row to done (%s)"
  4278. % (rows,))
  4279. retire_daemon(sleeper, "resume-sleeper")
  4280. print("== phase 2: answer resume — the parked daemon is woken (POST /answer retired) ==")
  4281. # Post Task A every park FREES the daemon: after the blocked push
  4282. # it lands idle and asks (204). The retired POST /answer channel
  4283. # always refuses; the verdict reaches the parked workspace purely
  4284. # through the wake -> ask -> affinity allocation carrying the
  4285. # resume_notice + a fresh session id.
  4286. parked = answerflow_daemon(ask_on_wake=True,
  4287. ask_on_terminal=False)
  4288. register_answerflow(parked, "resume-parked")
  4289. parked_task = create_answerflow_task("Parked resume task")
  4290. ask_status, assignment = parked.ask_now()
  4291. check(ask_status == 200 and int(assignment["task"]["id"]) == parked_task,
  4292. "the daemon's ask claims the task")
  4293. parked_session = assignment.get("session_id")
  4294. parked.state = "blocked"
  4295. check(parked.push_results("blocked", block={
  4296. "question": "Which database?", "type": "question",
  4297. "options": ["SQLite", "Postgres"]}) == 200,
  4298. "the blocked push parks the session mid-round")
  4299. check(wait_for(lambda: task_state(auth, parked_task).get("status") == "blocked"
  4300. or False, timeout=30, what="parked task blocked") is not None,
  4301. "the task parks as blocked")
  4302. check(saddle_card(auth, "resume-parked")["status"] == "idle",
  4303. "the parked session's saddle row reads idle")
  4304. # The freed daemon re-asks (its idle loop) and sleeps on 204.
  4305. ask_status, _ = parked.ask_now()
  4306. check(ask_status == 204,
  4307. "the freed parked daemon asks again and sleeps (204)")
  4308. # The retired channel always refuses — the web never depends on it.
  4309. request = urllib.request.Request(
  4310. "http://127.0.0.1:%d/answer" % parked.port,
  4311. data=json.dumps({"answer": "SQLite", "branch_advanced": False}).encode(),
  4312. method="POST")
  4313. request.add_header("Content-Type", "application/json")
  4314. try:
  4315. urllib.request.urlopen(request, timeout=5)
  4316. refused = False
  4317. except urllib.error.HTTPError as error:
  4318. refused = error.code == 409
  4319. check(refused, "POST /answer always refuses (409) — parks resume via allocation")
  4320. detail = task_state(auth, parked_task)
  4321. db_block = [b for b in detail["blocks"] if b["question"] == "Which database?"][0]
  4322. # branch_advanced on the resume assignment (the /poll verdict
  4323. # channel's hint lives here now): a sibling on the same shared
  4324. # working branch completed strictly after the block's park marker
  4325. # — the resumed allocation must report branch_advanced=true so the
  4326. # daemon knows to pull before continuing.
  4327. conn = db()
  4328. parked_block_created = conn.execute(
  4329. "select created_at from task_blocks where id = ?", (db_block["id"],)
  4330. ).fetchone()[0]
  4331. conn.execute("update tasks set task_branch = 'shared/resume-branch'"
  4332. " where id = ?", (parked_task,))
  4333. conn.commit()
  4334. conn.close()
  4335. wait_for(lambda: int(time.time()) > parked_block_created or False,
  4336. timeout=3, what="next clock second past the park marker")
  4337. conn = db()
  4338. conn.execute("insert into tasks (project_id, title, status, priority,"
  4339. " task_branch, summary, completed_at, created_at)"
  4340. " values (1, 'Resume sibling', 'done', 5, 'shared/resume-branch',"
  4341. " 'sibling finished', strftime('%s','now'),"
  4342. " strftime('%s','now','-300 seconds'))")
  4343. conn.commit()
  4344. conn.close()
  4345. act(auth, db_block["answer_action"], {"option": "SQLite"})
  4346. check(wait_for(lambda: len(parked.wake_log) > 0 or False,
  4347. timeout=30, what="wake after the answer") is not None,
  4348. "answering the block wakes the parked (idle) affine saddle")
  4349. check(wait_for(lambda: any(
  4350. int(p.get("task", {}).get("id") or 0) == parked_task
  4351. and p.get("session_id") != parked_session
  4352. and p.get("resume_notice") == "Human selected: SQLite"
  4353. and p.get("branch_advanced") is True
  4354. for p in parked.start_payloads[1:]) or False,
  4355. timeout=30, what="parked resume allocation") is not None,
  4356. "the parked daemon's ask allocates the resume — fresh session, formatted notice,"
  4357. " branch_advanced=true after the sibling's completion")
  4358. # The hold_state daemon drives its round manually: the resumed
  4359. # round completes on the fresh session row.
  4360. parked.push_results("done", summary="parked round resumed and finished")
  4361. check(wait_task_done(auth, parked_task, timeout=90) is not None,
  4362. "the parked session continues to completion")
  4363. conn = db()
  4364. row = conn.execute("select outcome from sessions where id = ?",
  4365. (parked_session,)).fetchone()
  4366. session_rows = conn.execute(
  4367. "select id, outcome from sessions where task_id = ? order by id",
  4368. (parked_task,)).fetchall()
  4369. conn.close()
  4370. check(row is not None and row[0] == "resumed"
  4371. and len(session_rows) == 2 and session_rows[0] == (parked_session, "resumed"),
  4372. "the parked session closed 'resumed' and the resumed round kept its own row (%s)"
  4373. % (session_rows,))
  4374. retire_daemon(parked, "resume-parked")
  4375. print("== plan 11: plan acceptance — accept / reject / auto-approve ==")
  4376. # A type 'plan' block parks like any other; Accept and
  4377. # Reject-with-feedback drive the same wake -> ask -> allocation
  4378. # resume as any verdict, the stable wording riding the
  4379. # resume_notice. Plan 12: the daemon only SENDS plan blocks on
  4380. # goal runs now (the goal page owns the review); this covers the
  4381. # generic verdict path a goal-run's mirrored block also uses. The
  4382. # auto-approve option is gone.
  4383. planner = answerflow_daemon(ask_on_wake=True, ask_on_terminal=False)
  4384. register_answerflow(planner, "plan-planner")
  4385. def park_on_plan(task_id, markdown):
  4386. planner.state = "blocked"
  4387. check(planner.push_results("blocked", block={
  4388. "question": "Plan submitted for review", "type": "plan",
  4389. "body": markdown}) == 200,
  4390. "the plan push parks the session")
  4391. # Wait for the block row itself (the auto-approve variant is
  4392. # answered — and may resume — before any status poll lands).
  4393. check(wait_for(lambda: any(b["block_type"] == "plan"
  4394. for b in task_state(auth, task_id)["blocks"])
  4395. or False, timeout=30, what="plan block raised") is not None,
  4396. "the plan block reaches the task")
  4397. return [b for b in task_state(auth, task_id)["blocks"]
  4398. if b["block_type"] == "plan"][0]
  4399. # (a) Accept resumes with the approval notice.
  4400. accept_task = create_answerflow_task("Plan accept task")
  4401. ask_status, accept_assignment = planner.ask_now()
  4402. check(ask_status == 200 and int(accept_assignment["task"]["id"]) == accept_task,
  4403. "the planner claims the accept task")
  4404. accept_parked_session = accept_assignment.get("session_id")
  4405. plan_block = park_on_plan(accept_task, "# Steps\n\n1. Wire\n2. Test")
  4406. check(plan_block.get("answered") is not True,
  4407. "the plan block waits for a human verdict")
  4408. # Plan 16 S2: the accept form posts the verdict marker; the server
  4409. # builds the stable constant (byte-identical wire).
  4410. act(auth, plan_block["answer_action"], {"verdict": "accept"})
  4411. check(wait_for(lambda: any(
  4412. int(p.get("task", {}).get("id") or 0) == accept_task
  4413. and p.get("session_id") != accept_parked_session
  4414. and p.get("resume_notice") == "Plan approved. Proceed with implementation."
  4415. for p in planner.start_payloads[1:]) or False,
  4416. timeout=30, what="accept resume allocation") is not None,
  4417. "accept resumes the parked daemon with the approval notice")
  4418. planner.push_results("done", summary="plan accepted and implemented")
  4419. check(wait_task_done(auth, accept_task, timeout=60) is not None,
  4420. "the accepted plan's task completes")
  4421. planner.set_state("idle")
  4422. # (b) Reject-with-feedback loops the feedback notice back.
  4423. reject_task = create_answerflow_task("Plan reject task")
  4424. ask_status, reject_assignment = planner.ask_now()
  4425. check(ask_status == 200 and int(reject_assignment["task"]["id"]) == reject_task,
  4426. "the planner claims the reject task")
  4427. reject_parked_session = reject_assignment.get("session_id")
  4428. plan_block = park_on_plan(reject_task, "# Draft\n\nJust wing it")
  4429. act(auth, plan_block["answer_action"], {"feedback": "Add a test section"})
  4430. reject_block = [b for b in task_state(auth, reject_task)["blocks"]
  4431. if b["block_type"] == "plan"][0]
  4432. check(reject_block.get("answer")
  4433. == "Plan feedback received:\nAdd a test section",
  4434. "reject stores the feedback in the stable wording (%s)"
  4435. % reject_block.get("answer"))
  4436. check(wait_for(lambda: any(
  4437. int(p.get("task", {}).get("id") or 0) == reject_task
  4438. and p.get("session_id") != reject_parked_session
  4439. and p.get("resume_notice") == "Plan feedback received:\nAdd a test section"
  4440. for p in planner.start_payloads[1:]) or False,
  4441. timeout=30, what="reject resume allocation") is not None,
  4442. "reject resumes the parked daemon with the feedback notice")
  4443. planner.push_results("done", summary="plan revised and delivered")
  4444. check(wait_task_done(auth, reject_task, timeout=60) is not None,
  4445. "the revised plan's task completes")
  4446. planner.set_state("idle")
  4447. retire_daemon(planner, "plan-planner")
  4448. print("== phase 2: offline requeue keeps parked-with-answered tasks ==")
  4449. # A daemon dying while its parked task carries an answered
  4450. # verdict: going offline must not requeue the task or close its
  4451. # session — the resume machinery (delivery once the daemon
  4452. # returns, or the affinity allocation on its next ask) needs
  4453. # both. The question park leaves the row idle; the review park
  4454. # keeps it busy@task (`saddle review` blocks the round without
  4455. # a parking state push), exercising the unreachable path's
  4456. # requeue decision both ways.
  4457. stranded = answerflow_daemon()
  4458. register_answerflow(stranded, "resume-stranded")
  4459. stranded_task = create_answerflow_task("Stranded answered task")
  4460. ask_status, _ = stranded.ask_now()
  4461. check(ask_status == 200, "the stranded daemon claims its task")
  4462. stranded.state = "blocked"
  4463. stranded.push_results("blocked", block={
  4464. "question": "Hold or fold?", "type": "question", "options": ["Hold", "Fold"]})
  4465. check(wait_for(lambda: task_state(auth, stranded_task).get("status") == "blocked"
  4466. or False, timeout=30, what="stranded task parked") is not None,
  4467. "the stranded task parks as blocked")
  4468. detail = task_state(auth, stranded_task)
  4469. hold_block = [b for b in detail["blocks"] if b["question"] == "Hold or fold?"][0]
  4470. act(auth, hold_block["answer_action"], {"option": "Hold"})
  4471. stranded.kill() # the daemon dies before the verdict is delivered
  4472. act(auth, saddle_card(auth, "resume-stranded")["refresh"], {})
  4473. check(wait_for(lambda: saddle_card(auth, "resume-stranded")["status"] == "offline"
  4474. or False, timeout=30, what="stranded saddle offline") is not None,
  4475. "the dead daemon's saddle row goes offline")
  4476. # Converted sleep(7): offline ticks run while the task MUST stay
  4477. # parked — a requeue during the window fails loudly.
  4478. check(quiet_window(quiet_seconds(),
  4479. lambda: task_state(auth, stranded_task)["status"] == "blocked",
  4480. "stranded task not requeued"),
  4481. "the parked-with-answered task is not requeued by the offline pass")
  4482. conn = db()
  4483. open_session = conn.execute(
  4484. "select count(*) from sessions where task_id = ? and ended_at is null",
  4485. (stranded_task,)).fetchone()[0]
  4486. conn.close()
  4487. check(open_session == 1,
  4488. "the parked-with-answered task keeps its open session for the resume")
  4489. reviewer = answerflow_daemon(ask_on_terminal=False)
  4490. register_answerflow(reviewer, "resume-reviewer")
  4491. review_task = create_answerflow_task("Review park task")
  4492. ask_status, _ = reviewer.ask_now()
  4493. check(ask_status == 200, "the reviewer claims its task")
  4494. # Post Task A the review park frees the daemon: the blocked push
  4495. # carries the review block and parks the row idle.
  4496. reviewer.state = "blocked"
  4497. check(reviewer.push_results("blocked", block={
  4498. "question": "Review requested", "type": "review",
  4499. "body": "Ready for review: wired and tested."}) == 200,
  4500. "the review park push carries the review block")
  4501. check(wait_for(lambda: task_state(auth, review_task).get("status") == "review_requested"
  4502. or False, timeout=30, what="review park raised") is not None,
  4503. "the review block parks the task as review_requested (row freed)")
  4504. # The verdict lands (nothing delivers it anymore); THEN the daemon
  4505. # dies and the manual refresh offlines it — the parked-with-answered
  4506. # task must keep its session for the eventual resume.
  4507. detail = task_state(auth, review_task)
  4508. verdict_block = [b for b in detail["blocks"]
  4509. if b["block_type"] == "review"][0]
  4510. act(auth, verdict_block["answer_action"], {"verdict": "accept"})
  4511. reviewer.kill()
  4512. act(auth, saddle_card(auth, "resume-reviewer")["refresh"], {})
  4513. check(wait_for(lambda: saddle_card(auth, "resume-reviewer")["status"] == "offline"
  4514. or False, timeout=30, what="reviewer offline") is not None,
  4515. "the dead reviewer goes offline on the failed probe")
  4516. # Converted sleep(7): offline ticks run while the task MUST stay
  4517. # review-parked — a requeue during the window fails loudly.
  4518. check(quiet_window(quiet_seconds(),
  4519. lambda: task_state(auth, review_task)["status"] == "review_requested",
  4520. "review-parked task not requeued"),
  4521. "the review-parked task with an answered verdict is never requeued")
  4522. conn = db()
  4523. open_session = conn.execute(
  4524. "select count(*) from sessions where task_id = ? and ended_at is null",
  4525. (review_task,)).fetchone()[0]
  4526. conn.close()
  4527. check(open_session == 1,
  4528. "the review-parked task keeps its open session for the resume")
  4529. print("== phase 2: staleness — one probe decides ==")
  4530. # A silent-but-alive runner (long think, nothing to flush): after
  4531. # SADDLE_STALE_S (3s here) of silence it owes ONE exceptional
  4532. # GET /status; the answer keeps it running and the silence
  4533. # episode gets no second probe.
  4534. task_id_tmp = create_queued(auth, {
  4535. "project_id": "1",
  4536. "title": "Silent thinker",
  4537. "priority": "2",
  4538. })
  4539. thinker = task_id_tmp
  4540. check(wait_for(lambda: task_state(auth, thinker).get("status") == "working"
  4541. or False, timeout=60, what="thinker working") is not None,
  4542. "silent thinker reaches working")
  4543. fake.silent = True # the daemon goes quiet mid-round (alive)
  4544. probes_at_start = fake.status_hits
  4545. check(wait_for(lambda: fake.status_hits > probes_at_start or False,
  4546. timeout=30, what="exceptional probe arrives") is not None,
  4547. "a silent runner gets its exceptional GET /status probe")
  4548. # Converted sleep(8)+sleep(6): a quiet window across several
  4549. # ticks in which the probe count MUST NOT move — a second probe
  4550. # (the poll drumbeat this guards against) fails loudly.
  4551. second_window = fake.status_hits
  4552. check(quiet_window(quiet_seconds(),
  4553. lambda: fake.status_hits == second_window,
  4554. "no second staleness probe"),
  4555. "the silence episode is probed exactly once (no poll drumbeat)")
  4556. check(task_state(auth, thinker).get("status") == "working"
  4557. and saddle_card(auth, "fake-saddle")["status"] == "busy",
  4558. "the answered probe keeps the task running (no requeue)")
  4559. # Output flows again: the episode ends, the clock resets.
  4560. fake.silent = False
  4561. fake.push_chunks(["resumed output\r\n"])
  4562. check(wait_task_done(auth, thinker, timeout=60) is not None,
  4563. "the silent thinker completes once output flows again")
  4564. # The daemon vanishing mid-task: silence for the window, the
  4565. # probe fails, the task requeues with an agent comment and the
  4566. # saddle goes offline.
  4567. task_id_tmp = create_queued(auth, {
  4568. "project_id": "1",
  4569. "title": "Vanish victim",
  4570. "priority": "2",
  4571. })
  4572. vanish = task_id_tmp
  4573. check(wait_for(lambda: task_state(auth, vanish).get("status") == "working"
  4574. or False, timeout=60, what="vanish victim working") is not None,
  4575. "vanish victim reaches working")
  4576. fake.kill()
  4577. check(wait_for(lambda: saddle_card(auth, "fake-saddle")["status"] == "offline" or False,
  4578. timeout=60, what="daemon death -> offline") is not None,
  4579. "vanishing daemon drives the saddle offline (failed staleness probe)")
  4580. check(wait_for(lambda: any("unreachable" in c["body"]
  4581. and c.get("is_system")
  4582. for c in task_state(auth, vanish).get("comments", []))
  4583. or False, timeout=30, what="unreachable comment") is not None,
  4584. "the affected task gets an unreachable system row (never an agent comment)")
  4585. check(task_state(auth, vanish)["status"] == "queued",
  4586. "the task requeues after the daemon death")
  4587. card = saddle_card(auth, "fake-saddle")
  4588. check(card.get("health") == "offline",
  4589. "health badge reports offline")
  4590. # The unreachable row flag marks the detection window: a task the
  4591. # web still sees working while its saddle is offline/fenced, before
  4592. # the requeue pass moves it. Controlled window (the real
  4593. # transitions above are atomic with their requeues).
  4594. conn = db()
  4595. conn.execute("insert into tasks (project_id, title, status, priority,"
  4596. " created_at) values (1, 'Unreachable row', 'working', 5,"
  4597. " strftime('%s','now'))")
  4598. unreachable_row = conn.execute("select max(id) from tasks").fetchone()[0]
  4599. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  4600. " values (?, ?, strftime('%s','now'))", (unreachable_row, fake_id))
  4601. conn.commit()
  4602. conn.close()
  4603. check(wait_for(lambda: (lambda row: row.get("unreachable") is True)(
  4604. [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  4605. if int(t["id"]) == unreachable_row][0]) or False,
  4606. timeout=20, what="unreachable flag") is not None,
  4607. "queue row shows the unreachable state in the window")
  4608. print("== phase 1: saddles.token is the single credential store (arch 24) ==")
  4609. # The legacy api_tokens table and every fallback read died with the
  4610. # poll era: a restart proves the migrated schema carries no
  4611. # api_tokens table, the row token survives and still authenticates,
  4612. # and a legacy-shaped token string authorises nothing (no fallback
  4613. # read exists to honour it). Fixture: this shard's own racer.
  4614. fake2 = FakeSaddle()
  4615. fake2.ask_on_register = False
  4616. fake2.serve(port=free_port())
  4617. saddles_state = slot_public(auth, "/saddles", "saddles")
  4618. act(auth, saddles_state["register"], {"name": "racer-two",
  4619. "host": "127.0.0.1:%d" % fake2.port})
  4620. racer_two_id = int([s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  4621. if s["name"] == "racer-two"][0]["id"])
  4622. racer_two_token = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  4623. if s["name"] == "racer-two"][0]["token"]
  4624. app.terminate()
  4625. app.wait(timeout=10)
  4626. app = start_app()
  4627. auth = login("admin", "adminpass123")
  4628. ctx.app = app # teardown tracks the restarted process
  4629. ctx.auth = auth
  4630. def schema_tables():
  4631. conn = db()
  4632. try:
  4633. return {row[0] for row in conn.execute(
  4634. "select name from sqlite_master where type = 'table'")}
  4635. finally:
  4636. conn.close()
  4637. check("api_tokens" not in schema_tables(),
  4638. "the fresh schema carries no api_tokens table (arch 24)")
  4639. conn = db()
  4640. row_token = conn.execute("select token from saddles where id = ?",
  4641. (racer_two_id,)).fetchone()[0]
  4642. conn.close()
  4643. check(row_token == racer_two_token and (row_token or "").startswith("saddle-"),
  4644. "the per-saddle row token survives the restart unchanged")
  4645. status, _ = saddle_api("POST", "/api/saddle/next-task", racer_two_token)
  4646. check(status in (200, 204),
  4647. "the row token still authenticates after the restart")
  4648. status, _ = saddle_api("POST", "/api/saddle/next-task", "saddle-legacy-copied-token")
  4649. check(status == 401,
  4650. "a legacy api_tokens-shaped credential authorises nothing (fallback gone)")
  4651. fake2.token = racer_two_token
  4652. fake2.ask_now()
  4653. check(True, "the fleet re-asks after the restart (sleeping re-established)")
  4654. # =============================================================================
  4655. # Bundle: push — plan 11 task E: real Web Push over VAPID. VAPID
  4656. # persistence, the service worker, subscription CRUD + validation,
  4657. # notify-me prefs, the block/lifecycle dispatch matrix (global vs
  4658. # opted-in), 410 pruning, retry-next-tick delivery and protocol-level
  4659. # payload verification (RFC 8291 decryption + ES256 JWT) against the
  4660. # fake push service.
  4661. # =============================================================================
  4662. def bundle_push(ctx):
  4663. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  4664. def subscription_count(endpoint):
  4665. conn = db()
  4666. try:
  4667. return conn.execute(
  4668. "select count(*) from push_subscriptions where endpoint = ?",
  4669. (endpoint,)).fetchone()[0]
  4670. finally:
  4671. conn.close()
  4672. print("== task E: service worker asset ==")
  4673. status, sw = http("GET", "/sw.js", timeout=15)
  4674. check(status == 200 and "addEventListener('push'" in sw
  4675. and "notificationclick" in sw and "clients.openWindow" in sw,
  4676. "the service worker serves at the origin root with push + click routing")
  4677. print("== task E: VAPID keypair generation + persistence ==")
  4678. config = json.load(open(os.path.join(WORK, "web-config.json")))
  4679. vapid = config.get("vapid") or {}
  4680. check(bool(vapid.get("public_key")) and bool(vapid.get("private_key")),
  4681. "the VAPID keypair persists into web-config.json's vapid section")
  4682. vapid_private_key = ec.derive_private_key(
  4683. int.from_bytes(b64url_dec(vapid["private_key"]), "big"), ec.SECP256R1())
  4684. derived_public = vapid_private_key.public_key().public_bytes(
  4685. Encoding.X962, PublicFormat.UncompressedPoint)
  4686. check(b64url_dec(vapid["public_key"]) == derived_public,
  4687. "the persisted public key matches the private scalar (a valid P-256 pair)")
  4688. dash = slot_public(auth, "/", "dashboard")
  4689. check(dash.get("push_vapid_key") == vapid["public_key"],
  4690. "the dashboard exposes the same VAPID key to subscribers")
  4691. check(dash.get("push_subscribed") is False and bool(dash.get("push_subscribe")),
  4692. "the bell starts unsubscribed and carries the subscribe action")
  4693. print("== task E: subscription CRUD + auth validation ==")
  4694. browser = ec.generate_private_key(ec.SECP256R1())
  4695. browser_public = browser.public_key().public_bytes(
  4696. Encoding.X962, PublicFormat.UncompressedPoint)
  4697. browser_auth = os.urandom(16)
  4698. subscribe_fields = {"p256dh": b64url(browser_public), "auth": b64url(browser_auth)}
  4699. valid_endpoint = "http://127.0.0.1:1/x"
  4700. check("absolute http(s)" in (action_notify(act(auth, dash["push_subscribe"],
  4701. {"endpoint": "ftp://example.com/push", **subscribe_fields})) or ""),
  4702. "a non-http endpoint is rejected")
  4703. check("p256dh" in (action_notify(act(auth, dash["push_subscribe"],
  4704. {"endpoint": valid_endpoint, "p256dh": b64url(os.urandom(64)),
  4705. "auth": subscribe_fields["auth"]})) or ""),
  4706. "a malformed p256dh point is rejected")
  4707. check("auth" in (action_notify(act(auth, dash["push_subscribe"],
  4708. {"endpoint": valid_endpoint, "p256dh": subscribe_fields["p256dh"],
  4709. "auth": b64url(os.urandom(15))})) or ""),
  4710. "a malformed auth secret is rejected")
  4711. push = FakePushService()
  4712. push.serve()
  4713. endpoint_main = push.url("/main")
  4714. act(auth, dash["push_subscribe"],
  4715. {"endpoint": endpoint_main, **subscribe_fields})
  4716. conn = db()
  4717. rows = conn.execute("select user_id, p256dh, auth from push_subscriptions"
  4718. " where endpoint = ?", (endpoint_main,)).fetchall()
  4719. admin_id = conn.execute("select id from spry_users where username = 'admin'"
  4720. ).fetchone()[0]
  4721. conn.close()
  4722. check(len(rows) == 1 and rows[0][0] == admin_id,
  4723. "a valid subscription stores against the signed-in user")
  4724. # Re-subscribing the same endpoint replaces (never duplicates).
  4725. rotated_auth = os.urandom(16)
  4726. act(auth, dash["push_subscribe"],
  4727. {"endpoint": endpoint_main, "p256dh": subscribe_fields["p256dh"],
  4728. "auth": b64url(rotated_auth)})
  4729. conn = db()
  4730. rows = conn.execute("select auth from push_subscriptions where endpoint = ?",
  4731. (endpoint_main,)).fetchall()
  4732. conn.close()
  4733. check(len(rows) == 1 and rows[0][0] == b64url(rotated_auth),
  4734. "re-subscribing an endpoint replaces its keys in place")
  4735. print("== task E: notify-me pref persistence ==")
  4736. act(auth, slot_public(auth, "/tasks", "tasks")["create"],
  4737. {"project_id": "1", "title": "Push pref task", "priority": "5"})
  4738. pref_task = task_id_of("Push pref task")
  4739. act(auth, task_state(auth, pref_task)["toggle_notify"], {})
  4740. conn = db()
  4741. check(conn.execute("select count(*) from task_notify_prefs where task_id = ?"
  4742. " and user_id = ?", (pref_task, admin_id)).fetchone()[0] == 1,
  4743. "Notify me persists a task_notify_prefs row for the user")
  4744. conn.close()
  4745. check(task_state(auth, pref_task).get("notify_me") is True,
  4746. "the task slot reflects notify_me after the toggle")
  4747. act(auth, task_state(auth, pref_task)["toggle_notify"], {})
  4748. conn = db()
  4749. check(conn.execute("select count(*) from task_notify_prefs where task_id = ?",
  4750. (pref_task,)).fetchone()[0] == 0,
  4751. "Don't notify me removes the pref row")
  4752. conn.close()
  4753. check(task_state(auth, pref_task).get("notify_me") is False,
  4754. "the task slot reflects the opt-out")
  4755. print("== task E: dispatch matrix — global blocks vs opted-in lifecycle ==")
  4756. # Two "browsers" for admin (/main, /gone) and one for another user
  4757. # (id 999, rows seeded directly — push targeting is per user).
  4758. endpoint_gone = push.url("/gone")
  4759. push.status_for["/gone"] = 410
  4760. act(auth, dash["push_subscribe"],
  4761. {"endpoint": endpoint_gone, **subscribe_fields})
  4762. other_browser = ec.generate_private_key(ec.SECP256R1())
  4763. other_public = other_browser.public_key().public_bytes(
  4764. Encoding.X962, PublicFormat.UncompressedPoint)
  4765. other_auth = os.urandom(16)
  4766. endpoint_other = push.url("/other")
  4767. conn = db()
  4768. conn.execute("insert into push_subscriptions (user_id, endpoint, p256dh, auth,"
  4769. " created_at) values (999, ?, ?, ?, strftime('%s','now'))",
  4770. (endpoint_other, b64url(other_public), b64url(other_auth)))
  4771. conn.commit()
  4772. conn.close()
  4773. # Task 1: admin opted in via the UI toggle; the other user is not.
  4774. # Baselines are captured BEFORE the enqueue: the first-start push
  4775. # fires at the allocation, the instant the dispatch lands.
  4776. act(auth, slot_public(auth, "/tasks", "tasks")["create"],
  4777. {"project_id": "1", "title": "Push lifecycle task", "priority": "5"})
  4778. task1 = task_id_of("Push lifecycle task")
  4779. act(auth, task_state(auth, task1)["toggle_notify"], {})
  4780. main_before = len(push.hits("/main"))
  4781. other_before = len(push.hits("/other"))
  4782. gone_before = len(push.hits("/gone"))
  4783. enqueue_task(auth, task1)
  4784. check(wait_for(lambda: fake.running_task == task1 or False, timeout=30,
  4785. what="task 1 assignment") is not None,
  4786. "task 1 dispatches to the fake daemon")
  4787. task1_session = fake.session_id
  4788. check(wait_for(lambda: len(push.hits("/main")) > main_before, timeout=15,
  4789. what="first-start push to the opted-in admin") is not None,
  4790. "first start notifies the opted-in user's subscription")
  4791. check(wait_for(lambda: len(push.hits("/gone")) > gone_before, timeout=15,
  4792. what="first-start push to the gone endpoint") is not None,
  4793. "first start reaches every subscription of the opted-in user")
  4794. check(quiet_window(quiet_seconds(),
  4795. lambda: len(push.hits("/other")) == other_before,
  4796. "no first-start push to the not-opted-in user"),
  4797. "first start never reaches users without the notify pref")
  4798. check(wait_task_done(auth, task1, timeout=60) is not None,
  4799. "task 1 completes")
  4800. _, done_payload = decrypt_webpush(push.hits("/main")[-1], browser,
  4801. browser_public, rotated_auth)
  4802. check(done_payload["title"] == "Push lifecycle task — completed"
  4803. and done_payload["url"] == "/tasks/%d" % task1,
  4804. "completion notifies the opted-in user (decrypted payload)")
  4805. check(len(push.hits("/other")) == other_before,
  4806. "completion stays with the opted-in user only")
  4807. check(wait_for(lambda: subscription_count(endpoint_gone) == 0, timeout=15,
  4808. what="410 endpoint pruned") is not None,
  4809. "a 410 Gone endpoint is pruned from push_subscriptions")
  4810. # Block events go to EVERY push-subscribed user globally. The
  4811. # poll-era /update raise is gone: a live (non-terminal) host task
  4812. # with an open session on the fake carries the block through the
  4813. # results push — task1 is done and terminal pushes are consumed
  4814. # silently, so the host is its own fixture row.
  4815. conn = db()
  4816. conn.execute("insert into tasks (project_id, title, status, priority,"
  4817. " created_at) values (1, 'Push block host', 'working', 5,"
  4818. " strftime('%s','now'))")
  4819. push_host = conn.execute("select max(id) from tasks").fetchone()[0]
  4820. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  4821. " values (?, ?, strftime('%s','now'))", (push_host, fake_id))
  4822. push_host_session = conn.execute(
  4823. "select max(id) from sessions").fetchone()[0]
  4824. conn.commit()
  4825. conn.close()
  4826. def raise_global_block(question):
  4827. return saddle_api("POST", "/api/saddle/results", fake_token, {
  4828. "session_id": push_host_session, "state": "blocked",
  4829. "block": {"question": question, "type": "question"}})
  4830. main_before = len(push.hits("/main"))
  4831. other_before = len(push.hits("/other"))
  4832. raise_global_block("Global question for everyone?")
  4833. check(wait_for(lambda: len(push.hits("/main")) > main_before
  4834. and len(push.hits("/other")) > other_before, timeout=15,
  4835. what="global block push") is not None,
  4836. "a raised block notifies every push-subscribed user globally")
  4837. block_hit = push.hits("/other")[-1]
  4838. _, payload = decrypt_webpush(block_hit, other_browser, other_public, other_auth)
  4839. check(payload["title"] == "Push block host — question raised"
  4840. and payload["body"] == "Global question for everyone?"
  4841. and payload["url"] == "/tasks/%d" % push_host,
  4842. "the block payload carries task title + kind, question and click URL (%s)"
  4843. % payload)
  4844. ciphertext_len = len(block_hit["body"]) - (21 + 65)
  4845. envelope = block_hit["body"]
  4846. record_size = int.from_bytes(envelope[16:20], "big")
  4847. # rs = plaintext + delimiter (1) + tag (16) == ciphertext-with-tag len.
  4848. check(record_size == ciphertext_len,
  4849. "the RFC 8291 envelope record size covers payload + delimiter + tag")
  4850. claims = verify_vapid(block_hit, vapid["public_key"], vapid["private_key"])
  4851. check(claims.get("aud") == "http://127.0.0.1:%d" % push.port
  4852. and claims.get("sub", "").startswith("mailto:")
  4853. and claims.get("exp", 0) > time.time(),
  4854. "the VAPID JWT verifies: aud = the endpoint origin, mailto sub, future exp")
  4855. check(block_hit["headers"].get("ttl") == "60"
  4856. and block_hit["headers"].get("content-type") == "application/octet-stream",
  4857. "the push POST carries TTL and the encrypted content type")
  4858. # Task 2: only the OTHER user opted in — start + failure reach them
  4859. # alone. Baseline again before the enqueue (first start fires at
  4860. # the allocation).
  4861. fake.fail_next = True
  4862. act(auth, slot_public(auth, "/tasks", "tasks")["create"],
  4863. {"project_id": "1", "title": "Push failure task", "priority": "5"})
  4864. task2 = task_id_of("Push failure task")
  4865. conn = db()
  4866. conn.execute("insert into task_notify_prefs (task_id, user_id) values (?, 999)",
  4867. (task2,))
  4868. conn.commit()
  4869. conn.close()
  4870. main_before2 = len(push.hits("/main"))
  4871. other_before2 = len(push.hits("/other"))
  4872. enqueue_task(auth, task2)
  4873. check(wait_for(lambda: fake.running_task == task2 or False, timeout=30,
  4874. what="task 2 assignment") is not None,
  4875. "task 2 dispatches (scripted to fail)")
  4876. check(wait_for(lambda: task_state(auth, task2)["status"] == "failed"
  4877. or False, timeout=30, what="task 2 failed") is not None,
  4878. "task 2 fails")
  4879. check(wait_for(lambda: len(push.hits("/other")) >= other_before2 + 2,
  4880. timeout=15, what="opted-in user's started+failed pushes"),
  4881. "the opted-in user gets first start AND failure for the task")
  4882. started_hit = [hit for hit in push.hits("/other")[other_before2:]][0]
  4883. _, started_payload = decrypt_webpush(started_hit, other_browser,
  4884. other_public, other_auth)
  4885. check(started_payload["title"] == "Push failure task — started",
  4886. "the first-start payload names task + kind (%s)" % started_payload)
  4887. _, failure_payload = decrypt_webpush(push.hits("/other")[-1], other_browser,
  4888. other_public, other_auth)
  4889. check(failure_payload["title"] == "Push failure task — failed"
  4890. and failure_payload["url"] == "/tasks/%d" % task2,
  4891. "the failure payload names the task and links its page")
  4892. check(len(push.hits("/main")) == main_before2,
  4893. "an agent failure never reaches users without the pref")
  4894. print("== task E: retry next tick (best-effort delivery) ==")
  4895. endpoint_flaky = push.url("/flaky")
  4896. act(auth, dash["push_subscribe"],
  4897. {"endpoint": endpoint_flaky, **subscribe_fields})
  4898. push.fail_next["/flaky"] = 1
  4899. flaky_before = len(push.hits("/flaky"))
  4900. raise_global_block("Flaky delivery question?")
  4901. check(wait_for(lambda: len(push.hits("/flaky")) >= flaky_before + 2,
  4902. timeout=20, what="flaky endpoint retried") is not None,
  4903. "a failed send retries on the next dispatcher tick and lands")
  4904. print("== plan 12: goal notifications — blocks/success/failure always ==")
  4905. # Goal blocks, completion approval and failure push to EVERY
  4906. # subscribed user (the mirror on the goal-run task stays quiet).
  4907. def goal_state(goal_id):
  4908. return slot_public(auth, "/goals/%d" % goal_id, "goal")
  4909. push_goal = None
  4910. act(auth, slot_public(auth, "/goals", "goals")["create"], {
  4911. "project_id": "1", "title": "Push goal", "brief": "Notifies everyone.",
  4912. "branch_name": "goal/push"})
  4913. conn = db()
  4914. row = conn.execute("select id from goals where title = 'Push goal'").fetchone()
  4915. conn.close()
  4916. push_goal = int(row[0])
  4917. fake.goal_script = [("plan", "# Push plan\n\n1. Ring the bells")]
  4918. main_before = len(push.hits("/main"))
  4919. other_before = len(push.hits("/other"))
  4920. act(auth, goal_state(push_goal)["start"], {})
  4921. check(wait_for(lambda: goal_state(push_goal)["status"] == "plan_review" or False,
  4922. timeout=60, what="push goal enters plan review") is not None,
  4923. "the push goal reaches plan review")
  4924. check(wait_for(lambda: len(push.hits("/main")) > main_before
  4925. and len(push.hits("/other")) > other_before, timeout=15,
  4926. what="goal plan-review push") is not None,
  4927. "a goal plan review notifies every subscribed user globally")
  4928. _, plan_payload = decrypt_webpush(push.hits("/other")[-1], other_browser,
  4929. other_public, other_auth)
  4930. check("Push goal" in plan_payload["title"] and "plan" in plan_payload["title"]
  4931. and plan_payload["url"] == "/goals/%d" % push_goal,
  4932. "the goal block payload carries the goal title + click URL (%s)"
  4933. % plan_payload)
  4934. main_before = len(push.hits("/main"))
  4935. fake.goal_script = [("suspend",)]
  4936. act(auth, goal_state(push_goal)["plan_pending_block"]["answer_action"],
  4937. {"answer": "Plan approved. Proceed with implementation."})
  4938. check(wait_for(lambda: goal_state(push_goal)["status"] == "active" or False,
  4939. timeout=30, what="push goal active") is not None,
  4940. "the push goal activates")
  4941. check(quiet_window(3, lambda: len(push.hits("/main")) == main_before,
  4942. "no push for an internal resume"),
  4943. "the goal agent's own park/resume does not notify (internal)")
  4944. # `saddle failure` in goal context fails the goal — global push.
  4945. fake.goal_script = [("fail", "the harness collapsed")]
  4946. other_before2 = len(push.hits("/other"))
  4947. main_before2 = len(push.hits("/main"))
  4948. conn = db()
  4949. conn.execute("insert into tasks (project_id, goal_id, kind, title, status,"
  4950. " priority, completed_at, created_at) values (1, ?, 'task',"
  4951. " 'Warm', 'done', 5,"
  4952. " strftime('%s','now'),"
  4953. " strftime('%s','now','-30 seconds'))", (push_goal,))
  4954. conn.commit()
  4955. conn.close()
  4956. check(wait_for(lambda: goal_state(push_goal)["status"] == "failed" or False,
  4957. timeout=60, what="goal failure lands") is not None,
  4958. "the goal agent's failure fails the goal")
  4959. check(wait_for(lambda: len(push.hits("/other")) > other_before2
  4960. and len(push.hits("/main")) > main_before2, timeout=15,
  4961. what="goal failure push") is not None,
  4962. "a goal failure notifies every subscribed user globally")
  4963. _, fail_payload = decrypt_webpush(push.hits("/other")[-1], other_browser,
  4964. other_public, other_auth)
  4965. check("Push goal" in fail_payload["title"] and "failed" in fail_payload["title"]
  4966. and fail_payload["url"] == "/goals/%d" % push_goal,
  4967. "the goal failure payload carries the goal title + click URL (%s)"
  4968. % fail_payload)
  4969. # Completion approval pushes "complete" globally (success always).
  4970. act(auth, slot_public(auth, "/goals", "goals")["create"], {
  4971. "project_id": "1", "title": "Push done goal", "brief": "Completes.",
  4972. "branch_name": "goal/push-done"})
  4973. conn = db()
  4974. row = conn.execute("select id from goals where title = 'Push done goal'").fetchone()
  4975. conn.close()
  4976. done_goal = int(row[0])
  4977. fake.goal_script = [("plan", "# Done plan")]
  4978. act(auth, goal_state(done_goal)["start"], {})
  4979. check(wait_for(lambda: goal_state(done_goal)["status"] == "plan_review" or False,
  4980. timeout=60, what="done goal plan review") is not None,
  4981. "the done goal reaches plan review")
  4982. fake.goal_script = [("done", "# Completion\n\nShipped.")]
  4983. act(auth, goal_state(done_goal)["plan_pending_block"]["answer_action"],
  4984. {"answer": "Plan approved. Proceed with implementation."})
  4985. check(wait_for(lambda: goal_state(done_goal)["status"] == "completion_review"
  4986. or False, timeout=60, what="done goal completion review") is not None,
  4987. "the approved done goal reaches completion review")
  4988. main_before = len(push.hits("/main"))
  4989. act(auth, goal_state(done_goal)["completion_pending_block"]["answer_action"],
  4990. {"answer": "Review approved. Continue with the task."})
  4991. check(wait_for(lambda: goal_state(done_goal)["status"] == "complete" or False,
  4992. timeout=30, what="done goal completes") is not None,
  4993. "the done goal completes")
  4994. check(wait_for(lambda: len(push.hits("/main")) > main_before, timeout=15,
  4995. what="goal success push") is not None,
  4996. "a goal completion notifies every subscribed user globally")
  4997. _, done_goal_payload = decrypt_webpush(push.hits("/main")[-1], browser,
  4998. browser_public, rotated_auth)
  4999. check("Push done goal" in done_goal_payload["title"]
  5000. and "complete" in done_goal_payload["title"]
  5001. and done_goal_payload["url"] == "/goals/%d" % done_goal,
  5002. "the goal success payload carries the goal title + click URL (%s)"
  5003. % done_goal_payload)
  5004. print("== task E: unsubscribe removes the subscription ==")
  5005. dash = slot_public(auth, "/", "dashboard")
  5006. check(dash.get("push_subscribed") is True,
  5007. "the bell shows subscribed while endpoints exist")
  5008. act(auth, dash["push_unsubscribe"], {"endpoint": endpoint_main})
  5009. check(subscription_count(endpoint_main) == 0,
  5010. "unsubscribing removes the endpoint's row")
  5011. # =============================================================================
  5012. # Bundle: goals — the plan-12 goal lifecycle: creation with branch-name
  5013. # uniqueness, the start → planning → plan review → active flow with
  5014. # revisions, the goal agent's task fan-out through the task CLI
  5015. # callbacks (ownership + creator-only enqueue), the event-driven
  5016. # coalesced resume, pause/resume queueing, the completion review with
  5017. # branch push state, the cancel cascade, and the human tasks'
  5018. # auto-enqueue-after predecessor.
  5019. # =============================================================================
  5020. def bundle_goals(ctx):
  5021. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  5022. app = ctx.app
  5023. def goal_state(goal_id):
  5024. return slot_public(auth, "/goals/%d" % goal_id, "goal")
  5025. def create_goal(fields):
  5026. act(auth, slot_public(auth, "/goals", "goals")["create"], fields)
  5027. rows = [g for g in slot_public(auth, "/goals", "goals")["goals"]
  5028. if g["title"] == fields["title"]]
  5029. return int(rows[0]["id"])
  5030. print("== plan 12: goal creation, validation, lifecycle actions ==")
  5031. goal1 = create_goal({"project_id": "1", "title": "Ship the flange",
  5032. "brief": "Make the flange module production-grade.",
  5033. "branch_name": "goal/flange", "priority": "3"})
  5034. state = goal_state(goal1)
  5035. check(state["status"] == "draft" and state["display"] == "draft"
  5036. and state["branch_name"] == "goal/flange"
  5037. and state["base_branch"] == "main",
  5038. "a goal lands in draft with its branch pair")
  5039. check(state.get("start") is not None and state.get("cancel") is not None
  5040. and state.get("pause") is None,
  5041. "draft offers start + cancel only")
  5042. check(state["run_task_id"] == 0 and state["comments"] == [],
  5043. "a draft goal has no thread yet (the run task owns it)")
  5044. draft_page = http_get_page(auth, "/goals/%d" % goal1)
  5045. check('stm-if="goal.run_task_id &gt; 0" stm-action="goal.comment"' in draft_page
  5046. and "comments open with its run" in draft_page,
  5047. "the draft goal page gates the composer on the run (muted note until then)")
  5048. # Branch-name uniqueness: same name, same involved project → refuse.
  5049. duplicate_directives = act(auth, slot_public(auth, "/goals", "goals")["create"], {
  5050. "project_id": "1", "title": "Clash", "brief": "b", "branch_name": "goal/flange"})
  5051. check(any(d.get("type") == "notify" and d.get("kind") == "error"
  5052. and "goal/flange" in (d.get("message") or "")
  5053. for d in duplicate_directives),
  5054. "a live goal's branch name cannot be reused in an involved project (%s)"
  5055. % [d.get("message") for d in duplicate_directives if d.get("type") == "notify"])
  5056. conn = db()
  5057. conn.execute("insert into projects (name, git_url, default_branch, created_at)"
  5058. " values ('other-repo', 'https://example.com/other.git', 'main',"
  5059. " strftime('%s','now'))")
  5060. conn.commit()
  5061. conn.close()
  5062. goal_elsewhere = create_goal({"project_id": "2", "title": "Elsewhere",
  5063. "brief": "b", "branch_name": "goal/flange"})
  5064. check(goal_elsewhere > 0,
  5065. "the same branch name in a disjoint project is fine (one per repo)")
  5066. # Brief validation.
  5067. brief_directives = act(auth, slot_public(auth, "/goals", "goals")["create"], {
  5068. "project_id": "1", "title": "No brief", "brief": " ", "branch_name": "goal/x1"})
  5069. check(any(d.get("type") == "notify" and d.get("kind") == "error"
  5070. and "brief" in (d.get("message") or "").lower()
  5071. for d in brief_directives),
  5072. "an empty brief rejects")
  5073. # Start: planning + the goal-run internal task.
  5074. act(auth, state["start"], {})
  5075. state = goal_state(goal1)
  5076. check(state["status"] == "planning" and state.get("start") is None,
  5077. "start moves the goal to planning")
  5078. conn = db()
  5079. run_row = conn.execute(
  5080. "select id, kind, mode, creator, goal_id, priority, status from tasks"
  5081. " where goal_id = ? and kind = 'goal_run'", (goal1,)).fetchone()
  5082. conn.close()
  5083. check(run_row is not None and run_row[1] == "goal_run"
  5084. and run_row[2] == "orchestrate" and run_row[3] == "goal"
  5085. and run_row[4] == goal1 and run_row[5] == 3
  5086. and run_row[6] in ("queued", "assigned", "working"),
  5087. "start creates the goal-run task (orchestrate, priority 3, creator goal,"
  5088. " queued)")
  5089. run_task = int(run_row[0])
  5090. check(goal_state(goal1)["run_task_id"] == run_task,
  5091. "the goal state carries the run task id")
  5092. check(any(e["kind"] == "started" for e in goal_state(goal1)["events"]),
  5093. "the narrative records the start")
  5094. print("== feedback: goal-agent runs are hidden from every task listing ==")
  5095. # The internal run task never lists anywhere tasks are browsed —
  5096. # only the goal page exposes it (worker tasks stay listed). The
  5097. # plan script is armed BEFORE the first dispatch so this section's
  5098. # waits never consume the round the plan-review flow below needs.
  5099. fake.goal_script = [("plan", "# Plan v1\n\n1. Add tests\n2. Ship")]
  5100. check(wait_for(lambda: task_state(auth, run_task)["status"]
  5101. in ("assigned", "working") or False,
  5102. timeout=60, what="run task dispatches") is not None,
  5103. "the run task reaches its saddle")
  5104. queue_rows = slot_public(auth, "/tasks", "tasks")["tasks"]
  5105. check(all(int(t["id"]) != run_task for t in queue_rows),
  5106. "the tasks queue never lists the goal-agent run task")
  5107. check(all(t["kind"] != "goal_run" for t in queue_rows),
  5108. "no goal_run rows leak into the queue at all")
  5109. project_state = slot_public(auth, "/projects/1", "project")
  5110. check(all(int(t["id"]) != run_task for t in project_state["tasks"]),
  5111. "the project page's task list excludes the run task")
  5112. dashboard = slot_public(auth, "/", "dashboard")
  5113. check(all(int(a["id"]) != run_task for a in dashboard["active_tasks"]),
  5114. "the dashboard's active tasks exclude the run task")
  5115. check(all(int(i["task_id"]) != run_task for i in dashboard["activity"]),
  5116. "the activity timeline never references the run task")
  5117. check(wait_for(lambda: int(next(
  5118. (s for s in slot_public(auth, "/", "dashboard")["saddles"]
  5119. if s["name"] == "fake-saddle"), {})
  5120. .get("current_goal_id") or 0) == goal1 or False,
  5121. timeout=30, what="saddle card links the goal") is not None,
  5122. "a saddle hosting the goal agent reads as supervising the goal")
  5123. fake_dash_card = next(s for s in slot_public(auth, "/", "dashboard")["saddles"]
  5124. if s["name"] == "fake-saddle")
  5125. check(fake_dash_card["current_goal_title"] == "Ship the flange",
  5126. "the supervision link carries the goal title")
  5127. print("== plan 12: the goal agent plans, plan review, revisions ==")
  5128. # (The plan script was armed before the run's first dispatch — see
  5129. # the listing-hiding section above — so the first round submits it.)
  5130. check(wait_for(lambda: goal_state(goal1)["status"] == "plan_review" or False,
  5131. timeout=120, what="goal agent submits its plan") is not None,
  5132. "the goal agent's plan push enters plan review")
  5133. state = goal_state(goal1)
  5134. check(task_state(auth, run_task)["status"] == "blocked",
  5135. "the goal-run task parks on the plan review")
  5136. # U3's plan-review counterpart: the run-task card never reads as a
  5137. # bare "blocked" while a review-shaped block parks it.
  5138. check(task_state(auth, run_task)["display"] == "awaiting your review",
  5139. "the run-task card reads 'awaiting your review' during the plan review (U3)")
  5140. pending = state.get("plan_pending_block")
  5141. check(pending and "# Plan v1" in (pending.get("body") or ""),
  5142. "the pending plan review carries the submitted markdown")
  5143. check(any(b["block_type"] == "plan" and b["is_plan_review"]
  5144. for b in state["blocks"]),
  5145. "the goal blocks list carries the pending plan block")
  5146. print("== feedback: goal visual identity ==")
  5147. # A goal never reads as a task at a glance: marked nav entry,
  5148. # iconed list page, and the dashboard pile's goal cards carry
  5149. # their own accent styling.
  5150. home_html = http_get_page(auth, "/")
  5151. check('class="nav-goal"' in home_html and "&#127919;" in home_html,
  5152. "the nav carries the marked 🎯 Goals entry beside Tasks")
  5153. check(any(p["item_type"] == "goal" for p in
  5154. slot_public(auth, "/", "dashboard")["pinned"]),
  5155. "the goal's pending block pins onto the dashboard pile")
  5156. check("'card goal-card'" in home_html and 'badge-kind-goal' in home_html,
  5157. "dashboard goal cards carry the distinct goal styling")
  5158. goals_html = http_get_page(auth, "/goals")
  5159. check("&#127919;" in goals_html and "goal-icon" in goals_html,
  5160. "the goals list page carries the goal icon")
  5161. goal_page_html = http_get_page(auth, "/goals/%d" % goal1)
  5162. check('class="card goal-head"' in goal_page_html and "goal-page" in goal_page_html,
  5163. "the goal page carries the goal-accented identity chrome")
  5164. # Reject with feedback: the agent revises.
  5165. fake.goal_script = [("plan", "# Plan v2\n\n1. Add tests\n2. Add docs\n3. Ship")]
  5166. reject_resume_index = len(fake.goal_assignments)
  5167. act(auth, pending["answer_action"], {"answer": "Add documentation please"})
  5168. check(wait_for(lambda: goal_state(goal1)["status"] == "plan_review" or False,
  5169. timeout=60, what="revised plan enters review") is not None,
  5170. "feedback sends the agent back to planning; the revision re-enters review")
  5171. # The rejection resume carried NO context_update (nothing in the
  5172. # instruction context had changed).
  5173. check(all("context_update" not in p
  5174. for p in fake.goal_assignments[reject_resume_index:]),
  5175. "a resume without context changes carries no context_update")
  5176. state = goal_state(goal1)
  5177. revisions = state["plan_revisions"]
  5178. check(len(revisions) == 2
  5179. and revisions[0]["answered"] and "Add documentation please" in revisions[0]["answer"]
  5180. and not revisions[1]["answered"],
  5181. "rejected revisions stay as answered history (historical-blocks data)")
  5182. # Plan 16 S1 pin: goals.plan is accept-gated — the pending revision
  5183. # lives only in its block; nothing lands on the goal before accept.
  5184. conn = db()
  5185. plan_column = conn.execute(
  5186. "select plan from goals where id = ?", (goal1,)).fetchone()[0]
  5187. conn.close()
  5188. check((plan_column is None or plan_column == "")
  5189. and (state["plan"] or "") == "",
  5190. "goals.plan stays empty while the revision is only pending (accept-gated, S1)")
  5191. # Approve: goal.plan = v2, status active, agent resumes and parks.
  5192. # Plan 16 S2: the accept posts the verdict MARKER — the server
  5193. # resolves the goal-plan kind and owns the constant.
  5194. fake.goal_script = [("suspend",)]
  5195. pending = state["plan_pending_block"]
  5196. act(auth, pending["answer_action"], {"verdict": "accept"})
  5197. check(wait_for(lambda: goal_state(goal1)["status"] == "active" or False,
  5198. timeout=60, what="approved plan activates the goal") is not None,
  5199. "approving the plan activates the goal")
  5200. state = goal_state(goal1)
  5201. check(state["plan"] == "# Plan v2\n\n1. Add tests\n2. Add docs\n3. Ship",
  5202. "the approved plan markdown is stored on the goal (the accept verdict's write)")
  5203. # The approval resume carries the plan delta as a context_update
  5204. # turn (the prompt-cache-friendly replacement for the daemon's
  5205. # old system-prompt refresh).
  5206. check(wait_for(lambda: len(fake.goal_assignments) > 0 and
  5207. "context_update" in fake.goal_assignments[-1] or False,
  5208. timeout=30, what="approval resume carries context_update") is not None,
  5209. "the plan-approval resume carries a context_update turn")
  5210. approval_update = fake.goal_assignments[-1]["context_update"]
  5211. check("Plan approved" in approval_update and "# Plan v2" in approval_update,
  5212. "the context_update summarises the approved plan markdown")
  5213. check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
  5214. or False, timeout=60, what="goal agent parks after approval") is not None,
  5215. "the resumed goal agent parks (saddle suspend)")
  5216. print("== plan 12: task fan-out via the task CLI callbacks ==")
  5217. # The goal agent (parked, open session on its saddle) creates,
  5218. # enqueues and deletes its tasks through /api/saddle/task/*.
  5219. conn = db()
  5220. agent_session = conn.execute(
  5221. "select id from sessions where task_id = ? and ended_at is null",
  5222. (run_task,)).fetchone()[0]
  5223. conn.close()
  5224. status, body = saddle_api("POST", "/api/saddle/task/create", fake_token, {
  5225. "session_id": agent_session, "title": "Flange tests",
  5226. "mandate": "Write the test suite for the flange module.",
  5227. "mode": "test"})
  5228. check(status == 200 and body.get("ok") is True,
  5229. "task create under the goal succeeds (%s)" % body.get("error"))
  5230. fan_task = int(body["task_id"])
  5231. detail = task_state(auth, fan_task)
  5232. check(detail["status"] == "draft" and detail["mode"] == "test"
  5233. and detail["creator"] == "goal" and detail.get("goal_id") == goal1,
  5234. "the created task is a draft goal task with its mode and creator")
  5235. check(detail["task_branch"] == "task/%d-%d" % (goal1, fan_task)
  5236. and detail["branch_from"] == "goal/flange",
  5237. "goal worker tasks get ephemeral branches cut from the goal branch")
  5238. check(any(e["kind"] == "task-created" and int(e["task_id"]) == fan_task
  5239. for e in goal_state(goal1)["events"]),
  5240. "the narrative records the agent's task creation")
  5241. # Draft until the agent enqueues; humans may not (creator-only).
  5242. check(detail.get("enqueue") is None,
  5243. "the human Enqueue action is not offered for goal tasks (creator-only)")
  5244. status, body = saddle_api("POST", "/api/saddle/task/enqueue", fake_token, {
  5245. "session_id": agent_session, "task_id": fan_task})
  5246. check(status == 200 and task_state(auth, fan_task)["status"] == "queued",
  5247. "task enqueue by the goal agent queues the draft")
  5248. # Ownership + shape errors, verbatim for the CLI.
  5249. status, body = saddle_api("POST", "/api/saddle/task/enqueue", fake_token, {
  5250. "session_id": agent_session, "task_id": 999999})
  5251. check(status == 400 and "No task exists" in (body.get("error") or ""),
  5252. "enqueue of a missing task errors verbatim")
  5253. create_queued(auth, {"project_id": "1", "title": "Innocent bystander", "priority": "4"})
  5254. bystander = task_id_of("Innocent bystander")
  5255. status, body = saddle_api("POST", "/api/saddle/task/delete", fake_token, {
  5256. "session_id": agent_session, "task_id": bystander})
  5257. check(status == 400 and "does not belong to your goal" in (body.get("error") or ""),
  5258. "the goal agent cannot touch tasks outside its goal")
  5259. status, body = saddle_api("POST", "/api/saddle/task/create", fake_token, {
  5260. "session_id": agent_session, "title": "Bad mode", "mandate": "m", "mode": "party"})
  5261. check(status == 400 and "valid modes" in (body.get("error") or ""),
  5262. "an unknown mode rejects with the valid list")
  5263. status, body = saddle_api("POST", "/api/saddle/task/create", fake_token, {
  5264. "session_id": agent_session, "title": "Bad tag", "mandate": "m",
  5265. "capabilities": ["warp-drive"]})
  5266. check(status == 400 and "warp-drive" in (body.get("error") or "")
  5267. and "valid tags" in (body.get("error") or ""),
  5268. "an unknown capability tag rejects naming the valid set")
  5269. # A plain task's session is not a goal-agent identity.
  5270. status, body = saddle_api("POST", "/api/saddle/task/create", fake_token,
  5271. {"session_id": 999999, "title": "x", "mandate": "m"})
  5272. check(status == 400 and "session" in (body.get("error") or ""),
  5273. "an unknown session_id errors verbatim")
  5274. # Delete: draft + owned.
  5275. status, body = saddle_api("POST", "/api/saddle/task/create", fake_token, {
  5276. "session_id": agent_session, "title": "Scratch task", "mandate": "m"})
  5277. scratch = int(body["task_id"])
  5278. status, body = saddle_api("POST", "/api/saddle/task/delete", fake_token, {
  5279. "session_id": agent_session, "task_id": scratch})
  5280. check(status == 200 and not any(
  5281. t["id"] == scratch for t in slot_public(auth, "/tasks", "tasks")["tasks"]),
  5282. "task delete removes the owned draft")
  5283. status, body = saddle_api("POST", "/api/saddle/task/delete", fake_token, {
  5284. "session_id": agent_session, "task_id": fan_task})
  5285. check(status == 400 and "draft" in (body.get("error") or ""),
  5286. "deleting a queued task refuses (draft only)")
  5287. print("== plan 12: goal context in the worker assignment + coalesced resume ==")
  5288. # The queued fan-out task dispatches to the daemon with the goal
  5289. # reference + ephemeral branch; its completion schedules the parked
  5290. # goal agent's resume (>= SADDLE_GOAL_RESUME_S after the event).
  5291. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == fan_task
  5292. for p in fake.start_payloads) or False,
  5293. timeout=60, what="fan-out task dispatches") is not None,
  5294. "the goal worker task dispatches")
  5295. fan_payload = [p for p in fake.start_payloads
  5296. if int(p.get("task", {}).get("id") or 0) == fan_task][-1]
  5297. goal_ctx = fan_payload.get("goal") or {}
  5298. check(goal_ctx.get("id") == goal1 and goal_ctx.get("branch") == "goal/flange"
  5299. and goal_ctx.get("base_branch") == "main"
  5300. and "flange module" in goal_ctx.get("brief", "")
  5301. and "# Plan v2" in goal_ctx.get("plan", ""),
  5302. "the worker assignment carries the goal context (read-only brief + plan)")
  5303. check(fan_payload.get("ephemeral_branch") == "task/%d-%d" % (goal1, fan_task)
  5304. and fan_payload.get("kind") == "task" and fan_payload.get("mode") == "test"
  5305. and fan_payload.get("creator") == "goal",
  5306. "the worker assignment carries the ephemeral branch + identity triplet")
  5307. check(wait_task_done(auth, fan_task, timeout=60) is not None,
  5308. "the goal worker task completes")
  5309. check(any(e["kind"] == "task-done" and int(e["task_id"]) == fan_task
  5310. for e in goal_state(goal1)["events"]),
  5311. "the narrative records the task's completion")
  5312. goal_runs_before = len(fake.goal_assignments)
  5313. check(wait_for(lambda: len(fake.goal_assignments) > goal_runs_before or False,
  5314. timeout=30, what="parked goal agent resumes") is not None,
  5315. "the task-terminal event resumes the parked goal agent")
  5316. resume_payload = fake.goal_assignments[-1]
  5317. notice = resume_payload.get("resume_notice") or ""
  5318. check("Task statuses" in notice and "Flange tests" in notice
  5319. and "done" in notice,
  5320. "the resume notice batches ALL plan task statuses")
  5321. check("New completion reports" in notice and "fake daemon finished" in notice,
  5322. "the resume notice carries the new completion report")
  5323. check("context_update" not in resume_payload,
  5324. "a later resume with no context changes omits context_update entirely")
  5325. check(resume_payload.get("kind") == "goal_run"
  5326. and resume_payload.get("mode") == "orchestrate",
  5327. "the resumed goal agent carries its identity triplet")
  5328. # The resumed agent parked again (script): two events at once must
  5329. # coalesce into ONE resume within the window.
  5330. check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
  5331. or False, timeout=60, what="goal agent re-parks") is not None,
  5332. "the resumed goal agent parks again")
  5333. print("== plan 16 S3: delta resume notices — one-line statuses, bodies once ==")
  5334. # A MULTI-LINE completion lands while the agent is parked. The next
  5335. # notice lists every status ONE-LINE (first line of the summary
  5336. # only) and embeds each NEW completion's full body exactly once —
  5337. # no statuses-section/reports-section double embedding. (Timestamps
  5338. # are second-granular: each fixture waits out the park's second so
  5339. # completed_at is strictly newer than suspended_at.)
  5340. def run_park_second_passed():
  5341. conn = db()
  5342. try:
  5343. row = conn.execute("select suspended_at from tasks where id = ?",
  5344. (run_task,)).fetchone()
  5345. if not row or not row[0]:
  5346. return False
  5347. return time.time() > float(row[0]) + 1.05
  5348. finally:
  5349. conn.close()
  5350. check(wait_for(run_park_second_passed, timeout=30,
  5351. what="S3 park second passes") is not None,
  5352. "the park stamp is in a strictly older second")
  5353. conn = db()
  5354. conn.execute("insert into tasks (project_id, goal_id, kind, title, status,"
  5355. " priority, summary, completed_at, created_at) values (1, ?, 'task',"
  5356. " 'Multiline report', 'done', 5,"
  5357. " 'Headline: the flange calibrates\n\nBody: the full report with all"
  5358. " the detail lines.',"
  5359. " strftime('%s','now'),"
  5360. " strftime('%s','now','-30 seconds'))", (goal1,))
  5361. conn.commit()
  5362. conn.close()
  5363. s3_before = len(fake.goal_assignments)
  5364. check(wait_for(lambda: len(fake.goal_assignments) > s3_before or False,
  5365. timeout=60, what="S3 delta resume fires") is not None,
  5366. "the multiline completion resumes the parked agent")
  5367. s3_notice = fake.goal_assignments[-1].get("resume_notice") or ""
  5368. s3_statuses = s3_notice.split("## Task statuses", 1)[1] \
  5369. .split("## New completion reports", 1)[0]
  5370. check("Headline: the flange calibrates" in s3_statuses
  5371. and "the full report with all the detail lines" not in s3_statuses,
  5372. "the statuses section carries ONE LINE per task (the summary's first line)")
  5373. check(s3_notice.count("the full report with all the detail lines.") == 1
  5374. and "### #" in s3_notice,
  5375. "each new completion's full body embeds EXACTLY ONCE (no double embedding)")
  5376. check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
  5377. or False, timeout=60, what="S3 agent re-parks") is not None,
  5378. "the S3 agent re-parks")
  5379. # Size constant per event, not cumulative: the NEXT resume (armed
  5380. # with a fresh single-line completion) reports only the NEW body —
  5381. # the consumed multiline report never re-embeds.
  5382. check(wait_for(run_park_second_passed, timeout=30,
  5383. what="S3 second park second passes") is not None,
  5384. "the second park stamp is in a strictly older second")
  5385. conn = db()
  5386. conn.execute("insert into tasks (project_id, goal_id, kind, title, status,"
  5387. " priority, summary, completed_at, created_at) values (1, ?, 'task',"
  5388. " 'Fresh completion', 'done', 5, 'fresh one-liner',"
  5389. " strftime('%s','now'),"
  5390. " strftime('%s','now','-30 seconds'))", (goal1,))
  5391. conn.commit()
  5392. conn.close()
  5393. s3_more = len(fake.goal_assignments)
  5394. check(wait_for(lambda: len(fake.goal_assignments) > s3_more or False,
  5395. timeout=60, what="S3 second delta resume") is not None,
  5396. "the fresh completion resumes the agent again")
  5397. s3_next = fake.goal_assignments[-1].get("resume_notice") or ""
  5398. check("the full report with all the detail lines." not in s3_next
  5399. and "### #" in s3_next and "fresh one-liner" in s3_next,
  5400. "the consumed completion's body never re-embeds (notice size constant per event)")
  5401. check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
  5402. or False, timeout=60, what="S3 agent parks a third time") is not None,
  5403. "the S3 agent parks for the pause section")
  5404. print("== feedback: goal page parity — progress feed, current activity, runs ==")
  5405. # The goal agent's runs feed the page's live surfaces: progress
  5406. # lines (green latest-first feed), the open round's newest
  5407. # activity line and its hosting saddle. Parked runs keep their
  5408. # open session, so fixtures land as rows without disturbing the
  5409. # scripted daemon.
  5410. conn = db()
  5411. open_run_session = conn.execute(
  5412. "select id from sessions where task_id = ? and ended_at is null",
  5413. (run_task,)).fetchone()[0]
  5414. conn.execute(
  5415. "insert into task_progress (task_id, summary, created_at)"
  5416. " values (?, 'Overseeing the flange fan-out', strftime('%s','now'))",
  5417. (run_task,))
  5418. conn.execute(
  5419. "insert into session_commands (session_id, command_text, type, started_at)"
  5420. " values (?, 'git -C demo log --oneline -5', 'command',"
  5421. " strftime('%s','now'))", (open_run_session,))
  5422. conn.commit()
  5423. conn.close()
  5424. # A manual refresh records the daemon's health snapshot (the
  5425. # parked daemon answers /status with its suspended state).
  5426. act(auth, saddle_card(auth, "fake-saddle")["refresh"], {})
  5427. state = goal_state(goal1)
  5428. check(state["progress"] and state["progress"][0]["summary"]
  5429. == "Overseeing the flange fan-out",
  5430. "the goal page carries the agent's live progress feed")
  5431. check(state["current_command"] == "git -C demo log --oneline -5"
  5432. and state["current_command_running"] is True,
  5433. "the current activity shows the open round's running command")
  5434. check(state["agent_saddle"] == "fake-saddle"
  5435. and "fake-saddle" in state["saddle_health"],
  5436. "the assigned agent saddle and its health display")
  5437. check(state["runs"] and state["runs"][0]["saddle_name"] == "fake-saddle"
  5438. and "session_id=" in state["runs"][0]["terminal_url"],
  5439. "the run table carries host saddles and deep-linked terminals")
  5440. goal_page = http_get_page(auth, "/goals/%d" % goal1)
  5441. check('stm-markdown="goal.brief"' in goal_page
  5442. and 'stm-markdown="goal.plan"' in goal_page
  5443. and 'stm-markdown="comment.body"' in goal_page
  5444. and 'stm-markdown="event.text"' in goal_page
  5445. and 'stm-markdown="block.body"' in goal_page,
  5446. "every long-form goal body renders through stm-markdown")
  5447. check('stm-attribute.href="run.terminal_url" target="_blank" rel="noopener"' in goal_page,
  5448. "the run table's terminal links open in a new tab")
  5449. check("progress-latest" in goal_page and "Agent progress" in goal_page,
  5450. "the green progress feed renders on the goal page")
  5451. # Plan revision history keeps its verdicts (approved + rejected).
  5452. revisions = goal_state(goal1)["plan_revisions"]
  5453. check(revisions[0]["answered"] is True and revisions[0]["approved"] is False
  5454. and revisions[1]["approved"] is True,
  5455. "plan revision history badges approved and rejected verdicts")
  5456. print("== plan 12: pause freezes supervision; events coalesce on resume ==")
  5457. state = goal_state(goal1)
  5458. act(auth, state["pause"], {})
  5459. check(goal_state(goal1)["status"] == "paused"
  5460. and goal_state(goal1).get("resume") is not None,
  5461. "an active goal pauses and offers resume")
  5462. # Terminal events under a paused goal queue without waking anybody.
  5463. # (The real completion path — complete_task — records the goal's
  5464. # task-done narrative event; the resume machinery keys on exactly
  5465. # those terminal kinds, never on the agent's own enqueue narrative.)
  5466. conn = db()
  5467. conn.execute("insert into tasks (project_id, goal_id, kind, title, status, priority,"
  5468. " summary, completed_at, created_at) values (1, ?, 'task',"
  5469. " 'Quiet task', 'done', 5, 'finished while paused',"
  5470. " strftime('%s','now'), strftime('%s','now'))", (goal1,))
  5471. quiet_id = conn.execute("select max(id) from tasks").fetchone()[0]
  5472. conn.commit()
  5473. conn.close()
  5474. paused_runs = len(fake.goal_assignments)
  5475. check(quiet_window(quiet_seconds(),
  5476. lambda: len(fake.goal_assignments) == paused_runs,
  5477. "paused goal does not resume"),
  5478. "a paused goal's events queue without scheduling a resume")
  5479. state = goal_state(goal1)
  5480. act(auth, state["resume"], {})
  5481. check(goal_state(goal1)["status"] == "active",
  5482. "resume returns the goal to active")
  5483. check(wait_for(lambda: len(fake.goal_assignments) > paused_runs or False,
  5484. timeout=30, what="queued events flush on resume") is not None,
  5485. "the queued events coalesce into the post-resume wake")
  5486. check("finished while paused" in (fake.goal_assignments[-1].get("resume_notice") or ""),
  5487. "the flushed notice reports the task that finished while paused")
  5488. check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
  5489. or False, timeout=60, what="goal agent parks again") is not None,
  5490. "the goal agent parks again after the flush")
  5491. # The watermark consumed the flushed transition: with nothing new
  5492. # under the goal, no further resume fires (parked stays parked).
  5493. flushed_assignments = len(fake.goal_assignments)
  5494. check(quiet_window(quiet_seconds(),
  5495. lambda: len(fake.goal_assignments) == flushed_assignments,
  5496. "consumed watermark never re-fires"),
  5497. "the consumed watermark never re-fires a resume")
  5498. print("== plan 12: completion review, push state, accept ==")
  5499. # The daemon exposes goal-branch push state on /status; the web
  5500. # caches it on the goal when it probes the goal agent's saddle.
  5501. def status_extra():
  5502. return None
  5503. fake.status_extra = {"goal_branches": [
  5504. {"branch": "goal/flange", "pushed": True},
  5505. {"branch": "goal/other-repo", "pushed": False}]}
  5506. fake_card = saddle_card(auth, "fake-saddle")
  5507. act(auth, fake_card["refresh"], {})
  5508. check(wait_for(lambda: goal_state(goal1).get("push_branches") or False,
  5509. timeout=30, what="push state lands on the goal") is not None,
  5510. "the daemon's /status goal_branches exposure caches on the goal")
  5511. branches = goal_state(goal1)["push_branches"]
  5512. check({(b["branch"], b["pushed"]) for b in branches}
  5513. == {("goal/flange", True), ("goal/other-repo", False)},
  5514. "the push state renders per branch")
  5515. # the daemon's real shape carries repo too — multi-repo rows stay
  5516. # distinguishable as "repo: branch" labels
  5517. fake.status_extra = {"goal_branches": [
  5518. {"repo": "repoa", "branch": "goal/flange", "pushed": True},
  5519. {"repo": "repob", "branch": "goal/flange", "pushed": False}]}
  5520. act(auth, fake_card["refresh"], {})
  5521. check(wait_for(lambda: {b["branch"] for b in goal_state(goal1).get("push_branches") or []}
  5522. >= {"repoa: goal/flange"}, timeout=30,
  5523. what="repo-qualified push state lands") is not None,
  5524. "repo-qualified goal_branches rows render as 'repo: branch'")
  5525. fake.status_extra = None
  5526. # `saddle done` in goal context: the completion review.
  5527. fake.goal_script = [("done", "# Completion\n\nThe flange ships.")]
  5528. # Wake the parked agent: a terminal transition newer than the
  5529. # watermark (the pause-flush consumed everything up to it, so arm
  5530. # one directly with a fresh row).
  5531. conn = db()
  5532. conn.execute("insert into tasks (project_id, goal_id, kind, title, status,"
  5533. " priority, completed_at, created_at) values (1, ?, 'task',"
  5534. " 'Warmup', 'done', 5,"
  5535. " strftime('%s','now'),"
  5536. " strftime('%s','now','-30 seconds'))", (goal1,))
  5537. conn.commit()
  5538. conn.close()
  5539. check(wait_for(lambda: goal_state(goal1)["status"] == "completion_review" or False,
  5540. timeout=60, what="goal agent submits its completion") is not None,
  5541. "`saddle done` in goal context enters completion review")
  5542. state = goal_state(goal1)
  5543. # Plan 16 S1 (canonical-on-accept): while the review is PENDING the
  5544. # goal row carries NO report — the document lives only in its
  5545. # block; completion_report is written solely by the accept verdict.
  5546. conn = db()
  5547. pending_report_column = conn.execute(
  5548. "select completion_report from goals where id = ?", (goal1,)).fetchone()[0]
  5549. conn.close()
  5550. check((pending_report_column is None or pending_report_column == "")
  5551. and (state["completion_report"] or "") == ""
  5552. and "# Completion" in (state["completion_pending_block"].get("body") or "")
  5553. and state["completion_pending"] is True
  5554. and state["completion_pending_block"] is not None
  5555. and state["completion_pending_block"] is not False,
  5556. "the pending report lives ONLY in its block (goal row clean, S1)")
  5557. check(task_state(auth, run_task)["status"] == "blocked",
  5558. "the goal-run task parks on the completion review")
  5559. # Plan 16 S5 (U3): label reconciliation — the goal awaiting its
  5560. # completion review reads "awaiting your review" and the run-task
  5561. # status card agrees (never a bare "blocked" for a review park).
  5562. check(goal_state(goal1)["display"] == "awaiting your review"
  5563. and task_state(auth, run_task)["display"] == "awaiting your review",
  5564. "the goal and the run-task card agree: 'awaiting your review' (U3)")
  5565. check(state["compare_url"].endswith("/compare/main...goal/flange"),
  5566. "the completion review links the compare view")
  5567. # Reject first: feedback returns the goal to active. Plan 16 S2:
  5568. # the goal page's reject posts raw `feedback` — the SERVER wraps
  5569. # it with the completion-review prefix (byte-identical to the
  5570. # historical client-built answer).
  5571. fake.goal_script = [("suspend",)]
  5572. act(auth, state["completion_pending_block"]["answer_action"],
  5573. {"feedback": "Not done — the docs are missing"})
  5574. check(wait_for(lambda: goal_state(goal1)["status"] == "active" or False,
  5575. timeout=30, what="rejected completion reactivates") is not None,
  5576. "rejecting the completion returns the goal to active")
  5577. conn = db()
  5578. rejected_answer = conn.execute(
  5579. "select answer from task_blocks where task_id = ?"
  5580. " and type = 'completion' order by id desc", (run_task,)).fetchone()[0]
  5581. conn.close()
  5582. check(rejected_answer
  5583. == "Review feedback received:\n\nNot done — the docs are missing",
  5584. "the completion rejection wraps feedback with the stable prefix (%r)"
  5585. % rejected_answer)
  5586. conn = db()
  5587. rejected_report = conn.execute(
  5588. "select completion_report from goals where id = ?", (goal1,)).fetchone()[0]
  5589. conn.close()
  5590. check(rejected_report is None or rejected_report == "",
  5591. "reject/revise leaves the goal row clean (no report leaked, S1)")
  5592. # Round 2: submit again and accept.
  5593. fake.goal_script = [("done", "# Completion v2\n\nThe flange ships, with docs.")]
  5594. conn = db()
  5595. conn.execute("insert into tasks (project_id, goal_id, kind, title, status,"
  5596. " priority, completed_at, created_at) values (1, ?, 'task',"
  5597. " 'Warmup2', 'done', 5,"
  5598. " strftime('%s','now'),"
  5599. " strftime('%s','now','-30 seconds'))", (goal1,))
  5600. conn.commit()
  5601. conn.close()
  5602. check(wait_for(lambda: goal_state(goal1)["status"] == "completion_review" or False,
  5603. timeout=60, what="resubmitted completion") is not None,
  5604. "the agent resubmits its completion after feedback")
  5605. state = goal_state(goal1)
  5606. # Regression (plan-15 followup): the daemon's done-park ALSO pushes a
  5607. # review-kind block payload (question "Goal completion review", the
  5608. # real daemon's wording) after the done entry raised the completion
  5609. # row — the two paths must converge on ONE row, not twin reviews.
  5610. fake.push_results("blocked", block={
  5611. "question": "Goal completion review", "type": "review",
  5612. "body": "# Completion v2\n\nThe flange ships, with docs.",
  5613. "options": None})
  5614. conn = db()
  5615. rows = conn.execute(
  5616. "select count(*) from task_blocks where task_id = ?"
  5617. " and answer is null and type in ('completion','review')",
  5618. (run_task,)).fetchone()[0]
  5619. conn.close()
  5620. check(rows == 1,
  5621. "the daemon's review-kind completion push adopts the pending row (no twin)")
  5622. state = goal_state(goal1)
  5623. review_family_in_open = [b for b in state.get("open_blocks", [])
  5624. if b.get("is_completion_review")]
  5625. check(state["completion_pending"] is True and len(review_family_in_open) == 1,
  5626. "the pending completion review renders ONCE — as the rich attention block")
  5627. fake.pending_block = None
  5628. # Raise-side single-row invariant (the render guard and the
  5629. # twin-answer sweep are gone — fresh DBs cannot hold pre-dedupe
  5630. # twins, and 4884259's raise-side cross-dedupe keeps it that way):
  5631. # a REDELIVERED review-kind completion push adopts the same pending
  5632. # row instead of stacking a twin.
  5633. fake.push_results("blocked", block={
  5634. "question": "Goal completion review", "type": "review",
  5635. "body": "# Completion v2\n\nThe flange ships, with docs.",
  5636. "options": None})
  5637. fake.pending_block = None
  5638. conn = db()
  5639. rows = conn.execute(
  5640. "select count(*) from task_blocks where task_id = ?"
  5641. " and answer is null and type in ('completion','review')",
  5642. (run_task,)).fetchone()[0]
  5643. conn.close()
  5644. check(rows == 1,
  5645. "a redelivered review-kind push adopts the pending row (raise-side single-row invariant)")
  5646. # Plan 15 C4: the completion-accept hard gate — the live probe
  5647. # flags still report repob's branch unpushed, so the operator's
  5648. # accept REFUSES with origin behind.
  5649. refusal = action_notify(act(auth, state["completion_pending_block"]["answer_action"],
  5650. {"answer": "Review approved. Continue with the task."}))
  5651. check(goal_state(goal1)["status"] == "completion_review",
  5652. "the accept REFUSES while origin is behind (hard gate, live flags)")
  5653. # One parser, one label: the gate's refusal names the SAME repo-
  5654. # qualified unpushed row the chips render — both surfaces read
  5655. # GoalService.parse_goal_branches (the gate-vs-chips divergence is
  5656. # structurally gone).
  5657. branches = {(b["branch"], b["pushed"]) for b in goal_state(goal1)["push_branches"]}
  5658. check("repob: goal/flange" in (refusal or "")
  5659. and ("repob: goal/flange", False) in branches
  5660. and ("repoa: goal/flange", True) in branches,
  5661. "the refusal and the chips read ONE parser (same labels, same flags)")
  5662. # Push the branch daemon-side and probe again — the flags flip
  5663. # live (no goal-row write), and the accept goes through.
  5664. fake.status_extra = {"goal_branches": [
  5665. {"repo": "repoa", "branch": "goal/flange", "pushed": True},
  5666. {"repo": "repob", "branch": "goal/flange", "pushed": True}]}
  5667. act(auth, fake_card["refresh"], {})
  5668. check(wait_for(lambda: (goal_state(goal1).get("push_branches") or [])
  5669. and all(b["pushed"] for b in goal_state(goal1)["push_branches"])
  5670. or False, timeout=30, what="pushed flags land") is not None,
  5671. "the push chips derive from the LIVE probe flags")
  5672. conn = db()
  5673. no_push_column = conn.execute(
  5674. "select count(*) from pragma_table_info('goals')"
  5675. " where name = 'branch_push_state'").fetchone()[0] == 0
  5676. conn.close()
  5677. check(no_push_column,
  5678. "no stored push snapshot exists (F9: the goal row never records it)")
  5679. state = goal_state(goal1)
  5680. # The passing accept posts the verdict MARKER (the goal page's
  5681. # accept form shape, plan 16 S2) — the server resolves the
  5682. # completion kind and stores the stable approval wording.
  5683. act(auth, state["completion_pending_block"]["answer_action"],
  5684. {"verdict": "accept"})
  5685. check(wait_for(lambda: goal_state(goal1)["status"] == "complete" or False,
  5686. timeout=30, what="accepted completion completes") is not None,
  5687. "accepting the completion completes the goal once origin is current")
  5688. state = goal_state(goal1)
  5689. conn = db()
  5690. accepted_report = conn.execute(
  5691. "select completion_report from goals where id = ?", (goal1,)).fetchone()[0]
  5692. conn.close()
  5693. check(state["terminal"] is True and state.get("cancel") is None
  5694. and accepted_report == "# Completion v2\n\nThe flange ships, with docs."
  5695. and "# Completion v2" in state["completion_report"],
  5696. "a complete goal is terminal with the WINNING block's body as its report (S1)")
  5697. check(wait_for(lambda: task_state(auth, run_task)["status"] == "done" or False,
  5698. timeout=30, what="goal run task closes") is not None,
  5699. "approving the completion closes the goal-run task done")
  5700. conn = db()
  5701. twins_left = conn.execute(
  5702. "select count(*) from task_blocks where task_id = ?"
  5703. " and answer is null and type in ('completion','review')",
  5704. (run_task,)).fetchone()[0]
  5705. conn.close()
  5706. check(twins_left == 0,
  5707. "the verdict answered the pending completion-family row (single-row invariant)")
  5708. print("== plan 15 C3: the narrative derives from the recorded shape ==")
  5709. # Content equivalence with the old projection on this run's exact
  5710. # shape: every kind the goal_events projection produced, keyed the
  5711. # same way, plus the watermark and no-ingest guarantees.
  5712. narrative = goal_state(goal1)["events"]
  5713. kinds = [e["kind"] for e in narrative]
  5714. for expected in ("created", "started", "plan-submitted", "plan-rejected",
  5715. "plan-approved", "task-created", "task-done",
  5716. "completion-submitted", "completion-rejected", "goal-approved"):
  5717. check(expected in kinds,
  5718. "the derived narrative carries the %s row" % expected)
  5719. check(all(k != "task-enqueued" and k != "resumed" and k != "retry" for k in kinds),
  5720. "timestamp-less bookkeeping kinds never return")
  5721. stamps = [e["at_display"] for e in narrative if e.get("at_display")]
  5722. check(stamps == sorted(stamps, reverse=True),
  5723. "the narrative reads newest first")
  5724. conn = db()
  5725. no_event_table = conn.execute(
  5726. "select count(*) from sqlite_master where type = 'table'"
  5727. " and name = 'goal_events'").fetchone()[0] == 0
  5728. conn.close()
  5729. check(no_event_table,
  5730. "the goal_events table is gone (no narrative rows on ingest)")
  5731. print("== plan 12: cancel cascades to the goal's tasks ==")
  5732. goal2 = create_goal({"project_id": "1", "title": "Doomed goal",
  5733. "brief": "This goal will be cancelled.",
  5734. "branch_name": "goal/doomed"})
  5735. fake.goal_script = [("suspend",)]
  5736. act(auth, goal_state(goal2)["start"], {})
  5737. conn = db()
  5738. run2 = conn.execute("select id from tasks where goal_id = ? and kind = 'goal_run'",
  5739. (goal2,)).fetchone()[0]
  5740. conn.close()
  5741. check(wait_for(lambda: task_state(auth, run2)["status"] == "suspended" or False,
  5742. timeout=60, what="doomed goal agent parks") is not None,
  5743. "the doomed goal's agent parks")
  5744. conn = db()
  5745. agent_session2 = conn.execute(
  5746. "select id from sessions where task_id = ? and ended_at is null",
  5747. (run2,)).fetchone()[0]
  5748. conn.close()
  5749. status, body = saddle_api("POST", "/api/saddle/task/create", fake_token, {
  5750. "session_id": agent_session2, "title": "Doomed draft", "mandate": "m"})
  5751. doomed_draft = int(body["task_id"])
  5752. doomed_queued_task = None
  5753. status, body = saddle_api("POST", "/api/saddle/task/create", fake_token, {
  5754. "session_id": agent_session2, "title": "Doomed runner", "mandate": "m"})
  5755. doomed_runner = int(body["task_id"])
  5756. saddle_api("POST", "/api/saddle/task/enqueue", fake_token,
  5757. {"session_id": agent_session2, "task_id": doomed_runner})
  5758. act(auth, goal_state(goal2)["cancel"], {})
  5759. check(goal_state(goal2)["status"] == "cancelled",
  5760. "the goal cancels from active (parked-agent) state")
  5761. check(task_state(auth, run2)["status"] == "failed",
  5762. "the cascade cancels the goal agent's run task")
  5763. check(wait_for(lambda: task_state(auth, doomed_runner)["status"] == "failed"
  5764. or False, timeout=30, what="cascade cancels the queued task") is not None,
  5765. "the cascade cancels the goal's queued task")
  5766. check(not any(t["id"] == doomed_draft
  5767. for t in slot_public(auth, "/tasks", "tasks")["tasks"]),
  5768. "the cascade deletes the goal's draft tasks")
  5769. # A cancelled goal frees its branch name.
  5770. reuse = create_goal({"project_id": "1", "title": "Reused name",
  5771. "brief": "b", "branch_name": "goal/doomed"})
  5772. check(reuse > goal2, "a cancelled goal's branch name is reusable")
  5773. print("== plan 13 B3: goal failure cascades to non-terminal tasks ==")
  5774. # A goal holding a draft, a queued task and a working task: the
  5775. # agent's failure cascades like a cancellation — drafts deleted (no
  5776. # orphan stays dispatchable), queued/working cancelled — and the
  5777. # narrative records the cascade.
  5778. collapse = create_goal({"project_id": "1", "title": "Collapse goal",
  5779. "brief": "Fails with tasks in flight.",
  5780. "branch_name": "goal/collapse"})
  5781. fake.goal_script = [("suspend",)]
  5782. act(auth, goal_state(collapse)["start"], {})
  5783. conn = db()
  5784. run3 = conn.execute("select id from tasks where goal_id = ? and kind = 'goal_run'",
  5785. (collapse,)).fetchone()[0]
  5786. conn.close()
  5787. check(wait_for(lambda: task_state(auth, run3)["status"] == "suspended" or False,
  5788. timeout=60, what="collapse goal agent parks") is not None,
  5789. "the collapse goal's agent parks first")
  5790. conn = db()
  5791. run3_session = conn.execute(
  5792. "select id from sessions where task_id = ? and ended_at is null",
  5793. (run3,)).fetchone()[0]
  5794. conn.close()
  5795. def agent_task(title):
  5796. status, body = saddle_api("POST", "/api/saddle/task/create", fake_token, {
  5797. "session_id": run3_session, "title": title, "mandate": "m"})
  5798. task_id = int(body["task_id"])
  5799. saddle_api("POST", "/api/saddle/task/enqueue", fake_token,
  5800. {"session_id": run3_session, "task_id": task_id})
  5801. return task_id
  5802. status, body = saddle_api("POST", "/api/saddle/task/create", fake_token, {
  5803. "session_id": run3_session, "title": "Collapse draft", "mandate": "m"})
  5804. collapse_draft = int(body["task_id"]) # stays draft (never enqueued)
  5805. collapse_queued = agent_task("Collapse queued")
  5806. collapse_worker = agent_task("Collapse worker")
  5807. # Both queued tasks race for the one saddle; freeze the daemon the
  5808. # moment the first dispatches so one stays genuinely WORKING (and
  5809. # the other queued) when the goal fails.
  5810. fake.hold_state = True
  5811. def collapse_working():
  5812. for task_id in (collapse_queued, collapse_worker):
  5813. if task_state(auth, task_id)["status"] == "working":
  5814. return task_id
  5815. return False
  5816. running_one = wait_for(collapse_working, timeout=60, what="a collapse task works")
  5817. check(running_one is not None,
  5818. "a collapse task reaches working before the failure")
  5819. queued_one = collapse_worker if running_one == collapse_queued else collapse_queued
  5820. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  5821. {"session_id": run3_session, "state": "failed",
  5822. "summary": "the foundation crumbled"})
  5823. check(status == 200 and goal_state(collapse)["status"] == "failed",
  5824. "the agent's failure fails the goal")
  5825. check(task_state(auth, run3)["status"] == "failed",
  5826. "the failure fails the goal agent's run task")
  5827. check(wait_for(lambda: task_state(auth, queued_one)["status"] == "failed"
  5828. and task_state(auth, running_one)["status"] == "failed" or False,
  5829. timeout=30, what="failure cascade settles") is not None,
  5830. "the failure cascade cancels the queued and working tasks")
  5831. check(not any(t["id"] == collapse_draft
  5832. for t in slot_public(auth, "/tasks", "tasks")["tasks"]),
  5833. "the failure cascade deletes the goal's draft tasks")
  5834. state = goal_state(collapse)
  5835. check(any(e["kind"] == "cascade" for e in state["events"]),
  5836. "the narrative records the failure cascade")
  5837. check(state["draft_tasks"] == 0 and state["queued_tasks"] == 0,
  5838. "nothing under the failed goal stays dispatchable (no orphan drafts)")
  5839. check(len([e for e in state["events"] if e["kind"] == "goal-failed"]) == 1,
  5840. "the failing goal records exactly one goal-failed event")
  5841. # Plan 16 S1: the FAILURE report keeps its goal-side home (failure
  5842. # is terminal — no pending duplication exists, so unlike the
  5843. # completion report it is written at failure time).
  5844. conn = db()
  5845. failure_report = conn.execute(
  5846. "select completion_report from goals where id = ?", (collapse,)).fetchone()[0]
  5847. conn.close()
  5848. check(failure_report == "the foundation crumbled",
  5849. "fail_goal's failure report keeps its goal-side home (S1)")
  5850. # Plan 16 S4: the narrative renders the task-failed rows from the
  5851. # terminal summary's lead — the run thread carries NO duplicate
  5852. # agent comment of the goal failure text.
  5853. failed_rows = [e for e in state["events"] if e["kind"] == "task-failed"]
  5854. check(failed_rows
  5855. and all(": Cancelled by user" in e["text"] for e in failed_rows),
  5856. "task-failed narrative rows render the summary lead (S4: summary is the terminal cache)")
  5857. run_comments = [c for c in task_state(auth, run3)["comments"]
  5858. if c["author"] == "agent"]
  5859. check(not any("the foundation crumbled" in c["body"] for c in run_comments),
  5860. "the goal failure text never duplicates as an agent comment (S4)")
  5861. fake.hold_state = False
  5862. print("== plan 12: auto-enqueue-after for human tasks ==")
  5863. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  5864. "project_id": "1", "title": "Predecessor task", "priority": "4"})
  5865. predecessor = task_id_of("Predecessor task")
  5866. enqueue_task(auth, predecessor)
  5867. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  5868. "project_id": "1", "title": "Successor task", "priority": "4",
  5869. "enqueue_after_task_id": str(predecessor)})
  5870. successor = task_id_of("Successor task")
  5871. check(task_state(auth, successor)["status"] == "draft"
  5872. and task_state(auth, successor).get("enqueue_after_task_id") == predecessor,
  5873. "a human task can carry an auto-enqueue predecessor")
  5874. check(wait_task_done(auth, predecessor, timeout=60) is not None,
  5875. "the predecessor completes")
  5876. check(wait_for(lambda: task_state(auth, successor)["status"] in
  5877. ("queued", "assigned", "working") or False,
  5878. timeout=30, what="successor auto-enqueues") is not None,
  5879. "the done predecessor auto-enqueues its dependent")
  5880. # Cycle validation refuses a predecessor chain that loops.
  5881. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  5882. "project_id": "1", "title": "Cycle A", "priority": "4"})
  5883. cycle_a = task_id_of("Cycle A")
  5884. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  5885. "project_id": "1", "title": "Cycle B", "priority": "4",
  5886. "enqueue_after_task_id": str(cycle_a)})
  5887. cycle_b = task_id_of("Cycle B")
  5888. cycle_directives = act(auth, task_state(auth, cycle_a)["update"], {
  5889. "title": "Cycle A", "description": "x", "priority": "4",
  5890. "enqueue_after_task_id": str(cycle_b)})
  5891. check(any(d.get("type") == "notify" and d.get("kind") == "error"
  5892. and "cycle" in (d.get("message") or "").lower()
  5893. for d in cycle_directives),
  5894. "a predecessor cycle rejects with a naming error")
  5895. # A failed predecessor leaves the dependent in draft.
  5896. fake.fail_next = True
  5897. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  5898. "project_id": "1", "title": "Failing predecessor", "priority": "1"})
  5899. failing_pred = task_id_of("Failing predecessor")
  5900. enqueue_task(auth, failing_pred)
  5901. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  5902. "project_id": "1", "title": "Stranded successor", "priority": "4",
  5903. "enqueue_after_task_id": str(failing_pred)})
  5904. stranded = task_id_of("Stranded successor")
  5905. check(wait_for(lambda: task_state(auth, failing_pred)["status"] == "failed"
  5906. or False, timeout=60, what="failing predecessor fails") is not None,
  5907. "the scripted-failure predecessor fails")
  5908. check(quiet_window(quiet_seconds(),
  5909. lambda: task_state(auth, stranded)["status"] == "draft",
  5910. "failed predecessor leaves the draft"),
  5911. "a failed predecessor leaves its dependent in draft")
  5912. # Successor cleanup: let the queued successor finish so the
  5913. # shard's app shuts down with an empty queue.
  5914. check(wait_task_done(auth, successor, timeout=90) is not None,
  5915. "the auto-enqueued successor completes")
  5916. print("== feedback: context_update carries rule/note deltas on resume ==")
  5917. # A standalone task parks on a question; while it is parked a rule
  5918. # is attached to the project and a note lands; the resuming
  5919. # assignment carries both as the context_update markdown turn.
  5920. probe_id = create_queued(auth, {"project_id": "1", "title": "Context probe",
  5921. "priority": "4"})
  5922. check(wait_for(lambda: fake.running_task == probe_id
  5923. and task_state(auth, probe_id)["status"] == "working" or False,
  5924. timeout=60, what="probe task runs") is not None,
  5925. "the probe task dispatches")
  5926. fake.set_state("blocked")
  5927. fake.push_results("blocked", block={"type": "question",
  5928. "question": "Which database dialect?"})
  5929. check(wait_for(lambda: task_state(auth, probe_id)["status"] == "blocked" or False,
  5930. timeout=60, what="probe task blocks") is not None,
  5931. "the probe task parks on its question")
  5932. detail = task_state(auth, probe_id)
  5933. open_block = [b for b in detail["blocks"] if not b["answered"]][0]
  5934. # Backdate the parked round's start: stamps are second-resolution
  5935. # epochs, and the delta compares strictly-later — the park must
  5936. # span a real interval like an operator's would.
  5937. conn = db()
  5938. conn.execute("update sessions set started_at = strftime('%s','now') - 60"
  5939. " where id = (select min(id) from sessions where task_id = ?)",
  5940. (probe_id,))
  5941. conn.commit()
  5942. conn.close()
  5943. # While parked: a new rule attached to the project + a new note.
  5944. rules_state = slot_public(auth, "/rules", "rules")
  5945. act(auth, rules_state["create"], {
  5946. "name": "Nightly gate", "body": "Every PR must pass the nightly suite before merge."})
  5947. conn = db()
  5948. rule_id = conn.execute("select id from rules where name = 'Nightly gate'").fetchone()[0]
  5949. conn.close()
  5950. act(auth, slot_public(auth, "/projects/1", "project")["attach_rule"],
  5951. {"rule_id": str(rule_id)})
  5952. conn = db()
  5953. conn.execute("insert into project_notes (project_id, body, created_at)"
  5954. " values (1, 'The demo repo moved its CI to nightly builds.',"
  5955. " strftime('%s','now'))")
  5956. conn.commit()
  5957. check(conn.execute("select count(*) from rule_projects where rule_id = ?",
  5958. (rule_id,)).fetchone()[0] == 1,
  5959. "the rule attaches to the project while the task is parked")
  5960. conn.close()
  5961. probe_assignments = len([p for p in fake.start_payloads
  5962. if int(p.get("task", {}).get("id") or 0) == probe_id])
  5963. act(auth, open_block["answer_action"], {"answer": "postgres"})
  5964. check(wait_for(lambda: len([p for p in fake.start_payloads
  5965. if int(p.get("task", {}).get("id") or 0) == probe_id])
  5966. > probe_assignments or False, timeout=60,
  5967. what="probe task resumes") is not None,
  5968. "answering the block resumes the probe task")
  5969. probe_resume = [p for p in fake.start_payloads
  5970. if int(p.get("task", {}).get("id") or 0) == probe_id][-1]
  5971. delta = probe_resume.get("context_update") or ""
  5972. check("Rules changed" in delta and "Nightly gate" in delta
  5973. and "nightly suite" in delta,
  5974. "the resume's context_update summarises the newly attached rule")
  5975. check("New project notes" in delta and "nightly builds" in delta,
  5976. "the resume's context_update summarises the new project note")
  5977. check("Plan approved" not in delta,
  5978. "standalone resumes carry no plan section")
  5979. check(wait_task_done(auth, probe_id, timeout=60) is not None,
  5980. "the probe task completes after its resume")
  5981. # =============================================================================
  5982. # Bundle: replay — plan 13 B1/B2: the daemon-global entry dedup and the
  5983. # idempotent terminal/narrative projections. Replays the live-failure
  5984. # post-mortem shapes directly against the push channels: an outbox
  5985. # redelivery arriving under the WRONG session id after churn (and after
  5986. # a web restart), and repeated failure pushes that must still project
  5987. # exactly once. Includes this shard's one deliberate app restart (the
  5988. # dedup must survive it — it is database state, not memory).
  5989. # =============================================================================
  5990. def bundle_replay(ctx):
  5991. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  5992. app = ctx.app
  5993. def rows_for(session):
  5994. conn = db()
  5995. try:
  5996. return conn.execute(
  5997. "select count(*) from session_commands where session_id = ?",
  5998. (session,)).fetchone()[0]
  5999. finally:
  6000. conn.close()
  6001. def dedup_rows():
  6002. conn = db()
  6003. try:
  6004. return conn.execute("select count(*) from saddle_entry_dedup").fetchone()[0]
  6005. finally:
  6006. conn.close()
  6007. def open_session_on_fake(task_id):
  6008. """A fixture open session row on the shard's saddle (the churn
  6009. shape: a fresh web-side run row the daemon never really drove)."""
  6010. conn = db()
  6011. try:
  6012. conn.execute(
  6013. "insert into sessions (task_id, saddle_id, started_at)"
  6014. " values (?, ?, strftime('%s','now'))", (task_id, fake_id))
  6015. conn.commit()
  6016. return conn.execute(
  6017. "select max(id) from sessions where task_id = ?",
  6018. (task_id,)).fetchone()[0]
  6019. finally:
  6020. conn.close()
  6021. print("== plan 13 B1: entry dedup is daemon-global, not per-session ==")
  6022. # A first task runs to completion; its session is the replay source.
  6023. first = create_queued(auth, {"project_id": "1", "title": "Replay source task",
  6024. "priority": "2"})
  6025. check(wait_task_done(auth, first, timeout=60) is not None,
  6026. "the replay-source task completes")
  6027. conn = db()
  6028. src_session = conn.execute(
  6029. "select max(id) from sessions where task_id = ?", (first,)).fetchone()[0]
  6030. conn.close()
  6031. # The outbox batch: two command entries the daemon already
  6032. # delivered once (their ids landed with the source session).
  6033. burst = [
  6034. {"id": 5001, "command": "make test", "output": "ok", "exit_code": 0,
  6035. "type": "command", "timestamp": "2026-09-11T00:00:00Z"},
  6036. {"id": 5002, "command": "grep -r flange src/", "output": "src/flange.vala",
  6037. "exit_code": 0, "type": "command", "timestamp": "2026-09-11T00:00:01Z"},
  6038. ]
  6039. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6040. {"session_id": src_session, "state": "done",
  6041. "entries": burst})
  6042. check(status == 200 and rows_for(src_session) >= 2,
  6043. "the entries batch stores under its session")
  6044. stored_once = rows_for(src_session)
  6045. dedup_after_first = dedup_rows()
  6046. # Redelivery under the SAME session: consumed, nothing new.
  6047. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6048. {"session_id": src_session, "state": "done",
  6049. "entries": burst})
  6050. check(status == 200 and rows_for(src_session) == stored_once
  6051. and dedup_rows() == dedup_after_first,
  6052. "a same-session redelivery stores nothing new (consumed silently)")
  6053. # Session churn: a brand-new session id on the same saddle replays
  6054. # the SAME entry ids — the post-mortem's mis-keyed outbox send.
  6055. # The entries' ids already populated another session, so the push
  6056. # may not create rows here either.
  6057. conn = db()
  6058. conn.execute(
  6059. "insert into tasks (project_id, title, status, priority, created_at,"
  6060. " completed_at) values (1, 'Churn target task', 'done', 5,"
  6061. " strftime('%s','now'), strftime('%s','now'))")
  6062. conn.commit()
  6063. churn_task = conn.execute(
  6064. "select max(id) from tasks where title = 'Churn target task'").fetchone()[0]
  6065. conn.close()
  6066. churn_session = open_session_on_fake(churn_task)
  6067. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6068. {"session_id": churn_session, "state": "done",
  6069. "entries": burst})
  6070. check(status == 200 and rows_for(churn_session) == 0,
  6071. "replay-under-wrong-session stores nothing new (daemon-global dedup)")
  6072. check(dedup_rows() == dedup_after_first,
  6073. "the consumed redelivery claims no fresh dedup rows")
  6074. # A fresh entry id under the churned session DOES store — the dedup
  6075. # keys on the entry id, never on the session.
  6076. fresh = [{"id": 5003, "command": "git status", "output": "clean",
  6077. "exit_code": 0, "type": "command", "timestamp": "2026-09-11T00:00:02Z"}]
  6078. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6079. {"session_id": churn_session, "state": "done",
  6080. "entries": fresh})
  6081. check(status == 200 and rows_for(churn_session) == 1
  6082. and dedup_rows() == dedup_after_first + 1,
  6083. "a fresh entry id stores under any session of its saddle")
  6084. # Web restart: the dedup survives (database state, not memory).
  6085. app.terminate()
  6086. app.wait(timeout=10)
  6087. app = start_app()
  6088. auth = login("admin", "adminpass123")
  6089. ctx.app = app # teardown tracks the restarted process
  6090. ctx.auth = auth
  6091. # The restart wiped the in-memory ask bookkeeping; the daemon
  6092. # re-establishes sleeping with a fresh ask (the real daemon's
  6093. # idle-awake ask / 15-minute safety self-ask).
  6094. fake.ask_now()
  6095. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6096. {"session_id": churn_session, "state": "done",
  6097. "entries": burst + fresh})
  6098. check(status == 200 and rows_for(churn_session) == 1
  6099. and rows_for(src_session) == stored_once,
  6100. "a post-restart redelivery still stores nothing (dedup is persistent)")
  6101. print("== plan 13 B2: duplicate failed entries project once (task) ==")
  6102. # The post-mortem's honest burst: the same failure entry redelivered
  6103. # under a churned session AND a second, DISTINCT failure entry id
  6104. # in the same session — one transition, one comment, ever.
  6105. def agent_comments(task_id, body=None):
  6106. conn = db()
  6107. try:
  6108. if body is None:
  6109. return conn.execute(
  6110. "select count(*) from task_comments"
  6111. " where task_id = ? and author = 'agent'",
  6112. (task_id,)).fetchone()[0]
  6113. return conn.execute(
  6114. "select count(*) from task_comments"
  6115. " where task_id = ? and author = 'agent' and body = ?",
  6116. (task_id, body)).fetchone()[0]
  6117. finally:
  6118. conn.close()
  6119. conn = db()
  6120. conn.execute(
  6121. "insert into tasks (project_id, title, status, priority, created_at)"
  6122. " values (1, 'Burst failure task', 'working', 5, strftime('%s','now'))")
  6123. conn.commit()
  6124. burst_task = conn.execute(
  6125. "select max(id) from tasks where title = 'Burst failure task'").fetchone()[0]
  6126. conn.close()
  6127. burst_session = open_session_on_fake(burst_task)
  6128. first_fail = [{"id": 6001, "command": "saddle failure",
  6129. "output": "bash -n rejected the script", "exit_code": 1,
  6130. "type": "failed", "timestamp": "2026-09-11T00:01:00Z"}]
  6131. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6132. {"session_id": burst_session, "state": "failed",
  6133. "entries": first_fail,
  6134. "summary": "bash -n rejected the script"})
  6135. check(status == 200
  6136. and task_state(auth, burst_task)["status"] == "failed",
  6137. "the failure entry + state pair fails the task")
  6138. check(agent_comments(burst_task, "bash -n rejected the script") == 0
  6139. and "bash -n rejected the script"
  6140. in (task_state(auth, burst_task).get("summary") or ""),
  6141. "the failure explanation lives ONCE — in tasks.summary, the terminal cache (S4: no duplicate agent comment)")
  6142. # The burst: a DISTINCT failure entry id in the same session, plus
  6143. # both ids redelivered under a churned session — nothing re-runs.
  6144. second_fail = [{"id": 6002, "command": "saddle failure",
  6145. "output": "bash -n rejected the script", "exit_code": 1,
  6146. "type": "failed", "timestamp": "2026-09-11T00:01:01Z"}]
  6147. churned_burst = open_session_on_fake(burst_task)
  6148. for target in (burst_session, churned_burst):
  6149. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6150. {"session_id": target, "state": "failed",
  6151. "entries": first_fail + second_fail,
  6152. "summary": "bash -n rejected the script"})
  6153. check(status == 200,
  6154. "the burst push under %s is consumed, not rejected" % (
  6155. "its session" if target == burst_session else "a churned session"))
  6156. check(task_state(auth, burst_task)["status"] == "failed"
  6157. and agent_comments(burst_task) == 0
  6158. and "bash -n rejected the script"
  6159. in (task_state(auth, burst_task).get("summary") or ""),
  6160. "duplicate failed entries leave one transition, one summary, zero agent comments (S4)")
  6161. print("== plan 13 B2: goal failure projects once ==")
  6162. # Goal context: `saddle failure` fails the goal ONCE — the 115
  6163. # goal-failed events of the live failure are structurally gone.
  6164. def goal_state(goal_id):
  6165. return slot_public(auth, "/goals/%d" % goal_id, "goal")
  6166. def goal_failed_events(goal_id):
  6167. # Plan 15 C3: the narrative derives at read time — the
  6168. # failed-goal row IS the goal-failed record (one row, ever).
  6169. return len([e for e in goal_state(goal_id)["events"]
  6170. if e["kind"] == "goal-failed"])
  6171. act(auth, slot_public(auth, "/goals", "goals")["create"], {
  6172. "project_id": "1", "title": "Burst goal", "brief": "Fails honestly.",
  6173. "branch_name": "goal/burst"})
  6174. conn = db()
  6175. burst_goal = conn.execute(
  6176. "select id from goals where title = 'Burst goal'").fetchone()[0]
  6177. conn.close()
  6178. fake.goal_script = [("suspend",)]
  6179. act(auth, goal_state(burst_goal)["start"], {})
  6180. conn = db()
  6181. run_task = conn.execute(
  6182. "select id from tasks where goal_id = ? and kind = 'goal_run'",
  6183. (burst_goal,)).fetchone()[0]
  6184. conn.close()
  6185. check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
  6186. or False, timeout=60, what="burst goal agent parks") is not None,
  6187. "the burst goal's agent parks first")
  6188. conn = db()
  6189. run_session = conn.execute(
  6190. "select id from sessions where task_id = ? and ended_at is null",
  6191. (run_task,)).fetchone()[0]
  6192. conn.close()
  6193. goal_fail = [{"id": 7001, "command": "saddle failure",
  6194. "output": "the harness collapsed", "exit_code": 1,
  6195. "type": "failed", "timestamp": "2026-09-11T00:02:00Z"},
  6196. {"id": 7002, "command": "saddle failure",
  6197. "output": "the harness collapsed", "exit_code": 1,
  6198. "type": "failed", "timestamp": "2026-09-11T00:02:01Z"}]
  6199. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6200. {"session_id": run_session, "state": "failed",
  6201. "entries": goal_fail, "summary": "the harness collapsed"})
  6202. check(status == 200 and goal_state(burst_goal)["status"] == "failed",
  6203. "the goal agent's failure fails the goal")
  6204. check(goal_failed_events(burst_goal) == 1,
  6205. "a two-entry failure burst records exactly one goal-failed event")
  6206. # Redeliveries — same ids under the run's session AND under a
  6207. # churned session — plus the bare failed state again: still one.
  6208. churned_run = open_session_on_fake(run_task)
  6209. for target in (run_session, churned_run):
  6210. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6211. {"session_id": target, "state": "failed",
  6212. "entries": goal_fail,
  6213. "summary": "the harness collapsed"})
  6214. check(status == 200, "the goal-failure redelivery under %s is consumed"
  6215. % ("its session" if target == run_session else "a churned session"))
  6216. check(goal_state(burst_goal)["status"] == "failed"
  6217. and goal_failed_events(burst_goal) == 1,
  6218. "redelivered failures never re-run the goal failure (one event, ever)")
  6219. check(task_state(auth, run_task)["status"] == "failed",
  6220. "the goal-run task stays failed after the replays")
  6221. conn = db()
  6222. burst_blocks = conn.execute(
  6223. "select count(*) from task_blocks where task_id = ?", (run_task,)
  6224. ).fetchone()[0]
  6225. conn.close()
  6226. check(burst_blocks == 0,
  6227. "the goal-failed path raises ZERO blocks (nothing pins the pile)")
  6228. print("== plan 13 B4: the comment projection stores the body ==")
  6229. # The heredoc shape: the command is the pipe invocation, the OUTPUT
  6230. # is the body. The projections must carry the body — the command
  6231. # echo never becomes comment text, and the literal `saddle
  6232. # comment` string never renders outside the terminal.
  6233. act(auth, slot_public(auth, "/goals", "goals")["create"], {
  6234. "project_id": "1", "title": "Comment goal", "brief": "Reports cleanly.",
  6235. "branch_name": "goal/comment"})
  6236. conn = db()
  6237. comment_goal = conn.execute(
  6238. "select id from goals where title = 'Comment goal'").fetchone()[0]
  6239. conn.close()
  6240. fake.goal_script = [("suspend",)]
  6241. act(auth, goal_state(comment_goal)["start"], {})
  6242. conn = db()
  6243. comment_run = conn.execute(
  6244. "select id from tasks where goal_id = ? and kind = 'goal_run'",
  6245. (comment_goal,)).fetchone()[0]
  6246. conn.close()
  6247. check(wait_for(lambda: task_state(auth, comment_run)["status"] == "suspended"
  6248. or False, timeout=60, what="comment goal agent parks") is not None,
  6249. "the comment goal's agent parks")
  6250. conn = db()
  6251. comment_session = conn.execute(
  6252. "select id from sessions where task_id = ? and ended_at is null",
  6253. (comment_run,)).fetchone()[0]
  6254. conn.close()
  6255. comment_body = "The flange weld holds. First sentence carries the information."
  6256. comment_entry = [{"id": 8001, "command": "cat << 'EOF' | saddle comment",
  6257. "output": comment_body, "exit_code": 0, "type": "comment",
  6258. "timestamp": "2026-09-11T00:03:00Z"}]
  6259. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6260. {"session_id": comment_session, "state": "suspended",
  6261. "entries": comment_entry})
  6262. check(status == 200, "the comment entry push is accepted")
  6263. conn = db()
  6264. task_comment_bodies = [r[0] for r in conn.execute(
  6265. "select body from task_comments where task_id = ?", (comment_run,))]
  6266. tables = {r[0] for r in conn.execute(
  6267. "select name from sqlite_master where type = 'table'").fetchall()}
  6268. conn.close()
  6269. check(task_comment_bodies == [comment_body],
  6270. "the run task's comment stores the body, not the command echo")
  6271. check("goal_comments" not in tables,
  6272. "the goal_comments table is gone (the run task's rows are the thread)")
  6273. narrative_comments = [e["text"] for e in goal_state(comment_goal)["events"]
  6274. if e["kind"] == "comment"]
  6275. check(len(narrative_comments) == 1
  6276. and narrative_comments[0].startswith("The flange weld holds.")
  6277. and "saddle comment" not in narrative_comments[0]
  6278. and "EOF" not in narrative_comments[0],
  6279. "the derived narrative's comment rows carry the body text")
  6280. state = goal_state(comment_goal)
  6281. check(state["current_command"] == comment_body,
  6282. "the goal page's current-activity line shows the comment body")
  6283. check(state["comments"] and state["comments"][0]["body"] == comment_body
  6284. and any(e["kind"] == "comment"
  6285. and e["text"].startswith("The flange weld holds.")
  6286. for e in state["events"]),
  6287. "the goal slot carries the body in its comments and narrative")
  6288. page = http_get_page(auth, "/goals/%d" % comment_goal)
  6289. check("saddle comment" not in page,
  6290. "the rendered goal page never shows the command echo")
  6291. check("event.kind == 'comment' ? ' footnote'" in page,
  6292. "the narrative's comment events render with footnote styling")
  6293. check(page.index('stm-markdown="comment.body"')
  6294. < page.index('stm-text="comment.quote"'),
  6295. "the comment card shows the body before the quote")
  6296. # Drain: nothing queued remains for the shard's shutdown.
  6297. check(queued_count() == 0, "the replay shard drains its queue")
  6298. def bundle_goalui(ctx):
  6299. """Plan 14 workstream B + plan 15 C1: the goal supervision surface.
  6300. B1 — goal-run control parity (Retry supervision / Requeue-to-resume
  6301. on the status head, the /tasks/<goal-run-id> redirect, no
  6302. notify-me toggle), B2 — the plan-review remarks queue, B4 —
  6303. sessions.resume_notice + the terminal dropdown/resume panel, B5 —
  6304. page order, effective rules and per-task capability badges, B6 —
  6305. narrative footnote weights. C1 — the single-row block world: the
  6306. goal's blocks ARE the run task's rows (answered from every
  6307. surface through one path; no dual rows, no stragglers, at most
  6308. one review-kind block per verdict path). C4 — the goal page
  6309. batch: one muted head activity line, plan between attention and
  6310. tasks, answered blocks behind an expander, footnote comments.
  6311. C5 — the dashboard deep-link closure: goal pile cards link the
  6312. goal page, every surface settles the single row, and no
  6313. goal-side store exists to hold stragglers.
  6314. """
  6315. auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
  6316. def goal_state(goal_id):
  6317. return slot_public(auth, "/goals/%d" % goal_id, "goal")
  6318. def run_task_of(goal_id):
  6319. conn = db()
  6320. try:
  6321. return int(conn.execute(
  6322. "select id from tasks where goal_id = ? and kind = 'goal_run'",
  6323. (goal_id,)).fetchone()[0])
  6324. finally:
  6325. conn.close()
  6326. def wait_run(status, timeout=60, what="run status"):
  6327. return wait_for(lambda: task_state(auth, run_task_of(goal))["status"] == status
  6328. or False, timeout=timeout, what=what)
  6329. def pile_questions():
  6330. return [b.get("question") for b in
  6331. slot_public(auth, "/", "dashboard")["pinned"]
  6332. if b.get("item_type") == "goal"]
  6333. def open_session(task_id):
  6334. conn = db()
  6335. try:
  6336. row = conn.execute(
  6337. "select id from sessions where task_id = ? and ended_at is null",
  6338. (task_id,)).fetchone()
  6339. return int(row[0]) if row else None
  6340. finally:
  6341. conn.close()
  6342. print("== B1: the goal-run task page redirects to the goal ==")
  6343. fake.goal_script = [("plan", "# Remarks plan\n\n## Steps\n\n1. First move")]
  6344. act(auth, slot_public(auth, "/goals", "goals")["create"], {
  6345. "project_id": "1", "title": "Goalui goal",
  6346. "brief": "Drive the plan-14 goal surface end to end.",
  6347. "branch_name": "goal/goalui"})
  6348. conn = db()
  6349. goal = conn.execute("select id from goals where title = 'Goalui goal'"
  6350. ).fetchone()[0]
  6351. conn.close()
  6352. act(auth, goal_state(goal)["start"], {})
  6353. wait_run("blocked", what="goalui goal agent parks on its plan") # plan review parks blocked
  6354. run_task = run_task_of(goal)
  6355. frames = directives("/tasks/%d" % run_task, auth)
  6356. navigations = [d for d in frames if d.get("type") == "navigate"]
  6357. check(any(d.get("uri") == "/goals/%d" % goal for d in navigations),
  6358. "the goal-run task page redirects to /goals/<id> (navigate directive)")
  6359. check(slot_public(auth, "/tasks/%d" % run_task, "task")["id"] == run_task,
  6360. "the redirect still hydrates the task slot (sealed refresh + API readers)")
  6361. status, page = http("GET", "/tasks/%d" % run_task, headers={
  6362. "X-Statum-Slot": "key=%s; as_at=%s" % (auth["key"], auth["as_at"])}, timeout=15)
  6363. check(status == 200, "the compiled goal-run page serves")
  6364. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  6365. "project_id": "1", "title": "Goalui worker", "priority": "4",
  6366. "description": "A plain worker under no goal."})
  6367. conn = db()
  6368. worker = conn.execute("select id from tasks where title = 'Goalui worker'"
  6369. ).fetchone()[0]
  6370. conn.close()
  6371. check(not [d for d in directives("/tasks/%d" % worker, auth)
  6372. if d.get("type") == "navigate"],
  6373. "worker task pages render normally (no redirect)")
  6374. state = goal_state(goal)
  6375. check("toggle_notify" not in state and state.get("notify") is None,
  6376. "no notify-me toggle on the goal (goals notify unconditionally)")
  6377. goal_page = http_get_page(auth, "/goals/%d" % goal)
  6378. check("Notify me" not in goal_page and "toggle-notify" not in goal_page,
  6379. "the goal page template ships no notify control")
  6380. check(state["branch_name"] == "goal/goalui" and state["base_branch"] == "main"
  6381. and isinstance(state["push_branches"], list)
  6382. and state["priority"] == 3
  6383. and len(state["started_display"]) > 0
  6384. and len(state["agent_saddle"]) > 0,
  6385. "the status head carries the full field set (branch pair, chips list,"
  6386. " priority, timestamps, agent saddle)")
  6387. check("Retry supervision" in goal_page and "Requeue to resume" in goal_page,
  6388. "the status head ships the run-control parity buttons")
  6389. check("status-corner" not in goal_page and "goal complete" not in goal_page
  6390. and goal_page.count("badge-yeehaw") == 1,
  6391. "yeehaw is rows-only (no goal-head corner badge; the row badge remains)")
  6392. check(goal_state(goal)["current_activity"] == "parked awaiting the plan review",
  6393. "the head line reads 'parked awaiting the plan review' while one is open")
  6394. goal_page = http_get_page(auth, "/goals/%d" % goal)
  6395. check(goal_page.index("Goal agent blocks") < goal_page.index("<h3>Plan</h3>")
  6396. < goal_page.index("<h3>Tasks</h3>"),
  6397. "the plan section sits between attention and tasks")
  6398. check("data-plan-review" in goal_page and "Approved plan &amp; revisions" in goal_page,
  6399. "an open review renders expanded beside the collapsed approved-plan expander")
  6400. print("== B2: the plan-review remarks queue ==")
  6401. check(goal_state(goal)["status"] == "plan_review"
  6402. and goal_state(goal)["plan_pending_block"] is not None,
  6403. "the goal sits in plan review with a pending block")
  6404. goal_page = http_get_page(auth, "/goals/%d" % goal)
  6405. review_at = goal_page.index("data-plan-review")
  6406. check(review_at > 0, "the plan review region is marked for the remarks queue")
  6407. check(goal_page.index('data-block-id="goal.plan_pending_block.id"') > 0,
  6408. "the queue keys on the pending goal block id")
  6409. body_at = goal_page.index('data-plan-body')
  6410. remarks_at = goal_page.index("data-plan-remarks")
  6411. check(body_at < remarks_at,
  6412. "the remarks queue renders at the bottom of the plan body")
  6413. for marker in ("data-plan-accept", "data-plan-toggle", "data-plan-submit",
  6414. "data-plan-remove", "plan-popover", "remarksByBlock"):
  6415. check(marker in goal_page, "the remarks queue ships %s" % marker)
  6416. check("(index + 1) + '. '" in goal_page,
  6417. "the rejection payload numbers every queued remark")
  6418. check("rejectOpenByBlock" in goal_page and "statum.afterEvaluate(syncPlanRemarks)" in goal_page,
  6419. "the queue survives pulse refreshes (maps outside statum re-renders)")
  6420. # The rejection payload exactly as the client composes it: the
  6421. # stable verdict wording, then each remark NUMBERED in queue order
  6422. # with its quote prefixed like a comment.
  6423. payload = ("Plan feedback received:\n\n"
  6424. "> ## Steps\n\n1. First move\n\n"
  6425. "1. Add a rollback step\n\n"
  6426. "2. Name the owner of the migration")
  6427. act(auth, goal_state(goal)["plan_pending_block"]["answer_action"],
  6428. {"answer": payload})
  6429. state = goal_state(goal)
  6430. check(state["status"] == "planning",
  6431. "the remarks rejection returns the agent to planning")
  6432. rejected = [r for r in state["plan_revisions"] if r.get("answered")]
  6433. check(any(r["answer"] == payload for r in rejected),
  6434. "the rejection carries the remarks verbatim, in order")
  6435. check(payload.index("1. Add a rollback step") < payload.index("2. Name the owner"),
  6436. "the numbering is queue-ordered")
  6437. print("== B4: resume_notice lands on the resumed session row ==")
  6438. fake.goal_script = [("question", "Which datastore?", ["SQLite", "Redis"])]
  6439. check(wait_for(lambda: goal_state(goal)["status"] == "planning" or False,
  6440. timeout=30, what="rejection observed") is not None,
  6441. "the agent revises")
  6442. # The parked plan agent resumes with the verdict; the next round
  6443. # raises the scripted question.
  6444. check(wait_for(lambda: any(b["question"] == "Which datastore?"
  6445. and not b.get("answered")
  6446. for b in goal_state(goal)["blocks"]) or False,
  6447. timeout=60, what="question raised") is not None,
  6448. "the resumed agent raises its question")
  6449. conn = db()
  6450. notices = dict(conn.execute(
  6451. "select id, resume_notice from sessions where task_id = ?", (run_task,)
  6452. ).fetchall())
  6453. conn.close()
  6454. resumed_ids = [i for i, text in notices.items() if text]
  6455. fresh_ids = [i for i, text in notices.items() if not text]
  6456. check(len(resumed_ids) >= 1 and any("Plan feedback received" in notices[i]
  6457. for i in resumed_ids),
  6458. "the resumed session stores the notice it was allocated")
  6459. check(min(notices) in fresh_ids,
  6460. "session 1 (the fresh opening round) carries no resume notice")
  6461. print("== C1: the single run-task row IS the goal's block ==")
  6462. check("Which datastore?" in pile_questions(),
  6463. "the open goal question pins onto the dashboard pile")
  6464. goal_rows = [b for b in goal_state(goal)["blocks"]
  6465. if b["question"] == "Which datastore?"]
  6466. run_rows = [b for b in task_state(auth, run_task)["blocks"]
  6467. if b["question"] == "Which datastore?"]
  6468. check(len(run_rows) == 1 and len(goal_rows) == 1
  6469. and int(run_rows[0]["id"]) == int(goal_rows[0]["id"]),
  6470. "the goal page and the run task read the SAME single row")
  6471. pile_card = [b for b in slot_public(auth, "/", "dashboard")["pinned"]
  6472. if b.get("question") == "Which datastore?"][0]
  6473. check(pile_card.get("item_type") == "goal"
  6474. and int(pile_card.get("goal_id") or 0) == goal
  6475. and int(pile_card.get("id") or 0) == int(run_rows[0]["id"])
  6476. and pile_card.get("goal_title") == "Goalui goal",
  6477. "the pile card is the run-task row styled as a goal item")
  6478. act(auth, [b for b in goal_state(goal)["blocks"]
  6479. if b["question"] == "Which datastore?" and not b.get("answered")]
  6480. [0]["answer_action"], {"option": "SQLite"})
  6481. conn = db()
  6482. unanswered = conn.execute(
  6483. "select count(*) from task_blocks where question = 'Which datastore?'"
  6484. " and answer is null").fetchone()[0]
  6485. conn.close()
  6486. check(unanswered == 0, "answering from the goal page closes the single row")
  6487. check("Which datastore?" not in pile_questions(),
  6488. "the answered row no longer pins the dashboard pile")
  6489. # The task-page deep link (the AnswerBlockAction API surface)
  6490. # answers the SAME row through the one goal path — there is no
  6491. # second row to sweep and no stale copy can survive.
  6492. conn = db()
  6493. conn.execute("insert into task_blocks (task_id, type, question, created_at)"
  6494. " values (?, 'question', 'Deep link probe?',"
  6495. " strftime('%s','now'))", (run_task,))
  6496. conn.commit()
  6497. conn.close()
  6498. check("Deep link probe?" in pile_questions(),
  6499. "the fixture row pins like any goal question")
  6500. deep_block = [b for b in task_state(auth, run_task)["blocks"]
  6501. if b["question"] == "Deep link probe?" and not b.get("answered")][0]
  6502. act(auth, deep_block["answer_action"], {"answer": "Answered via the deep link"})
  6503. conn = db()
  6504. answered_row = conn.execute(
  6505. "select answer from task_blocks where question = 'Deep link probe?'"
  6506. ).fetchone()[0]
  6507. tables = {r[0] for r in conn.execute(
  6508. "select name from sqlite_master where type = 'table'").fetchall()}
  6509. conn.close()
  6510. check(answered_row == "Answered via the deep link",
  6511. "the task-page deep link answers the same single row (the API path)")
  6512. check("Deep link probe?" not in pile_questions(),
  6513. "the deep-link answer closes the pile card too")
  6514. check("goal_blocks" not in tables,
  6515. "the goal_blocks table is gone (run-task blocks are canonical)")
  6516. print("== C5: deep-link closure on the single-row world ==")
  6517. # The pile card's goal item deep-links the goal page (the answer
  6518. # surface); answering from either surface settled THE row above.
  6519. home_markup = http_get_page(auth, "/")
  6520. check("'/goals/' + blockview.goal_id" in home_markup,
  6521. "the pile card deep-links the goal page for goal items")
  6522. conn = db()
  6523. dupes = conn.execute(
  6524. "select count(*) from (select question from task_blocks where task_id = ?"
  6525. " group by question having count(*) > 1)", (run_task,)).fetchone()[0]
  6526. stragglers = conn.execute(
  6527. "select count(*) from task_blocks where task_id = ? and answer is null",
  6528. (run_task,)).fetchone()[0]
  6529. goal_side_tables = [r[0] for r in conn.execute(
  6530. "select name from sqlite_master where type = 'table'"
  6531. " and name in ('goal_blocks','goal_comments','goal_events')").fetchall()]
  6532. conn.close()
  6533. check(dupes == 0 and stragglers == 0,
  6534. "one row per question on the run task, none left unanswered")
  6535. check(not goal_side_tables,
  6536. "no goal-side store exists — historical stragglers are impossible")
  6537. print("== B4: the terminal session dropdown + resume-info panel ==")
  6538. fake.goal_script = [("suspend",)]
  6539. check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
  6540. or False, timeout=60, what="agent parks suspended") is not None,
  6541. "the agent parks suspended for the requeue scenario")
  6542. terminal = slot_public(auth, "/tasks/%d/terminal" % run_task, "terminal")
  6543. ids = [int(s["id"]) for s in terminal["sessions"]]
  6544. check(ids == sorted(ids, reverse=True),
  6545. "the session dropdown lists newest-first")
  6546. check(int(terminal["session_id"]) == ids[0],
  6547. "the terminal still defaults to the latest session")
  6548. labels = {int(s["id"]): s["label"] for s in terminal["sessions"]}
  6549. check(all(label.startswith("Session #") for label in labels.values()),
  6550. "dropdown labels read 'Session #N — started <time>'")
  6551. # Recompute at the check point: the C1 answers above each
  6552. # resumed the agent too, so the earlier snapshot would go stale.
  6553. conn = db()
  6554. resumed_ids = [int(i) for (i, text) in conn.execute(
  6555. "select id, resume_notice from sessions where task_id = ?",
  6556. (run_task,)).fetchall() if text]
  6557. conn.close()
  6558. resumed_markers = [i for i in ids if "[resumed]" in labels[i]]
  6559. check(set(resumed_markers) == set(resumed_ids),
  6560. "the [resumed] marker rides exactly the notice-carrying sessions")
  6561. notices_by_id = {int(s["id"]): s.get("notice") or "" for s in terminal["sessions"]}
  6562. check("Opening brief" in notices_by_id[min(ids)]
  6563. and "Drive the plan-14 goal surface" in notices_by_id[min(ids)],
  6564. "session 1's panel shows the opening-brief framing (from the goal brief)")
  6565. check(any("Plan feedback received" in (notices_by_id[i] or "") for i in resumed_ids),
  6566. "a resumed session's panel shows the stored resume notice")
  6567. terminal_page = http_get_page(auth, "/tasks/%d/terminal" % run_task)
  6568. check('id="session-select"' in terminal_page
  6569. and "window.terminalPickSession" in terminal_page,
  6570. "the session picker is an options dropdown")
  6571. check('stm-markdown="window.terminalSessionNotice(terminal)"' in terminal_page,
  6572. "the panel renders the selected session's notice as markdown")
  6573. check("No opening brief or resume notice recorded" in terminal_page,
  6574. "the panel ships its graceful empty state")
  6575. css = open(os.path.join(REPO, "src/pages/app.css")).read()
  6576. check("resume-notice-panel" in css and "max-height: 200px" in css,
  6577. "the panel is a 200px vertical-scroll strip (stylesheet)")
  6578. print("== B1: requeue-to-resume from the goal head ==")
  6579. conn = db()
  6580. conn.execute("update saddles set status = 'offline' where id = ?", (fake_id,))
  6581. conn.commit()
  6582. conn.close()
  6583. def requeue_offered():
  6584. # Act on the SAME state that satisfied the wait: a second read
  6585. # once raced the scripted daemon's ask reviving the saddle row,
  6586. # and the conditionally-authored action vanished between the
  6587. # two reads (KeyError on act).
  6588. offered = goal_state(goal)
  6589. return offered if offered.get("requeue_run") is not None else False
  6590. state = wait_for(requeue_offered, timeout=30, what="requeue offered")
  6591. check(state is not None,
  6592. "a parked run on a dead saddle offers Requeue-to-resume")
  6593. check(state.get("retry_run") is None,
  6594. "an active goal offers no Retry supervision")
  6595. open_before = open_session(run_task)
  6596. def parked_session_ended():
  6597. if open_before is None:
  6598. return True
  6599. conn = db()
  6600. try:
  6601. return conn.execute(
  6602. "select ended_at is not null from sessions where id = ?",
  6603. (open_before,)).fetchone()[0] == 1
  6604. finally:
  6605. conn.close()
  6606. act(auth, state["requeue_run"], {})
  6607. check(task_state(auth, run_task)["status"] == "queued",
  6608. "the requeue returns the run to the queue")
  6609. # The PARKED session itself must end (the restore-on-dispatch
  6610. # successor may open immediately — the check keys the old row,
  6611. # not the task's transient open-session window).
  6612. check(wait_for(parked_session_ended, timeout=60,
  6613. what="parked session closed") is not None,
  6614. "the requeue closed the parked session (restore-on-dispatch)")
  6615. conn = db()
  6616. conn.execute("update saddles set status = 'idle' where id = ?", (fake_id,))
  6617. conn.commit()
  6618. conn.close()
  6619. fake.goal_script = [("suspend",)]
  6620. check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
  6621. or False, timeout=60, what="requeued run re-dispatches") is not None,
  6622. "the requeued run re-dispatches and re-parks")
  6623. fresh_session = open_session(run_task)
  6624. conn = db()
  6625. fresh_notice = conn.execute(
  6626. "select resume_notice from sessions where id = ?", (fresh_session,)
  6627. ).fetchone()[0]
  6628. conn.close()
  6629. check(fresh_notice is None,
  6630. "a requeue's fresh session carries no resume notice (empty panel state)")
  6631. print("== B1: Retry supervision revives a failed goal ==")
  6632. fake.push_results("failed", summary="the supervision collapsed (scripted)")
  6633. check(wait_for(lambda: goal_state(goal)["status"] == "failed" or False,
  6634. timeout=30, what="goal fails") is not None,
  6635. "the run's failure fails the goal")
  6636. check(wait_for(lambda: goal_state(goal).get("retry_run") is not None
  6637. or False, timeout=30, what="retry offered") is not None,
  6638. "a failed goal offers Retry supervision")
  6639. fake.goal_script = [("suspend",)]
  6640. act(auth, goal_state(goal)["retry_run"], {})
  6641. state = goal_state(goal)
  6642. check(state["status"] == "active",
  6643. "the retry revives supervision (goal back to active)")
  6644. check(task_state(auth, run_task)["status"] in ("queued", "assigned", "working",
  6645. "suspended", "blocked"),
  6646. "the retry re-offered the run task")
  6647. check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
  6648. or False, timeout=60, what="retried supervision parks") is not None,
  6649. "the retried supervision dispatches and parks again")
  6650. state = goal_state(goal)
  6651. check(any(e["kind"] == "system" and "Supervision retried" in e["text"]
  6652. for e in state["events"]),
  6653. "the narrative records the supervision retry (a system row)")
  6654. print("== B5 + C4: order, effective rules, capability badges ==")
  6655. goal_page = http_get_page(auth, "/goals/%d" % goal)
  6656. markers = [
  6657. ('class="card goal-head"', "status head"),
  6658. ("Agent progress", "progress"),
  6659. ("Goal agent blocks", "attention blocks"),
  6660. ("<h3>Plan</h3>", "plan"),
  6661. ("<h3>Tasks</h3>", "tasks"),
  6662. ("Initial brief", "brief"),
  6663. ("Involved projects", "projects"),
  6664. ("<h3>Rules</h3>", "rules"),
  6665. ("Supervision narrative", "narrative"),
  6666. ("<h3>Comments</h3>", "comments"),
  6667. ("Goal agent runs", "runs"),
  6668. ]
  6669. positions = []
  6670. for marker, _ in markers:
  6671. check(marker in goal_page, "the goal page ships the %s section" % _)
  6672. positions.append(goal_page.index(marker))
  6673. check(positions == sorted(positions),
  6674. "the page reads status → progress → attention → plan → tasks → brief →"
  6675. " rules → narrative → comments → runs (in order)")
  6676. check("goal.current_activity" in goal_page and "Current activity" not in goal_page,
  6677. "the activity lives as ONE muted head line (the card is gone)")
  6678. check(".goal-page .card" in open(os.path.join(REPO, "src/pages/app.css")).read(),
  6679. "the stylesheet restores the goal page's card rhythm (padding regression)")
  6680. rules_state = slot_public(auth, "/rules", "rules")
  6681. act(auth, rules_state["create"], {"name": "goalui-primary", "body": "Primary rule"})
  6682. act(auth, rules_state["create"], {"name": "goalui-codev", "body": "Co-dev rule"})
  6683. act(auth, rules_state["create"], {"name": "goalui-ref", "body": "Reference rule"})
  6684. conn = db()
  6685. rule_ids = {name: int(conn.execute("select id from rules where name = ?",
  6686. (name,)).fetchone()[0])
  6687. for name in ("goalui-primary", "goalui-codev", "goalui-ref")}
  6688. conn.execute("insert into rule_projects (rule_id, project_id, created_at)"
  6689. " values (?, 1, strftime('%s','now'))", (rule_ids["goalui-primary"],))
  6690. conn.execute("insert into projects (name, git_url, default_branch, created_at)"
  6691. " values ('goalui-codev-project', 'https://example.com/g.git',"
  6692. " 'main', strftime('%s','now'))")
  6693. codev_project = conn.execute(
  6694. "select id from projects where name = 'goalui-codev-project'").fetchone()[0]
  6695. conn.execute("insert into projects (name, git_url, default_branch, created_at)"
  6696. " values ('goalui-ref-project', 'https://example.com/r.git',"
  6697. " 'main', strftime('%s','now'))")
  6698. ref_project = conn.execute(
  6699. "select id from projects where name = 'goalui-ref-project'").fetchone()[0]
  6700. conn.execute("insert into rule_projects (rule_id, project_id, created_at)"
  6701. " values (?, ?, strftime('%s','now'))",
  6702. (rule_ids["goalui-codev"], codev_project))
  6703. conn.execute("insert into rule_projects (rule_id, project_id, created_at)"
  6704. " values (?, ?, strftime('%s','now'))",
  6705. (rule_ids["goalui-ref"], ref_project))
  6706. conn.commit()
  6707. conn.close()
  6708. act(auth, goal_state(goal)["add_link"], {"linked_project_id": str(codev_project),
  6709. "relationship": "co-developed"})
  6710. act(auth, goal_state(goal)["add_link"], {"linked_project_id": str(ref_project),
  6711. "relationship": "reference"})
  6712. rule_names = [r["name"] for r in goal_state(goal)["rules"]]
  6713. check(rule_names == ["goalui-primary", "goalui-codev"],
  6714. "the Rules section lists the goal's effective set (primary then"
  6715. " co-developed; reference contributes nothing)")
  6716. act(auth, slot_public(auth, "/saddles", "saddles")["create_capability"],
  6717. {"name": "goalui-capability"})
  6718. conn = db()
  6719. capability = conn.execute(
  6720. "select id from capabilities where name = 'goalui-capability'"
  6721. ).fetchone()[0]
  6722. # The early redirect-fixture worker joins the goal (a fanned
  6723. # worker's shape) so its row carries the badge too.
  6724. conn.execute("update tasks set goal_id = ? where id = ?", (goal, worker))
  6725. conn.execute("insert into task_capabilities (task_id, capability_id)"
  6726. " values (?, ?)", (worker, capability))
  6727. conn.commit()
  6728. conn.close()
  6729. goal_worker = {"id": worker, "caps": None}
  6730. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  6731. "project_id": "1", "title": "Goalui fanned worker", "priority": "4",
  6732. "description": "A worker the agent fanned out."})
  6733. conn = db()
  6734. fanned = conn.execute(
  6735. "select id from tasks where title = 'Goalui fanned worker'").fetchone()[0]
  6736. conn.execute("update tasks set goal_id = ? where id = ?", (goal, fanned))
  6737. conn.execute("insert into task_capabilities (task_id, capability_id)"
  6738. " values (?, ?)", (fanned, capability))
  6739. conn.commit()
  6740. conn.close()
  6741. rows = {int(t["id"]): t for t in goal_state(goal)["goal_tasks"]}
  6742. check(rows[fanned].get("capabilities") == ["goalui-capability"]
  6743. and rows[worker].get("capabilities") == ["goalui-capability"],
  6744. "task rows carry their required capability tags")
  6745. goal_page = http_get_page(auth, "/goals/%d" % goal)
  6746. check('stm-for-capability-in="goal_task.capabilities"' in goal_page
  6747. and "badge-capability" in goal_page,
  6748. "the task rows render capability badges")
  6749. print("== B6: narrative weights — events full, comments footnote ==")
  6750. # An AGENT comment projects the narrative's comment event (human
  6751. # comments only thread) — pushed through the entries channel while
  6752. # parked, so the suspended state stays put.
  6753. fake.push_results("suspended", entries=[{
  6754. "id": 9001, "command": "saddle comment", "output": "An agent aside.",
  6755. "exit_code": 0, "type": "comment", "timestamp": "2026-09-12T00:00:00Z"}])
  6756. check(wait_for(lambda: any(e["kind"] == "comment"
  6757. for e in goal_state(goal)["events"]) or False,
  6758. timeout=15, what="comment event lands") is not None,
  6759. "an agent comment lands as a narrative event")
  6760. goal_page = http_get_page(auth, "/goals/%d" % goal)
  6761. check("stm-attribute.class=\"'goal-event' + (event.kind == 'comment'"
  6762. " ? ' footnote' : '')\"" in goal_page.replace("\n", " ").replace(" ", " ")
  6763. or "'goal-event' + (event.kind == 'comment' ? ' footnote' : '')" in goal_page,
  6764. "the WHOLE comment event row carries .footnote (badge included)")
  6765. check("'badge badge-goal-' + event.kind" in goal_page,
  6766. "non-comment events keep their full-weight kind badges")
  6767. css = open(os.path.join(REPO, "src/pages/app.css")).read()
  6768. check(".goal-event.footnote" in css and ".footnote {" in css,
  6769. "the stylesheet ships the footnote row weight beside the utility")
  6770. print("== C2: system rows render distinctly on the ONE thread ==")
  6771. run_comments = task_state(auth, run_task)["comments"]
  6772. system_rows = [c for c in run_comments if c.get("is_system")]
  6773. check(system_rows and all(c["author"] == "system" for c in system_rows)
  6774. and not any(c.get("from_agent") for c in system_rows),
  6775. "the harness bookkeeping rows are authored 'system' (never agent)")
  6776. goal_thread = goal_state(goal)["comments"]
  6777. check(any(c.get("is_system") for c in goal_thread),
  6778. "the goal page reads the same thread (the system rows included)")
  6779. css = open(os.path.join(REPO, "src/pages/app.css")).read()
  6780. check(".comment.system" in css and ".system-label" in css,
  6781. "the stylesheet ships the distinct system-event styling")
  6782. page = http_get_page(auth, "/goals/%d" % goal)
  6783. check("comment.is_system" in page and "system-label" in page,
  6784. "the goal page renders system rows with the labelled muted style")
  6785. task_page = http_get_page(auth, "/tasks/%d" % worker)
  6786. check("comment.is_system" in task_page and "system-label" in task_page,
  6787. "the task page renders system rows with the same style")
  6788. print("== S5 U2: mechanical rows filtered from user-facing timelines ==")
  6789. # The daemon's mechanical system entries (the park notice, the
  6790. # restore footnote) ride the wire typed 'system': the raw terminal
  6791. # row STORES (session_commands), but no comment row, narrative
  6792. # event or dashboard timeline row is authored for them.
  6793. open_run = open_session(run_task)
  6794. mechanical = [
  6795. {"id": 9501, "command": "saddle suspend",
  6796. "output": "goal agent parked until an owned task completes or"
  6797. " fails; resume via /start resume_notice",
  6798. "exit_code": 0, "type": "system", "timestamp": "2026-09-12T00:00:00Z"},
  6799. {"id": 9502, "command": "harness resume",
  6800. "output": "[harness] session context restored — task resumed",
  6801. "exit_code": 0, "type": "system", "timestamp": "2026-09-12T00:00:01Z"},
  6802. ]
  6803. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6804. {"session_id": open_run, "state": "suspended",
  6805. "entries": mechanical})
  6806. check(status == 200, "the mechanical system entries push cleanly")
  6807. conn = db()
  6808. raw_rows = conn.execute(
  6809. "select count(*) from session_commands where session_id = ?"
  6810. " and type = 'system'", (open_run,)).fetchone()[0]
  6811. conn.close()
  6812. check(raw_rows >= 2,
  6813. "the raw terminal keeps the mechanical rows (typed entries store)")
  6814. check(all("parked until" not in c["body"]
  6815. and "session context restored" not in c["body"]
  6816. for c in task_state(auth, run_task)["comments"]),
  6817. "no mechanical comment row lands on the thread (U2)")
  6818. check(all("parked until" not in e["text"]
  6819. and "session context restored" not in e["text"]
  6820. for e in goal_state(goal)["events"]),
  6821. "no mechanical narrative event lands on the goal page (U2)")
  6822. dash_events = slot_public(auth, "/", "dashboard")["activity"]
  6823. check(all("parked until" not in e["text"]
  6824. and "session context restored" not in e["text"]
  6825. for e in dash_events),
  6826. "no mechanical row lands on the dashboard timeline (U2)")
  6827. print("== C4: head line, answered-blocks expander, footnote comments ==")
  6828. state = goal_state(goal)
  6829. check(state["current_activity"] == "parked awaiting its tasks' terminal transitions",
  6830. "a suspended run folds into the one muted head line")
  6831. check(len(state.get("answered_blocks") or []) >= 2
  6832. and len(state.get("open_blocks") or []) == 0,
  6833. "the historic answered blocks expose as their own vector (open ones empty)")
  6834. goal_page = http_get_page(auth, "/goals/%d" % goal)
  6835. check("Answered blocks (" in goal_page and "comment-meta" in goal_page,
  6836. "the answered blocks and the footnote comment meta ship in the markup")
  6837. check("timeless note for a future reader" in goal_page,
  6838. "the composer hint states the timeless-comment rule")
  6839. css = open(os.path.join(REPO, "src/pages/app.css")).read()
  6840. check(".goal-page .card { margin-bottom" in css and ".comment-meta" in css,
  6841. "the stylesheet ships the card rhythm and the comment footnote meta")
  6842. print("== C1: one review block per verdict path (F6) ==")
  6843. # The daemon's honest round pushes the ENTRY and the STATE
  6844. # together — the done pair must raise exactly ONE completion
  6845. # block, and a redelivered pair nothing more.
  6846. fake.goal_script = [("done", "# Completion\n\nOne block only.")]
  6847. conn = db()
  6848. conn.execute("insert into tasks (project_id, goal_id, kind, title, status,"
  6849. " priority, completed_at, created_at) values (1, ?, 'task',"
  6850. " 'Warmup', 'done', 5,"
  6851. " strftime('%s','now'),"
  6852. " strftime('%s','now','-30 seconds'))", (goal,))
  6853. conn.commit()
  6854. conn.close()
  6855. check(wait_for(lambda: goal_state(goal)["status"] == "completion_review" or False,
  6856. timeout=60, what="goalui completion review opens") is not None,
  6857. "the scripted done round enters completion review")
  6858. def completion_block_count():
  6859. conn = db()
  6860. try:
  6861. return conn.execute(
  6862. "select count(*) from task_blocks where task_id = ?"
  6863. " and type = 'completion'", (run_task,)).fetchone()[0]
  6864. finally:
  6865. conn.close()
  6866. check(completion_block_count() == 1,
  6867. "the done state raises exactly ONE completion block")
  6868. # Plan 16 S5 (U1): the verdict-shaped block renders the rich review
  6869. # form — explicit accept/reject affordances with the free-text
  6870. # feedback path, never a free-text-only verdict.
  6871. goal_page = http_get_page(auth, "/goals/%d" % goal)
  6872. check("Accept goal" in goal_page and "Reject with feedback" in goal_page
  6873. and 'name="feedback"' in goal_page,
  6874. "the completion review presents explicit accept/reject + free-text feedback (U1)")
  6875. # The burst shape: the done ENTRY and the done STATE in one push
  6876. # (fresh entry ids), then the whole pair redelivered — the
  6877. # pending-report idempotency leaves the one row alone.
  6878. conn = db()
  6879. burst_session = conn.execute(
  6880. "select id from sessions where task_id = ? and ended_at is null",
  6881. (run_task,)).fetchone()[0]
  6882. conn.close()
  6883. done_pair = [{"id": 9401, "command": "saddle done",
  6884. "output": "# Completion\n\nOne block only.", "exit_code": 0,
  6885. "type": "done", "timestamp": "2026-09-12T00:00:00Z"}]
  6886. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  6887. {"session_id": burst_session, "state": "done",
  6888. "entries": done_pair,
  6889. "summary": "# Completion\n\nOne block only."})
  6890. check(status == 200 and completion_block_count() == 1,
  6891. "the done entry + state pair raises nothing more (one block, ever)")
  6892. state = goal_state(goal)
  6893. act(auth, state["completion_pending_block"]["answer_action"],
  6894. {"verdict": "accept"})
  6895. check(wait_for(lambda: goal_state(goal)["status"] == "complete" or False,
  6896. timeout=30, what="goalui goal completes") is not None,
  6897. "accepting the completion completes the goal")
  6898. check(completion_block_count() == 1,
  6899. "the accept path leaves exactly one review-kind block behind")
  6900. conn = db()
  6901. stragglers = conn.execute(
  6902. "select count(*) from task_blocks where task_id = ? and answer is null",
  6903. (run_task,)).fetchone()[0]
  6904. conn.close()
  6905. check(stragglers == 0,
  6906. "no unanswered straggler rows survive the completion")
  6907. print("== S2: server-owned verdicts + the container-resolved reject ==")
  6908. # A fresh goal drives the verdict paths the goal page's forms now
  6909. # use: verdict markers for accept, raw `feedback` for reject, and
  6910. # the block resolution living in the review CONTAINER (not
  6911. # goal.plan_pending_block — the diagnosed completion-review bug).
  6912. fake.goal_script = [("plan", "# S2 plan\n\nOne step")]
  6913. act(auth, slot_public(auth, "/goals", "goals")["create"], {
  6914. "project_id": "1", "title": "S2 verdict goal",
  6915. "brief": "Covers the plan-16 S2 verdict paths.",
  6916. "branch_name": "goal/s2-verdicts"})
  6917. conn = db()
  6918. s2_goal = int(conn.execute(
  6919. "select id from goals where title = 'S2 verdict goal'").fetchone()[0])
  6920. conn.close()
  6921. act(auth, goal_state(s2_goal)["start"], {})
  6922. check(wait_for(lambda: goal_state(s2_goal)["status"] == "plan_review" or False,
  6923. timeout=90, what="S2 goal enters plan review") is not None,
  6924. "the S2 goal enters plan review")
  6925. s2_state = goal_state(s2_goal)
  6926. s2_pending = s2_state["plan_pending_block"]
  6927. # (1) A remark-less, text-less rejection is refused: the action
  6928. # errors and the block keeps waiting.
  6929. refusal = act(auth, s2_pending["answer_action"], {"feedback": " "})
  6930. check(any(d.get("type") == "notify" and d.get("kind") == "error"
  6931. and "answer is required" in (d.get("message") or "").lower()
  6932. for d in refusal),
  6933. "a remark-less, text-less rejection is refused with an error")
  6934. conn = db()
  6935. s2_still_open = conn.execute(
  6936. "select answer from task_blocks where id = ?",
  6937. (s2_pending["id"],)).fetchone()[0]
  6938. conn.close()
  6939. check(s2_still_open is None,
  6940. "the refused rejection leaves the block unanswered")
  6941. # (3) The goal page template ships no verdict constants and its
  6942. # script resolves the review block from ANY [data-plan-review]
  6943. # container via the open-blocks list (the completion-review fix).
  6944. # Fetched while the review is open so the accept form renders.
  6945. s2_page = http_get_page(auth, "/goals/%d" % s2_goal)
  6946. check("Plan approved. Proceed with implementation." not in s2_page
  6947. and "Review approved. Continue with the task." not in s2_page,
  6948. "the goal page ships no verdict constants (server-owned wording)")
  6949. check('name="verdict" value="accept"' in s2_page,
  6950. "the goal page's accept forms post the verdict marker")
  6951. check("resolveReviewBlock" in s2_page and "goal.open_blocks" in s2_page,
  6952. "the reject flow resolves its block from the open-blocks list (container-keyed)")
  6953. check("statum.act(block.answer_action, { feedback: feedback })" in s2_page,
  6954. "the reject flow posts raw feedback (the server wraps the prefix)")
  6955. check("extraText.length === 0" in s2_page,
  6956. "a remark-less rejection with textarea content is allowed (S2)")
  6957. # (2) The goal page's reject rides `feedback` (the client composes
  6958. # the numbered-remarks body) and the SERVER wraps it with the plan
  6959. # prefix — byte-identical to the historical client-built answer.
  6960. act(auth, s2_pending["answer_action"],
  6961. {"feedback": "> ## One step\n\n1. Tighten it\n\n2. Name the owner"})
  6962. conn = db()
  6963. s2_wrapped = conn.execute(
  6964. "select answer from task_blocks where id = ?",
  6965. (s2_pending["id"],)).fetchone()[0]
  6966. conn.close()
  6967. check(s2_wrapped
  6968. == "Plan feedback received:\n\n"
  6969. "> ## One step\n\n1. Tighten it\n\n2. Name the owner",
  6970. "the plan rejection wraps the client-shaped body with the stable prefix (%r)"
  6971. % s2_wrapped)
  6972. check(goal_state(s2_goal)["status"] == "planning",
  6973. "the feedback rejection returns the agent to planning")
  6974. BUNDLES_ORDER = ("pages", "dispatch", "subtasks", "lineage",
  6975. "allocation", "lifecycle", "answers", "push", "goals",
  6976. "replay", "goalui")
  6977. def run_bundle(name):
  6978. """Runs one shard to completion; returns a failure count."""
  6979. started = time.monotonic()
  6980. ctx = start_bundle(name)
  6981. runner = globals()["bundle_" + name]
  6982. try:
  6983. runner(ctx)
  6984. finally:
  6985. # Exit-code diagnostic: distinguishes a crashed app (negative
  6986. # returncode, e.g. -11 SIGSEGV) from a clean shutdown. When
  6987. # tearing down mid-exception, also capture the still-alive
  6988. # app's stacks before killing it (a wedged-but-alive server
  6989. # otherwise leaves no evidence).
  6990. print("(app returncode at teardown: %s)" % ctx.app.poll())
  6991. if ctx.app.poll() is None and sys.exc_info()[0] is not None:
  6992. dump_app_stacks(ctx.app.pid)
  6993. ctx.fake.kill()
  6994. try:
  6995. ctx.app.terminate()
  6996. try:
  6997. ctx.app.wait(timeout=5)
  6998. except subprocess.TimeoutExpired:
  6999. ctx.app.kill()
  7000. check(ctx.app.poll() in (0, -15),
  7001. "bundle %s app shut down cleanly (exit %s)"
  7002. % (name, ctx.app.poll()))
  7003. except OSError:
  7004. pass
  7005. print("%d passed, %d failed" % (passes, failures))
  7006. print("[bundle %s] %.1fs" % (name, time.monotonic() - started))
  7007. return failures
  7008. def run_all_parallel():
  7009. """The default meson entry: one subprocess per bundle (each with its
  7010. own app, daemons and database), verdicts aggregated into one exit
  7011. code and one combined check-count summary."""
  7012. base = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
  7013. log_dir = os.path.join(os.path.dirname(base) or "/tmp/kilo",
  7014. "webstack-shards")
  7015. os.makedirs(log_dir, exist_ok=True)
  7016. procs = {}
  7017. started = time.monotonic()
  7018. for name in BUNDLES_ORDER:
  7019. out_path = os.path.join(log_dir, "bundle-%s.log" % name)
  7020. handle = open(out_path, "w")
  7021. procs[name] = (subprocess.Popen(
  7022. [sys.executable, "-u", os.path.abspath(__file__), APP,
  7023. "--bundle", name],
  7024. stdout=handle, stderr=subprocess.STDOUT), handle, out_path)
  7025. total_pass = total_fail = 0
  7026. broken = []
  7027. for name, (proc, handle, out_path) in procs.items():
  7028. try:
  7029. proc.wait(timeout=SHARD_TIMEOUT_S)
  7030. except subprocess.TimeoutExpired:
  7031. proc.kill()
  7032. proc.wait()
  7033. handle.close()
  7034. text = open(out_path).read()
  7035. match = re.search(r"(\d+) passed, (\d+) failed", text)
  7036. took = re.search(r"\[bundle %s\] ([0-9.]+)s" % name, text)
  7037. bundle_pass = int(match.group(1)) if match else 0
  7038. bundle_fail = int(match.group(2)) if match else -1
  7039. total_pass += max(bundle_pass, 0)
  7040. total_fail += max(bundle_fail, 0)
  7041. status = "ok"
  7042. if proc.returncode != 0 or bundle_fail != 0 or match is None:
  7043. status = "BROKEN (exit %s)" % proc.returncode
  7044. broken.append(name)
  7045. print("[shard %-10s] %6ss %3d passed %3d failed %s"
  7046. % (name, took.group(1) if took else "?",
  7047. bundle_pass, bundle_fail, status))
  7048. if status != "ok":
  7049. tail = "\n".join(text.rstrip().split("\n")[-60:])
  7050. print("--- web-stack/%s tail (full log: %s) ---" % (name, out_path))
  7051. print(tail)
  7052. print("--- end web-stack/%s tail ---" % name)
  7053. print("shards wall: %.1fs (logs under %s)" % (time.monotonic() - started, log_dir))
  7054. print("ALL SHARDS: %d passed, %d failed%s"
  7055. % (total_pass, total_fail,
  7056. "" if not broken else " BROKEN: " + ", ".join(broken)))
  7057. return 1 if broken or total_fail else 0
  7058. def main():
  7059. if not os.path.exists(APP):
  7060. raise SystemExit("app binary not found at %s (build first)" % APP)
  7061. argv = sys.argv[2:]
  7062. if "--bundle" in argv:
  7063. sys.exit(1 if run_bundle(argv[argv.index("--bundle") + 1]) else 0)
  7064. if os.environ.get("WEBSTACK_SEQUENTIAL"):
  7065. # Debug path: every bundle in-process, one after another.
  7066. for name in BUNDLES_ORDER:
  7067. if run_bundle(name):
  7068. return 1
  7069. return 0
  7070. return run_all_parallel()
  7071. if __name__ == "__main__":
  7072. sys.exit(main())