SaddleService.vala 101 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031
  1. using Invercargill;
  2. using Invercargill.DataStructures;
  3. using Invercargill.Expressions;
  4. using InvercargillSql;
  5. using InvercargillSql.Orm;
  6. using Inversion;
  7. using Soup;
  8. using ClankerSaddle.Data;
  9. namespace ClankerSaddle.Services {
  10. /**
  11. * Valid saddle statuses (the `saddles.status` wire values, plan 16
  12. * R2 / arch item 26 — the raw literals died with these). The row
  13. * default and the migration seed keep their literals (the Data
  14. * layer does not reference the services).
  15. */
  16. public const string SADDLE_IDLE = "idle";
  17. public const string SADDLE_BUSY = "busy";
  18. public const string SADDLE_OFFLINE = "offline";
  19. /**
  20. * Session outcomes (the `sessions.outcome` values the park, the
  21. * ONE release path and the terminal pushes write): `parked` (a
  22. * round that parked on a block or a suspend — the row closes at
  23. * park time and the workspace awaits its resuming allocation;
  24. * plan 16 D4, aligned with the daemon's file-log brackets),
  25. * `stopped` (a cancelled/requeued/unreachable release),
  26. * `done`/`failed` (the terminal pushes).
  27. */
  28. public const string OUTCOME_STOPPED = "stopped";
  29. public const string OUTCOME_PARKED = "parked";
  30. public const string OUTCOME_DONE = "done";
  31. public const string OUTCOME_FAILED = "failed";
  32. /** Caps of the `project_notes` array in the /start payload. */
  33. public const int64 PROJECT_NOTES_MAX_ENTRIES = 30;
  34. public const int64 PROJECT_NOTES_MAX_CHARS = 20000;
  35. /**
  36. * Cap of the `understanding` member in the /start payload: the
  37. * primary project's current understanding document (goal 24).
  38. * The whole document rides the payload and renders into the
  39. * system prompt next to the project notes, so an over-grown
  40. * revision is head-clamped rather than bloating every prompt.
  41. */
  42. public const int64 UNDERSTANDING_MAX_CHARS = 16000;
  43. /**
  44. * Caps of the `discoveries` array in the /start payload (goal 24):
  45. * the fleet's newest discoveries — non-project facts earlier
  46. * agents recorded — so a fresh agent inherits the knowledge base
  47. * before it ever searches. Head-kept char cap across the whole
  48. * array, like PROJECT_NOTES_MAX_CHARS.
  49. */
  50. public const int64 DISCOVERIES_MAX_ENTRIES = 10;
  51. public const int64 DISCOVERIES_MAX_CHARS = 8000;
  52. /**
  53. * Upper bound of a saddle's prompt_note served to the daemon (goal 8
  54. * task 5): the note rides EVERY assignment payload, so a fat note
  55. * would bloat every system prompt and cache.
  56. */
  57. public const int64 PROMPT_NOTE_MAX_CHARS = 2000;
  58. /** A block answer awaiting collection by one saddle's next poll. */
  59. public class PendingAnswer : Object {
  60. public int64 block_id { get; set; }
  61. public string answer { get; set; default = ""; }
  62. public bool branch_advanced { get; set; }
  63. }
  64. /** A saddle registration plus the API token generated for it. */
  65. public class SaddleRegistration : Object {
  66. public Saddle saddle { get; set; }
  67. public string token { get; set; default = ""; }
  68. }
  69. /**
  70. * A saddle's current work as the cards render it: the running
  71. * task's title, plus the GOAL it stands for when that task is a
  72. * goal agent's internal run.
  73. */
  74. public class CurrentWork : Object {
  75. public int64 task_id { get; set; }
  76. public string task_title { get; set; default = ""; }
  77. /** Set when the current task is a goal agent's run: the card links the goal. */
  78. public int64 goal_id { get; set; }
  79. public string goal_title { get; set; default = ""; }
  80. /** The running session the derivation keyed on (plan 16 item 23). */
  81. public int64 session_id { get; set; }
  82. /** When that session started — the "allocated" display stamp. */
  83. public DateTime? started_at { get; set; default = null; }
  84. }
  85. /**
  86. * The OpenAI-compatible endpoint every saddle assignment is sent to,
  87. * read from the environment: `SADDLE_LLM_BASE_URL` (required — dispatch
  88. * pauses without it), `SADDLE_LLM_API_KEY` and `SADDLE_LLM_MODEL`.
  89. */
  90. public class LlmEndpoint : Object {
  91. public string base_url { get; set; default = ""; }
  92. public string api_key { get; set; default = ""; }
  93. public string model { get; set; default = ""; }
  94. /** The configured endpoint, or null when the base URL is unset. */
  95. public static LlmEndpoint? from_environment() {
  96. var base_url = (Environment.get_variable("SADDLE_LLM_BASE_URL") ?? "").strip();
  97. if (base_url.length == 0) {
  98. return null;
  99. }
  100. return new LlmEndpoint() {
  101. base_url = base_url,
  102. api_key = (Environment.get_variable("SADDLE_LLM_API_KEY") ?? "").strip(),
  103. model = (Environment.get_variable("SADDLE_LLM_MODEL") ?? "").strip()
  104. };
  105. }
  106. }
  107. /**
  108. * Saddle registry and control-plane client (plan 10 phase 2): registers
  109. * containers, derives idle capacity for the dispatcher's wake
  110. * computation (plan 16 item 23: busy iff an open session on an
  111. * assigned/working task exists — {@link current_work_view} is the ONE
  112. * truth accessor; the saddle row keeps only the OFFLINE probe state),
  113. * wakes sleeping daemons, and ingests the telemetry the daemons PUSH —
  114. * raw PTY chunks (`POST /api/saddle/chunks`) and round results with
  115. * state transitions (`POST /api/saddle/results`) — reusing the storage
  116. * logic the deleted `/output` + `/log` pollers ran. GET /status
  117. * against a daemon is exceptional only: the staleness probe for a
  118. * silent runner and the manual refresh button. Block answers are
  119. * handed out exactly once (delivered ids are remembered in memory; a
  120. * restart re-delivers and the daemon deduplicates).
  121. */
  122. public class SaddleService : Object {
  123. private OrmSession db = inject<OrmSession>();
  124. private Connection connection = inject<Connection>();
  125. /**
  126. * Every daemon request shares this session and therefore its
  127. * timeouts: without them a single hung request (a daemon wedged
  128. * mid-response over the keep-alive pool) froze the dispatcher's
  129. * tick latch forever. `timeout` bounds each request
  130. * (SADDLE_HTTP_TIMEOUT_S, default 30s); `idle_timeout` reaps
  131. * pooled connections so a peer that vanished between requests is
  132. * not dialed again.
  133. */
  134. private Soup.Session http = new Soup.Session() {
  135. timeout = http_timeout_seconds(),
  136. idle_timeout = 10
  137. };
  138. private static uint http_timeout_seconds() {
  139. var raw = Environment.get_variable("SADDLE_HTTP_TIMEOUT_S") ?? "";
  140. int64 parsed = int64.parse(raw);
  141. return parsed > 0 ? (uint) parsed : 30;
  142. }
  143. private SaddlePollState poll_state = inject<SaddlePollState>();
  144. private StatePulse pulse = inject<StatePulse>();
  145. private PushService push = inject<PushService>();
  146. private RuleService rules = inject<RuleService>();
  147. private CapabilityService capability_service = inject<CapabilityService>();
  148. private TaskService task_service = inject<TaskService>();
  149. private GoalService goal_service = inject<GoalService>();
  150. private EventService event_service = inject<EventService>();
  151. private UnderstandingService understandings = inject<UnderstandingService>();
  152. private DiscoveryService discoveries = inject<DiscoveryService>();
  153. private Astralis.WebConfig config = inject<Astralis.WebConfig>();
  154. /**
  155. * Registers a saddle container: inserts the row with a freshly
  156. * minted per-saddle token (plan 10) and hands the credential to
  157. * the daemon via `POST /register {saddle_token, web_base_url}`
  158. * (best effort — an older daemon without the route stays
  159. * registered; the operator copies the token from the saddles
  160. * page instead). `host` is host:port of its control server; the
  161. * returned {@link SaddleRegistration} carries the token the
  162. * daemon must present as `Authorization: Bearer …` on
  163. * `/api/saddle/*` calls.
  164. */
  165. public async SaddleRegistration register_saddle(string name, string host) throws Error {
  166. var saddle = new Saddle() {
  167. name = name,
  168. host = host,
  169. status = SADDLE_IDLE,
  170. token = mint_saddle_token()
  171. };
  172. yield db.insert_async<Saddle>(saddle);
  173. yield register_daemon(saddle);
  174. yield pulse.bump(0, "saddle registered");
  175. return new SaddleRegistration() { saddle = saddle, token = saddle.token ?? "" };
  176. }
  177. /** A fresh per-saddle bearer token (`saddle-<random>`). */
  178. private static string mint_saddle_token() {
  179. return @"saddle-$(Uuid.string_random().replace("-", ""))";
  180. }
  181. /**
  182. * Delivers the per-saddle token to the daemon's
  183. * `POST /register` endpoint (wire contract): the daemon
  184. * persists it for its calls home and answers
  185. * `{ok, ssh_public_key, version}` — the key is stored on the
  186. * row the moment the daemon reports one. A daemon without the
  187. * route (HTTP 404/405) or unreachable at registration time is
  188. * tolerated: the row keeps the token, the operator can rety
  189. * via a refresh or copy the token manually.
  190. */
  191. public async void register_daemon(Saddle saddle) throws Error {
  192. var payload = new Json.Builder();
  193. payload.begin_object();
  194. payload.set_member_name("saddle_token");
  195. payload.add_string_value(saddle.token ?? "");
  196. payload.set_member_name("web_base_url");
  197. payload.add_string_value(callback_base_url());
  198. payload.end_object();
  199. try {
  200. var response = yield request_saddle_json(saddle.host, "POST", "/register",
  201. (owned) payload);
  202. string reported_key = EntryJson.string_member(response, "ssh_public_key");
  203. // goal 8 task 6: the daemon reports the workspace root it
  204. // mounts so assignment paths address the ASKING saddle's
  205. // filesystem; absent on older daemons (kept null → the
  206. // SADDLE_PROJECTS_ROOT fallback chain)
  207. string reported_root = EntryJson.string_member(response, "workspace_root");
  208. if ((reported_key.length > 0 && reported_key != (saddle.ssh_public_key ?? ""))
  209. || reported_root != (saddle.workspace_root ?? "")) {
  210. if (reported_key.length > 0) {
  211. saddle.ssh_public_key = reported_key;
  212. }
  213. saddle.workspace_root = reported_root.length > 0 ? reported_root : null;
  214. yield db.update_async<Saddle>(saddle);
  215. }
  216. print(@"[ClankerSaddle] Registered with saddle '$(saddle.name)'s daemon (token delivered)\n");
  217. } catch (Error e) {
  218. print(@"[ClankerSaddle] Saddle '$(saddle.name)'s daemon did not accept /register ($(e.message)); token stays on the row\n");
  219. }
  220. }
  221. /**
  222. * The saddle a bearer token belongs to, or null — the
  223. * credential check for the id-less `/api/saddle/*` endpoints
  224. * (`next-task`, `note`, `task`): the per-saddle row token,
  225. * the single credential store since plan 10 (arch 24 dropped
  226. * the legacy api_tokens fallback — nothing deployed predates
  227. * the row token). Removing the saddle revokes it.
  228. */
  229. public async Saddle? saddle_for_token(string token) throws Error {
  230. if (token.length == 0) {
  231. return null;
  232. }
  233. return yield db.query<Saddle>()
  234. .where(expr("token == $0", elem(token)))
  235. .first_async();
  236. }
  237. /**
  238. * This app's externally reachable base URL — what daemons use to
  239. * POST notes back — read from the `saddle_callback_base_url` key
  240. * of web-config.json, falling back to the
  241. * `SADDLE_CALLBACK_BASE_URL` environment variable and then to
  242. * `http://localhost:8080`. The daemon typically runs in a
  243. * container or on another host, so its view of this app differs
  244. * from the browser's; set the key to whatever reaches this
  245. * process from there.
  246. */
  247. public string callback_base_url() {
  248. string configured = config.get_string("saddle_callback_base_url", "").strip();
  249. if (configured.length > 0) {
  250. return configured;
  251. }
  252. var from_env = (Environment.get_variable("SADDLE_CALLBACK_BASE_URL") ?? "").strip();
  253. if (from_env.length > 0) {
  254. return from_env;
  255. }
  256. return "http://localhost:8080";
  257. }
  258. /**
  259. * Removes a saddle registration; the task it was running (the
  260. * DERIVED running work — plan 16 item 23: sessions are the
  261. * truth) releases through the ONE path — back to the queue,
  262. * its open sessions closed stopped (no orphan open session
  263. * outlives the row). Revocation: the row dies here, and its
  264. * per-saddle token with it — nothing authorises
  265. * `/api/saddle/*` for it afterwards.
  266. */
  267. public async void unregister_saddle(int64 id) throws Error {
  268. var saddle = yield get_saddle(id);
  269. if (saddle == null) {
  270. return;
  271. }
  272. var work = yield current_work_view((!)saddle);
  273. if (work != null) {
  274. yield task_service.release_task(((!)work).task_id, OUTCOME_STOPPED, true);
  275. }
  276. yield db.delete_async<Saddle>((!)saddle);
  277. yield pulse.bump(0, "saddle removed");
  278. }
  279. /** Loads the saddle identified by `id`, or null. */
  280. public async Saddle? get_saddle(int64 id) throws Error {
  281. return yield db.query<Saddle>()
  282. .where(expr("id == $0", elem<int64?>(id)))
  283. .first_async();
  284. }
  285. /** Lists every saddle ordered by name. */
  286. public async ImmutableLot<Saddle> list_saddles() throws Error {
  287. return yield db.query<Saddle>()
  288. .order_by(expr("name"))
  289. .materialise_async();
  290. }
  291. /**
  292. * The saddle's current work resolved from the SESSION records —
  293. * the ONE truth accessor every surface reads (plan 16 item 23:
  294. * sessions own "who runs what"; the saddle row's cache columns
  295. * are gone). The running session is the newest open session on
  296. * the saddle whose task is still assigned/working — busy iff it
  297. * exists, idle otherwise; a parked task's session closed
  298. * 'parked' at its park (D4), so parks read as free exactly
  299. * like the daemon's file-log brackets say. The task join also
  300. * makes the derivation drift-tolerant: a stale open row on a
  301. * task that has since moved on cannot wedge a saddle busy.
  302. * Null when the saddle holds no running work.
  303. */
  304. public async CurrentWork? current_work_view(Saddle saddle) throws Error {
  305. var session = yield running_session_on_saddle(saddle.id);
  306. if (session == null) {
  307. return null;
  308. }
  309. var work = new CurrentWork() {
  310. task_id = ((!)session).task_id,
  311. session_id = ((!)session).id,
  312. started_at = ((!)session).started_at
  313. };
  314. var task = yield db.query<TaskEntity>()
  315. .where(expr("id == $0", elem<int64?>(((!)session).task_id)))
  316. .first_async();
  317. if (task == null) {
  318. return work;
  319. }
  320. work.task_title = ((!)task).title;
  321. if (((!)task).kind == TASK_KIND_GOAL_RUN) {
  322. var goal = yield db.query<Goal>()
  323. .where(expr("id == $0", elem<int64?>(((!)task).goal_id)))
  324. .first_async();
  325. if (goal != null) {
  326. work.goal_id = ((!)goal).id;
  327. work.goal_title = ((!)goal).title;
  328. }
  329. }
  330. return work;
  331. }
  332. /**
  333. * The derived busy/idle/offline status label every saddle view
  334. * shares — 'busy' iff {@link current_work_view} holds running
  335. * work, else the row's stored OFFLINE probe state ('offline' or
  336. * 'idle').
  337. */
  338. public static string derived_status(Saddle saddle, CurrentWork? work) {
  339. if (work != null) {
  340. return SADDLE_BUSY;
  341. }
  342. return saddle.status == SADDLE_OFFLINE ? SADDLE_OFFLINE : SADDLE_IDLE;
  343. }
  344. /**
  345. * The saddle's RUNNING session — the newest open session whose
  346. * task is assigned/working (the null check runs in SQL: the
  347. * expression layer cannot compare columns against null). At
  348. * most one exists per saddle by construction: every park closes
  349. * its row (D4) and the claim's guard refuses a saddle that
  350. * already holds one.
  351. */
  352. private async ClankerSaddle.Data.Session? running_session_on_saddle(int64 saddle_id)
  353. throws Error {
  354. var command = connection.create_command(
  355. "SELECT s.id AS session_id, s.task_id AS task_id, s.started_at AS started"
  356. + " FROM sessions s JOIN tasks t ON t.id = s.task_id"
  357. + " WHERE s.saddle_id = :saddle AND s.ended_at IS NULL"
  358. + " AND t.status IN ('assigned', 'working')"
  359. + " ORDER BY s.id DESC LIMIT 1");
  360. command.with_parameter<Invercargill.Element>("saddle",
  361. elem<int64?>(saddle_id));
  362. ClankerSaddle.Data.Session? found = null;
  363. foreach (var row in yield command.execute_query_async()) {
  364. int64? session_id = 0;
  365. int64? task_id = 0;
  366. ((!)row.get("session_id")).try_get_as<int64?>(out session_id);
  367. ((!)row.get("task_id")).try_get_as<int64?>(out task_id);
  368. found = new ClankerSaddle.Data.Session() {
  369. id = session_id ?? 0,
  370. task_id = task_id ?? 0,
  371. saddle_id = saddle_id,
  372. started_at = raw_date_time_of(row.get("started"))
  373. };
  374. }
  375. return found;
  376. }
  377. /** A raw query column back as a DateTime (null when null/unreadable). */
  378. private static DateTime? raw_date_time_of(Invercargill.Element? element) {
  379. if (element == null) {
  380. return null;
  381. }
  382. DateTime value;
  383. return ((!)element).try_get_as<DateTime>(out value) ? value : null;
  384. }
  385. /**
  386. * The ids (decimal strings — the OR-chain batch convention) of
  387. * every saddle holding a running session — the derived busy
  388. * set the fleet lists and the offline stamp share.
  389. */
  390. private async Invercargill.DataStructures.HashSet<string> busy_saddle_ids()
  391. throws Error {
  392. var busy = new Invercargill.DataStructures.HashSet<string>();
  393. var command = connection.create_command(
  394. "SELECT DISTINCT s.saddle_id AS saddle_id FROM sessions s"
  395. + " JOIN tasks t ON t.id = s.task_id"
  396. + " WHERE s.ended_at IS NULL"
  397. + " AND t.status IN ('assigned', 'working')");
  398. foreach (var row in yield command.execute_query_async()) {
  399. int64? saddle_id = 0;
  400. ((!)row.get("saddle_id")).try_get_as<int64?>(out saddle_id);
  401. if (saddle_id != null) {
  402. busy.add(((!)saddle_id).to_string());
  403. }
  404. }
  405. return busy;
  406. }
  407. /**
  408. * Saddles holding a running session (the dispatcher's poll
  409. * candidates) — derived from the session records, registration
  410. * order.
  411. */
  412. public async ImmutableLot<Saddle> list_busy_saddles() throws Error {
  413. var busy = yield busy_saddle_ids();
  414. if (busy.count() == 0) {
  415. return (new Vector<Saddle>()).to_immutable_buffer();
  416. }
  417. var rows = new Vector<Saddle>();
  418. foreach (var saddle in yield db.query<Saddle>()
  419. .where(TaskService.id_in("id", busy))
  420. .order_by(expr("id"))
  421. .materialise_async()) {
  422. rows.add(saddle);
  423. }
  424. return rows.to_immutable_buffer();
  425. }
  426. /**
  427. * Persists the saddle's git identity — what its daemon commits
  428. * as (`git_identity` in every /start). Empty/null clears the
  429. * fields so the daemon's fallback applies.
  430. */
  431. public async void update_git_identity(int64 id, string? git_name,
  432. string? git_email) throws Error {
  433. var saddle = yield get_saddle(id);
  434. if (saddle == null) {
  435. throw new Error(Quark.from_string("clanker-saddle"), 17,
  436. @"No saddle exists with id $(id)");
  437. }
  438. ((!)saddle).git_name = git_name != null && git_name.length > 0 ? git_name : null;
  439. ((!)saddle).git_email = git_email != null && git_email.length > 0 ? git_email : null;
  440. yield db.update_async<Saddle>((!)saddle);
  441. yield pulse.bump(0, "saddle identity saved");
  442. }
  443. /**
  444. * Saves the saddle admin's standing prompt note (goal 8 task 5):
  445. * the message about THIS machine rendered into every
  446. * assignment's system prompt. Empty clears it (the daemon then
  447. * omits the section entirely); the value is clamped to
  448. * {@link PROMPT_NOTE_MAX_CHARS} so a fat note can never bloat
  449. * every prompt.
  450. */
  451. public async void update_prompt_note(int64 id, string? prompt_note) throws Error {
  452. var saddle = yield get_saddle(id);
  453. if (saddle == null) {
  454. throw new Error(Quark.from_string("clanker-saddle"), 17,
  455. @"No saddle exists with id $(id)");
  456. }
  457. var trimmed = prompt_note != null ? prompt_note.strip() : "";
  458. if (trimmed.length > PROMPT_NOTE_MAX_CHARS) {
  459. trimmed = trimmed.substring(0, (int) PROMPT_NOTE_MAX_CHARS);
  460. }
  461. ((!)saddle).prompt_note = trimmed.length > 0 ? trimmed : null;
  462. yield db.update_async<Saddle>((!)saddle);
  463. yield pulse.bump(0, "saddle prompt note saved");
  464. }
  465. /**
  466. * Persists the daemon's reported `ssh_public_key` onto the saddle
  467. * row (the plan-9 contract: the key travels on /status, empty
  468. * until the daemon generates one; the saddles page shows it for
  469. * adding as a deploy key). A single-column UPDATE — the
  470. * full-row write could stomp a concurrent ask claim. Only writes
  471. * when it changed.
  472. */
  473. private async void store_reported_ssh_key(Saddle saddle, Json.Object status) throws Error {
  474. if (!status.has_member("ssh_public_key")) {
  475. return;
  476. }
  477. string reported = status.get_string_member_with_default("ssh_public_key", "");
  478. string stored = saddle.ssh_public_key ?? "";
  479. if (reported == stored) {
  480. return;
  481. }
  482. var command = connection.create_command(
  483. "UPDATE saddles SET ssh_public_key = :key WHERE id = :id");
  484. command.with_parameter<Invercargill.Element>("key", elem(reported.length > 0 ? reported : null));
  485. command.with_parameter<Invercargill.Element>("id", elem<int64?>(saddle.id));
  486. yield command.execute_non_query_async();
  487. }
  488. /**
  489. * Records a /status observation: the health snapshot, the
  490. * daemon-reported ssh key, and the goal-branch push flags
  491. * (plan 15 C4). The `resumed` flag needs no projection — the
  492. * restore's record is the daemon's own system entry in the
  493. * raw terminal log (plan 16 S5/U2).
  494. */
  495. private async void record_status_probe(Saddle saddle, Json.Object status) throws Error {
  496. string state = status.has_member("state")
  497. ? status.get_string_member_with_default("state", "") : "";
  498. int64 uptime = status.has_member("uptime_seconds")
  499. && status.get_member("uptime_seconds").get_value_type() == typeof(int64)
  500. ? status.get_int_member("uptime_seconds") : 0;
  501. int64 activity = status.has_member("last_activity_epoch")
  502. && status.get_member("last_activity_epoch").get_value_type() == typeof(int64)
  503. ? status.get_int_member("last_activity_epoch") : 0;
  504. poll_state.record_probe(saddle.id, state, uptime, activity);
  505. yield store_reported_ssh_key(saddle, status);
  506. // Plan 15 C4: a daemon running (or parking — the
  507. // workspace stays while the saddle frees) a goal agent
  508. // exposes the per-repo push flags of the goal branches on
  509. // /status; they are recorded as LIVE probe state on the
  510. // saddle (never a stored snapshot on the goal row) and
  511. // the goal page reads them back at render time for its
  512. // advisory push chips. The push rule itself is enforced
  513. // daemon-side at the agent's completion attempt.
  514. if (status.has_member("goal_branches")) {
  515. var goal_branches = status.get_member("goal_branches");
  516. if (goal_branches.get_node_type() == Json.NodeType.ARRAY
  517. || goal_branches.get_node_type() == Json.NodeType.OBJECT) {
  518. var generator = new Json.Generator();
  519. generator.set_root(goal_branches);
  520. poll_state.record_goal_branches(saddle.id, generator.to_data(null));
  521. }
  522. }
  523. // Plan 16 S5 (U2): the daemon's own system entry (typed
  524. // 'system' in the results stream) is the raw-terminal
  525. // record of a workspace restore — the web no longer authors
  526. // a duplicate timeline row for it. The `resumed` flag
  527. // carries no further web-side projection.
  528. }
  529. /**
  530. * Wakes a sleeping daemon (plan 10 wire contract): `POST /wake`
  531. * `{}` — a capable task is ready; sleeping daemons ask for work,
  532. * busy ones ignore it, idle-awake ones re-ask. Idempotent. The
  533. * caller records the pending wake for the lost-wake guard.
  534. */
  535. public async void wake_saddle(Saddle saddle) throws Error {
  536. var payload = new Json.Builder();
  537. payload.begin_object();
  538. payload.end_object();
  539. Json.Node? root = null;
  540. int status_code = yield request_saddle(saddle.host, "POST", "/wake",
  541. (owned) payload, out root);
  542. if (status_code >= 400) {
  543. throw new Error(Quark.from_string("clanker-saddle"), 49,
  544. @"wake returned HTTP $(status_code)");
  545. }
  546. }
  547. /**
  548. * Cancelling / requeueing / stopping a task with a live session
  549. * reaches its daemon (wire contract): `POST /task-update
  550. * {kind: "cancel"}` — the authoritative stop through the daemon's
  551. * abort machinery, the workspace kept on disk per the park
  552. * contract. Success is narrated as "<verb>; daemon aborted"; a
  553. * transport failure marks the saddle offline (the daemon's next
  554. * ask proves it alive again and flips the row back to idle). The
  555. * daemon must never keep running a task the web has moved on
  556. * from as a zombie.
  557. */
  558. public async void abort_live_session(TaskService tasks, int64 task_id,
  559. string verb) throws Error {
  560. var session = yield tasks.open_session_for_task(task_id);
  561. if (session == null) {
  562. return;
  563. }
  564. yield abort_session_on_its_saddle(tasks, (!)session, verb);
  565. }
  566. /**
  567. * SADDLE_STALE_S (default 90s): the silence window after which a
  568. * running daemon owes an exceptional GET /status probe.
  569. */
  570. public static uint stale_after_seconds() {
  571. var raw = Environment.get_variable("SADDLE_STALE_S") ?? "";
  572. int64 parsed = int64.parse(raw);
  573. return parsed > 0 ? (uint) parsed : 90;
  574. }
  575. /**
  576. * {@link abort_live_session} keyed by saddle instead of task —
  577. * the unregister path. The cancel is delivered to the saddle's
  578. * RUNNING session (the derived record; plan 16 item 23) — not
  579. * re-resolved through a task's lowest-id open session, which a
  580. * stale fixture row on another saddle would redirect to the
  581. * wrong daemon.
  582. */
  583. public async void abort_saddle_session(TaskService tasks, int64 saddle_id,
  584. string verb) throws Error {
  585. var session = yield running_session_on_saddle(saddle_id);
  586. if (session == null) {
  587. return;
  588. }
  589. yield abort_session_on_its_saddle(tasks, (!)session, verb);
  590. }
  591. /**
  592. * The cancel delivery core shared by the task-keyed and
  593. * saddle-keyed abort paths: `POST /task-update {kind: cancel}`
  594. * to the daemon holding THIS session.
  595. */
  596. private async void abort_session_on_its_saddle(TaskService tasks,
  597. ClankerSaddle.Data.Session session, string verb) throws Error {
  598. var saddle = yield get_saddle(session.saddle_id);
  599. if (saddle == null) {
  600. return;
  601. }
  602. var payload = new Json.Builder();
  603. payload.begin_object();
  604. payload.set_member_name("kind");
  605. payload.add_string_value("cancel");
  606. payload.end_object();
  607. Json.Node? root = null;
  608. try {
  609. int status_code = yield request_saddle(((!)saddle).host, "POST",
  610. "/task-update", (owned) payload, out root);
  611. if (status_code >= 400) {
  612. throw new Error(Quark.from_string("clanker-saddle"), 45,
  613. @"task-update cancel returned HTTP $(status_code)");
  614. }
  615. yield tasks.add_system_comment(session.task_id, @"$(verb); daemon aborted");
  616. } catch (Error e) {
  617. warning(@"[ClankerSaddle] Abort for task $(session.task_id) failed: $(e.message); marking saddle '$(((!)saddle).name)' offline");
  618. yield mark_saddle_unreachable(((!)saddle).id, tasks,
  619. @"abort delivery failed ($(e.message))");
  620. yield tasks.add_system_comment(session.task_id,
  621. @"$(verb); daemon unreachable — saddle marked offline");
  622. }
  623. }
  624. /**
  625. * Requeues tasks caught running on saddles that have since gone
  626. * offline (a manual status refresh, a restart) — without this
  627. * pass they would wait forever for a daemon that will not answer.
  628. * Only `assigned`/`working` tasks move: parked tasks
  629. * (blocked/review/suspended, their sessions closed 'parked')
  630. * keep their records, which the resume machinery needs. Each
  631. * move is {@link TaskService.requeue_task} — the release owner
  632. * closes the session (the close IS the free, item 23) — plus a
  633. * system comment so the timeline shows why the task restarted.
  634. */
  635. public async void requeue_tasks_on_offline_saddles(TaskService tasks) throws Error {
  636. foreach (var status in new string[] { STATUS_ASSIGNED, STATUS_WORKING }) {
  637. foreach (var task in yield db.query<TaskEntity>()
  638. .where(expr("status == $0", elem(status)))
  639. .materialise_async()) {
  640. var session = yield tasks.open_session_for_task(task.id);
  641. if (session == null) {
  642. continue;
  643. }
  644. if (yield tasks.newest_block_answered(task.id)) {
  645. // Parked-with-answered: a verdict awaits the
  646. // resume machinery — never requeue it.
  647. continue;
  648. }
  649. var saddle = yield get_saddle(((!)session).saddle_id);
  650. if (saddle == null || ((!)saddle).status != SADDLE_OFFLINE) {
  651. continue;
  652. }
  653. print(@"[ClankerSaddle] Task $(task.id) requeued: its saddle went offline\n");
  654. yield tasks.requeue_task(task.id);
  655. yield tasks.add_system_comment(task.id,
  656. "saddle went offline; task requeued");
  657. }
  658. }
  659. }
  660. /**
  661. * Every saddle free for work — derived (plan 16 item 23): not
  662. * offline (the probe state) and holding no running session —
  663. * registration order, the dispatcher's capability-matching
  664. * candidate list. A parked task's closed session reads free
  665. * exactly like the freed daemon it brackets.
  666. */
  667. public async ImmutableLot<Saddle> list_idle_saddles() throws Error {
  668. var busy = yield busy_saddle_ids();
  669. var rows = new Vector<Saddle>();
  670. foreach (var saddle in yield db.query<Saddle>()
  671. .order_by(expr("id"))
  672. .materialise_async()) {
  673. if (saddle.status == SADDLE_OFFLINE
  674. || busy.contains(((int64) saddle.id).to_string())) {
  675. continue;
  676. }
  677. rows.add(saddle);
  678. }
  679. return rows.to_immutable_buffer();
  680. }
  681. /**
  682. * Marks a saddle offline when its control server cannot be
  683. * reached. The task it was RUNNING (the derived running
  684. * session — plan 16 item 23) releases back to 'queued' through
  685. * {@link TaskService.release_task} (the ONE owner; it closes
  686. * the session 'stopped'), with a system comment so the timeline
  687. * shows the interruption; combined with the daemon's
  688. * crash-resume, the next dispatch restores the workspace and
  689. * reports `resumed` on /status. A task that parked mid-flight
  690. * (its session already closed 'parked', the verdict awaiting
  691. * the resume) keeps its park. The offline stamp itself is the
  692. * ONE surviving guarded write — see {@link stamp_offline}.
  693. */
  694. public async void mark_saddle_unreachable(int64 id, TaskService tasks,
  695. string? reason = null) throws Error {
  696. var saddle = yield get_saddle(id);
  697. if (saddle == null) {
  698. return;
  699. }
  700. var session = yield running_session_on_saddle(id);
  701. int64 seen_session = session != null ? ((!)session).id : 0;
  702. int64 held_task = 0;
  703. if (session != null) {
  704. held_task = ((!)session).task_id;
  705. // The requeue flip is guarded: a park (or terminal)
  706. // interleaving this path keeps its state — the parked
  707. // verdict must survive for the resume.
  708. var flip = connection.create_command(
  709. "UPDATE tasks SET status = 'queued' WHERE id = :id"
  710. + " AND status IN ('assigned', 'working')");
  711. flip.with_parameter<Invercargill.Element>("id", elem<int64?>(held_task));
  712. if ((yield flip.execute_non_query_async()) > 0) {
  713. string unreachable_text = reason != null && reason.length > 0
  714. ? @"Saddle became unreachable ($(reason)); task requeued"
  715. : "Saddle became unreachable; task requeued";
  716. yield tasks.add_system_comment(held_task, unreachable_text);
  717. // Closes the held session FIRST — the offline
  718. // stamp's guard refuses to land over a running
  719. // session, and by then there is none.
  720. yield tasks.release_task(held_task, OUTCOME_STOPPED, false);
  721. } else {
  722. // Parked (or finished) mid-flight — the park closed
  723. // its session and the verdict must survive for the
  724. // resume; nothing to release.
  725. held_task = 0;
  726. }
  727. }
  728. yield stamp_offline(id, seen_session);
  729. // An offline saddle is neither sleeping nor wake-awaiting;
  730. // a restarted daemon re-establishes both by asking.
  731. poll_state.clear_ask(id);
  732. yield pulse.bump(0, "saddle unreachable");
  733. }
  734. /**
  735. * The ONE surviving guarded saddle write (plan 16 item 23):
  736. * the OFFLINE probe stamp. The guard is the sessions-truth
  737. * analogue of the old seen-assignment guard — the stamp lands
  738. * unless a running session the probe did NOT see at entry
  739. * appeared meanwhile (an ask claim is proof of life and must
  740. * not be stomped offline). The session observed at entry
  741. * (`seen_session`) never blocks its own stamp.
  742. */
  743. private async void stamp_offline(int64 saddle_id, int64 seen_session) throws Error {
  744. var command = connection.create_command(
  745. "UPDATE saddles SET status = '" + SADDLE_OFFLINE + "' WHERE id = :id"
  746. + " AND NOT EXISTS (SELECT 1 FROM sessions s"
  747. + " JOIN tasks t ON t.id = s.task_id"
  748. + " WHERE s.saddle_id = :id AND s.ended_at IS NULL"
  749. + " AND t.status IN ('assigned', 'working')"
  750. + " AND s.id != :seen)");
  751. command.with_parameter<Invercargill.Element>("id", elem<int64?>(saddle_id));
  752. command.with_parameter<Invercargill.Element>("seen", elem<int64?>(seen_session));
  753. yield command.execute_non_query_async();
  754. }
  755. /**
  756. * Assembles the assignment payload shared by the /start push and
  757. * the `POST /api/saddle/next-task` response — `{task, kind,
  758. * mode, creator, goal?, rules, project_notes, callback,
  759. * llm_config, session_id}` — with the project, links (the
  760. * goal's own links for goal work, else the project's), branch
  761. * fields and recent comments inside the task object, plus any
  762. * block answers not yet handed over (a resumed workspace picks
  763. * its pending answers up from the assignment itself), the
  764. * effective rule set ({@link RuleService.effective_rules}) as
  765. * the top-level `rules` array, the capped `project_notes` of
  766. * the primary and co-developed projects, the note-creation
  767. * `callback` (base URL + the saddle's own bearer token), the
  768. * saddle's `git_identity`, a `capabilities` context (plan 10
  769. * phase 3: `saddle_tags` — the allocated saddle's own tags —
  770. * and `fleet_tags` — every tag name known across the fleet),
  771. * the plan-12 goal context (`goal` for goal work; the
  772. * worker's `ephemeral_branch`) and — for resumed workspaces —
  773. * the `resume_notice`/`branch_advanced` context. `session_id`
  774. * (the web-side run row the daemon reports against) rides
  775. * along so pushed telemetry can key on it. Project checkouts
  776. * are addressed by the `SADDLE_PROJECTS_ROOT` convention
  777. * (`root/task-<task id>/name`) and cloned from each project's
  778. * `git_url`; the LLM endpoint by the {@link LlmEndpoint}
  779. * environment.
  780. */
  781. public async Json.Object build_assignment_payload(Saddle saddle, TaskEntity task,
  782. int64 session_id, ProjectService projects, string? resume_notice = null,
  783. bool? branch_advanced = null,
  784. string? context_update = null) throws Error {
  785. var endpoint = LlmEndpoint.from_environment();
  786. if (endpoint == null) {
  787. throw new Error(Quark.from_string("clanker-saddle"), 24,
  788. "No LLM endpoint configured (set SADDLE_LLM_BASE_URL)");
  789. }
  790. var project = task.project_id > 0
  791. ? yield db.query<Project>()
  792. .where(expr("id == $0", elem<int64?>(task.project_id)))
  793. .first_async()
  794. : null;
  795. // Goal 8 task 4: project_id 0 is a STANDALONE task — no
  796. // project row exists (and none is looked up), the
  797. // co-developed/reference arrays ship EMPTY and the daemon
  798. // renders the standalone prompt section. A task with a
  799. // non-zero project_id whose row is missing is still the
  800. // data-integrity error it always was.
  801. if (project == null && task.project_id > 0) {
  802. throw new Error(Quark.from_string("clanker-saddle"), 12,
  803. @"Task $(task.id) has no project $(task.project_id)");
  804. }
  805. // Plan 12: goal work (the agent's own run and its worker
  806. // tasks) scopes its co-developed/reference set through the
  807. // GOAL's links; standalone tasks through the project's.
  808. // Standalone tasks (project 0) own no links at all — the
  809. // simplest consistent shape: both arrays stay empty.
  810. Goal? goal = null;
  811. if (task.goal_id > 0) {
  812. goal = yield db.query<Goal>()
  813. .where(expr("id == $0", elem<int64?>(task.goal_id)))
  814. .first_async();
  815. }
  816. var co_developed = new Vector<Project>();
  817. var references = new Vector<Project>();
  818. if (goal != null) {
  819. foreach (var link in yield goal_service.list_links(task.goal_id)) {
  820. var linked = yield db.query<Project>()
  821. .where(expr("id == $0", elem<int64?>(link.project_id)))
  822. .first_async();
  823. if (linked == null) {
  824. continue;
  825. }
  826. if (link.relationship == LINK_CO_DEVELOPED) {
  827. co_developed.add((!)linked);
  828. } else if (link.relationship == LINK_REFERENCE) {
  829. references.add((!)linked);
  830. }
  831. }
  832. } else if (project != null) {
  833. foreach (var link in yield projects.list_links(task.project_id)) {
  834. var linked = yield db.query<Project>()
  835. .where(expr("id == $0", elem<int64?>(link.linked_project_id)))
  836. .first_async();
  837. if (linked == null) {
  838. continue;
  839. }
  840. if (link.relationship == LINK_CO_DEVELOPED) {
  841. co_developed.add((!)linked);
  842. } else if (link.relationship == LINK_REFERENCE) {
  843. references.add((!)linked);
  844. }
  845. }
  846. }
  847. var payload = new Json.Builder();
  848. payload.begin_object();
  849. payload.set_member_name("task");
  850. payload.begin_object();
  851. payload.set_member_name("id");
  852. payload.add_int_value(task.id);
  853. payload.set_member_name("title");
  854. payload.add_string_value(task.title);
  855. payload.set_member_name("description");
  856. payload.add_string_value(task.description ?? "");
  857. payload.set_member_name("project_name");
  858. payload.add_string_value(project != null ? ((!)project).name : "standalone");
  859. payload.set_member_name("project_branch");
  860. payload.add_string_value(project != null ? ((!)project).default_branch : "");
  861. payload.set_member_name("task_branch");
  862. payload.add_string_value(task.task_branch ?? "");
  863. payload.set_member_name("branch_from");
  864. payload.add_string_value(task.branch_from
  865. ?? (project != null ? ((!)project).default_branch : ""));
  866. payload.set_member_name("co_developed");
  867. payload.begin_array();
  868. if (project != null) {
  869. payload.add_value(project_node(((!)project), task.id, false, saddle));
  870. }
  871. foreach (var linked in co_developed) {
  872. payload.add_value(project_node(linked, task.id, false, saddle));
  873. }
  874. payload.end_array();
  875. payload.set_member_name("reference");
  876. payload.begin_array();
  877. foreach (var linked in references) {
  878. payload.add_value(project_node(linked, task.id, true, saddle));
  879. }
  880. payload.end_array();
  881. payload.set_member_name("comments");
  882. payload.begin_array();
  883. var comments = yield db.query<TaskComment>()
  884. .where(expr("task_id == $0", elem<int64?>(task.id)))
  885. .order_by(expr("id"))
  886. .materialise_async();
  887. foreach (var comment in comments) {
  888. payload.begin_object();
  889. payload.set_member_name("author");
  890. payload.add_string_value(comment.author);
  891. payload.set_member_name("body");
  892. payload.add_string_value(comment.body);
  893. if (comment.quote != null) {
  894. payload.set_member_name("quote");
  895. payload.add_string_value(((!)comment.quote));
  896. }
  897. payload.end_object();
  898. }
  899. payload.end_array();
  900. payload.set_member_name("answers");
  901. payload.begin_array();
  902. foreach (var pending in yield take_pending_answers_for_task(task.id)) {
  903. payload.begin_object();
  904. payload.set_member_name("block_id");
  905. payload.add_int_value(pending.block_id);
  906. payload.set_member_name("answer");
  907. payload.add_string_value(raw_answer_for_saddle(pending.answer));
  908. if (branch_advanced != null) {
  909. payload.set_member_name("branch_advanced");
  910. payload.add_boolean_value((!)branch_advanced);
  911. }
  912. payload.end_object();
  913. }
  914. payload.end_array();
  915. payload.end_object();
  916. // The plan-12 wire additions: kind ("task"|"goal_run"),
  917. // mode and creator ride the top level beside the task.
  918. payload.set_member_name("kind");
  919. payload.add_string_value(task.kind);
  920. payload.set_member_name("mode");
  921. payload.add_string_value(task.mode);
  922. payload.set_member_name("creator");
  923. payload.add_string_value(task.creator);
  924. // The goal context (goal work only): the goal agent gets
  925. // the full brief + approved plan; a goal worker gets the
  926. // same object as read-only reference (the daemon adds the
  927. // stick-to-mandate shaping) plus its ephemeral branch.
  928. if (goal != null) {
  929. payload.set_member_name("goal");
  930. payload.begin_object();
  931. payload.set_member_name("id");
  932. payload.add_int_value(((!)goal).id);
  933. payload.set_member_name("branch");
  934. payload.add_string_value(((!)goal).branch_name);
  935. payload.set_member_name("base_branch");
  936. payload.add_string_value(((!)goal).base_branch
  937. ?? (project != null ? ((!)project).default_branch : ""));
  938. payload.set_member_name("brief");
  939. payload.add_string_value(((!)goal).brief);
  940. payload.set_member_name("plan");
  941. payload.add_string_value(((!)goal).plan ?? "");
  942. payload.end_object();
  943. if (task.kind == TASK_KIND_TASK) {
  944. payload.set_member_name("ephemeral_branch");
  945. payload.add_string_value(@"task/$(task.goal_id)-$(task.id)");
  946. }
  947. }
  948. // The wire contract's top-level `rules` addition: the
  949. // effective rule set (task ∪ primary ∪ co-developed, deduped,
  950. // stable order — see RuleService.effective_rules). Goal work
  951. // expands co-developed through the goal's links.
  952. payload.set_member_name("rules");
  953. payload.begin_array();
  954. foreach (var rule in yield rules.effective_rules_for(task, projects,
  955. goal != null ? task.goal_id : 0)) {
  956. payload.begin_object();
  957. payload.set_member_name("name");
  958. payload.add_string_value(rule.name);
  959. payload.set_member_name("body");
  960. payload.add_string_value(rule.body);
  961. payload.end_object();
  962. }
  963. payload.end_array();
  964. // The wire contract's `git_identity` addition (plan 9): the
  965. // name/email the daemon commits as, from the saddle row.
  966. // Omitted entirely when unset so the daemon's fallback
  967. // identity applies.
  968. string identity_name = saddle.git_name ?? "";
  969. string identity_email = saddle.git_email ?? "";
  970. if (identity_name.length > 0 || identity_email.length > 0) {
  971. payload.set_member_name("git_identity");
  972. payload.begin_object();
  973. payload.set_member_name("name");
  974. payload.add_string_value(identity_name);
  975. payload.set_member_name("email");
  976. payload.add_string_value(identity_email);
  977. payload.end_object();
  978. }
  979. // The wire contract's `prompt_note` addition (goal 8 task
  980. // 5): the saddle admin's standing message about THIS
  981. // machine, served on EVERY assignment (the daemon renders
  982. // it verbatim under an admin heading). Empty string when
  983. // unset, clamped so a fat note can never bloat prompts.
  984. var prompt_note_text = saddle.prompt_note ?? "";
  985. if (prompt_note_text.length > PROMPT_NOTE_MAX_CHARS) {
  986. prompt_note_text = prompt_note_text.substring(0, (int) PROMPT_NOTE_MAX_CHARS);
  987. }
  988. payload.set_member_name("prompt_note");
  989. payload.add_string_value(prompt_note_text);
  990. // The wire contract's `capabilities` context (plan 10
  991. // phase 3): the allocated saddle's own tags plus every tag
  992. // name known across the fleet (the operator-managed
  993. // catalogue) — the daemon renders both into its system
  994. // prompt and validates `task create --capability`.
  995. payload.set_member_name("capabilities");
  996. payload.begin_object();
  997. payload.set_member_name("saddle_tags");
  998. payload.begin_array();
  999. foreach (var carried in yield capability_service.list_for_saddle(saddle.id)) {
  1000. payload.add_string_value(carried.name);
  1001. }
  1002. payload.end_array();
  1003. payload.set_member_name("fleet_tags");
  1004. payload.begin_array();
  1005. foreach (var known in yield capability_service.list_capabilities()) {
  1006. payload.add_string_value(known.name);
  1007. }
  1008. payload.end_array();
  1009. payload.end_object();
  1010. // The wire contract's `resume_notice` addition: on a
  1011. // workspace-adopting start (a suspended parent's resume), the
  1012. // daemon injects it as the next user turn instead of the
  1013. // opening prompt.
  1014. if (resume_notice != null && resume_notice.length > 0) {
  1015. payload.set_member_name("resume_notice");
  1016. payload.add_string_value(resume_notice);
  1017. }
  1018. // Goal 28 task F: `pending_question` — the task's newest
  1019. // block is UNANSWERED (a crashed parked task resumed
  1020. // through the allocation). The daemon appends the
  1021. // still-open notice to the restored conversation's opening
  1022. // turn so the agent does not barrel past its own question.
  1023. var newest_open = yield task_service.newest_block(task.id);
  1024. if (newest_open != null && ((!)newest_open).answer == null) {
  1025. payload.set_member_name("pending_question");
  1026. payload.add_boolean_value(true);
  1027. }
  1028. // The wire's `branch_advanced` (resume pushes): whether the
  1029. // shared branch moved while this task was parked — false or
  1030. // absent means the daemon omits the pull instruction entirely.
  1031. if (branch_advanced != null) {
  1032. payload.set_member_name("branch_advanced");
  1033. payload.add_boolean_value((!)branch_advanced);
  1034. }
  1035. // The wire's `context_update` (resume pushes): a short
  1036. // markdown delta of the instruction context that changed
  1037. // while the workspace was parked — the goal's plan
  1038. // approved since the run's last round, rules created,
  1039. // edited or attached, or new project notes. The daemon
  1040. // appends it as a TURN (prompt-cache friendly) instead of
  1041. // refreshing the system prompt; absent when nothing
  1042. // changed.
  1043. if (context_update != null && context_update.length > 0) {
  1044. payload.set_member_name("context_update");
  1045. payload.add_string_value(context_update);
  1046. }
  1047. // The wire contract's `project_notes` addition: notes of the
  1048. // primary and co-developed projects, oldest first, capped at
  1049. // 30 entries / 20 000 chars (head kept).
  1050. var note_project_names = new Dictionary<string, string>();
  1051. var project_notes_rows = new Vector<ClankerSaddle.Data.ProjectNote>();
  1052. if (project != null) {
  1053. note_project_names.set(((!)project).id.to_string(), ((!)project).name);
  1054. foreach (var note in yield projects.list_notes(((!)project).id)) {
  1055. project_notes_rows.add(note);
  1056. }
  1057. }
  1058. foreach (var linked in co_developed) {
  1059. note_project_names.set(linked.id.to_string(), linked.name);
  1060. foreach (var note in yield projects.list_notes(linked.id)) {
  1061. project_notes_rows.add(note);
  1062. }
  1063. }
  1064. var project_notes = project_notes_rows.sort((a, b) => (int)(a.id - b.id));
  1065. payload.set_member_name("project_notes");
  1066. payload.begin_array();
  1067. int64 note_entries = 0;
  1068. int64 note_chars = 0;
  1069. foreach (var note in project_notes) {
  1070. if (note_entries >= PROJECT_NOTES_MAX_ENTRIES
  1071. || note_chars >= PROJECT_NOTES_MAX_CHARS) {
  1072. break;
  1073. }
  1074. string note_body = note.body;
  1075. if (note_chars + note_body.length > PROJECT_NOTES_MAX_CHARS) {
  1076. note_body = note_body.substring(0,
  1077. (int)(PROJECT_NOTES_MAX_CHARS - note_chars));
  1078. }
  1079. payload.begin_object();
  1080. payload.set_member_name("project");
  1081. payload.add_string_value(note_project_names.get_or_default(note.project_id.to_string()) ?? "");
  1082. payload.set_member_name("body");
  1083. payload.add_string_value(note_body);
  1084. payload.end_object();
  1085. note_entries++;
  1086. note_chars += note_body.length;
  1087. }
  1088. payload.end_array();
  1089. // The wire contract's `understanding` addition (goal 24):
  1090. // the PRIMARY project's current understanding document —
  1091. // a markdown summary of the project's structure and its
  1092. // unusual corners, stamped with the commit it was written
  1093. // against. Present but empty when the project has none;
  1094. // head-clamped at UNDERSTANDING_MAX_CHARS.
  1095. string understanding_body = "";
  1096. if (project != null) {
  1097. var current_understanding = yield understandings
  1098. .current_understanding(((!)project).id);
  1099. if (current_understanding != null) {
  1100. understanding_body = ((!)current_understanding).body;
  1101. if (understanding_body.length > UNDERSTANDING_MAX_CHARS) {
  1102. understanding_body = understanding_body.substring(0,
  1103. (int) UNDERSTANDING_MAX_CHARS);
  1104. }
  1105. }
  1106. }
  1107. payload.set_member_name("understanding");
  1108. payload.add_string_value(understanding_body);
  1109. // The wire contract's `discoveries` addition (goal 24):
  1110. // the fleet's NEWEST discoveries — non-project facts
  1111. // (compiler quirks, library behaviours, tool flags,
  1112. // protocol gotchas) earlier agents recorded — newest
  1113. // first, capped at 10 entries / 8 000 chars (head kept),
  1114. // so a fresh agent inherits the knowledge base before it
  1115. // ever searches. An empty array when there are none.
  1116. payload.set_member_name("discoveries");
  1117. payload.begin_array();
  1118. int64 discovery_entries = 0;
  1119. int64 discovery_chars = 0;
  1120. foreach (var discovery in yield discoveries.list_newest_first(
  1121. (int) DISCOVERIES_MAX_ENTRIES)) {
  1122. string discovery_body = discovery.body;
  1123. if (discovery_chars + discovery_body.length
  1124. > DISCOVERIES_MAX_CHARS) {
  1125. discovery_body = discovery_body.substring(0,
  1126. (int)(DISCOVERIES_MAX_CHARS - discovery_chars));
  1127. }
  1128. payload.begin_object();
  1129. payload.set_member_name("body");
  1130. payload.add_string_value(discovery_body);
  1131. payload.end_object();
  1132. discovery_entries++;
  1133. discovery_chars += discovery_body.length;
  1134. if (discovery_entries >= DISCOVERIES_MAX_ENTRIES
  1135. || discovery_chars >= DISCOVERIES_MAX_CHARS) {
  1136. break;
  1137. }
  1138. }
  1139. payload.end_array();
  1140. // The wire contract's `ethos` addition (goal 27, canon
  1141. // centralisation): the ONE web-side ethos text rides every
  1142. // assignment so the daemon's system prompt carries it
  1143. // verbatim without a second fetch (its compiled-in copy is
  1144. // the compatibility fallback for older webs).
  1145. payload.set_member_name("ethos");
  1146. payload.add_string_value(ClankerSaddle.Services.Ethos.TEXT);
  1147. // The wire contract's `callback` addition: this app's
  1148. // externally reachable base URL plus the bearer token the
  1149. // daemon presents when posting back — its own per-saddle
  1150. // credential (plan 10), which every `/api/saddle/*` call
  1151. // carries.
  1152. payload.set_member_name("callback");
  1153. payload.begin_object();
  1154. payload.set_member_name("base_url");
  1155. payload.add_string_value(callback_base_url());
  1156. payload.set_member_name("token");
  1157. payload.add_string_value(saddle.token ?? "");
  1158. payload.end_object();
  1159. payload.set_member_name("llm_config");
  1160. payload.begin_object();
  1161. payload.set_member_name("base_url");
  1162. payload.add_string_value(((!)endpoint).base_url);
  1163. payload.set_member_name("api_key");
  1164. payload.add_string_value(((!)endpoint).api_key);
  1165. payload.set_member_name("model");
  1166. payload.add_string_value(((!)endpoint).model);
  1167. payload.end_object();
  1168. // The web-side run row the daemon's pushed telemetry keys on
  1169. // (plan-10 phase-1 wire addition: it rides every assignment
  1170. // so /start consumers can ignore it harmlessly).
  1171. payload.set_member_name("session_id");
  1172. payload.add_int_value(session_id);
  1173. payload.end_object();
  1174. var root = payload.get_root().copy();
  1175. return root.get_object();
  1176. }
  1177. /**
  1178. * Probes a saddle's `GET /status` and records what a probe can
  1179. * tell: the health snapshot, the daemon-reported ssh key and
  1180. * the goal-branch push flags. Busy/idle need NO write anymore —
  1181. * they are derived from the session records (plan 16 item 23).
  1182. * The row's only remaining write is the OFFLINE probe state: a
  1183. * successful probe is proof of life and clears it; a failed
  1184. * probe stamps it through the guarded write (an ask claim that
  1185. * landed while the probe was in flight is a NEWER proof of
  1186. * life and must not be stomped).
  1187. */
  1188. public async void refresh_saddle_status(int64 id) throws Error {
  1189. var saddle = yield get_saddle(id);
  1190. if (saddle == null) {
  1191. throw new Error(Quark.from_string("clanker-saddle"), 17,
  1192. @"No saddle exists with id $(id)");
  1193. }
  1194. var session = yield running_session_on_saddle(id);
  1195. int64 seen_session = session != null ? ((!)session).id : 0;
  1196. try {
  1197. var status = yield request_saddle_json(((!)saddle).host, "GET", "/status", null);
  1198. yield record_status_probe(((!)saddle), status);
  1199. var clear = connection.create_command(
  1200. "UPDATE saddles SET status = '" + SADDLE_IDLE
  1201. + "' WHERE id = :id AND status = '" + SADDLE_OFFLINE + "'");
  1202. clear.with_parameter<Invercargill.Element>("id", elem<int64?>(id));
  1203. yield clear.execute_non_query_async();
  1204. } catch (Error e) {
  1205. yield stamp_offline(id, seen_session);
  1206. }
  1207. yield pulse.bump(0, "saddle status refreshed");
  1208. }
  1209. /**
  1210. * Answered blocks of `task_id` not yet handed over anywhere;
  1211. * marks them delivered — a resumed assignment carries its
  1212. * pending answers in the payload ({@link push_task_to_saddle}).
  1213. */
  1214. public async ImmutableLot<PendingAnswer> take_pending_answers_for_task(int64 task_id) throws Error {
  1215. var blocks = yield db.query<TaskBlock>()
  1216. .where(expr("task_id == $0", elem<int64?>(task_id)))
  1217. .materialise_async();
  1218. var rows = new Vector<PendingAnswer>();
  1219. foreach (var block in blocks) {
  1220. if (block.answer == null) {
  1221. continue;
  1222. }
  1223. if (!poll_state.mark_delivered(block.id)) {
  1224. continue;
  1225. }
  1226. rows.add(new PendingAnswer() {
  1227. block_id = block.id,
  1228. answer = raw_answer_for_saddle(((!)block).answer ?? "")
  1229. });
  1230. }
  1231. return rows.to_immutable_buffer();
  1232. }
  1233. /**
  1234. * Pushes the task's undelivered human comments to the RUNNING
  1235. * daemon's `POST /task-update {kind: "comment"}` endpoint so
  1236. * they land in the live conversation. Best effort: a daemon
  1237. * without the route (HTTP 404/405) keeps the comments pending,
  1238. * and other failures are logged and retried on the next
  1239. * dispatcher tick.
  1240. */
  1241. public async void deliver_pending_comments(int64 saddle_id, int64 task_id) throws Error {
  1242. var saddle = yield get_saddle(saddle_id);
  1243. if (saddle == null) {
  1244. return;
  1245. }
  1246. var comments = yield db.query<TaskComment>()
  1247. .where(expr("task_id == $0", elem<int64?>(task_id)))
  1248. .order_by(expr("id"))
  1249. .materialise_async();
  1250. foreach (var comment in comments) {
  1251. if (!comment.author.has_prefix("user:")
  1252. || poll_state.is_comment_delivered(comment.id)) {
  1253. continue;
  1254. }
  1255. var payload = new Json.Builder();
  1256. payload.begin_object();
  1257. payload.set_member_name("kind");
  1258. payload.add_string_value("comment");
  1259. payload.set_member_name("text");
  1260. payload.add_string_value(comment.body);
  1261. payload.set_member_name("quote");
  1262. payload.add_string_value(comment.quote ?? "");
  1263. payload.end_object();
  1264. Json.Node? root = null;
  1265. int status_code = yield request_saddle(((!)saddle).host, "POST",
  1266. "/task-update", (owned) payload, out root);
  1267. if (status_code == 200 || status_code == 204) {
  1268. poll_state.mark_comment_delivered(comment.id);
  1269. } else if (status_code != 404 && status_code != 405) {
  1270. warning(@"[ClankerSaddle] Comment task-update to saddle $(saddle_id) returned HTTP $(status_code)");
  1271. }
  1272. }
  1273. }
  1274. /**
  1275. * Marks the task's human comments as delivered — they were handed
  1276. * over inside an assignment payload, so neither the poll endpoint
  1277. * nor `/notify` should repeat them.
  1278. */
  1279. public async void mark_task_comments_delivered(int64 task_id) throws Error {
  1280. var comments = yield db.query<TaskComment>()
  1281. .where(expr("task_id == $0", elem<int64?>(task_id)))
  1282. .materialise_async();
  1283. foreach (var comment in comments) {
  1284. if (comment.author.has_prefix("user:")) {
  1285. poll_state.mark_comment_delivered(comment.id);
  1286. }
  1287. }
  1288. }
  1289. /**
  1290. * Goal 28 task F (busy-answer delivery): the hub drainer. An
  1291. * answer whose task holds a RUNNING session reaches that
  1292. * session THROUGH the daemon's task-update channel (the same
  1293. * injection path human comments ride) — `POST /task-update
  1294. * {kind: "answer", text, question}` — because no affinity
  1295. * resume can deliver it: the busy saddle cannot ask, and the
  1296. * running round's conversation predates the block.
  1297. *
  1298. * Delivery-once: the poll-state answers ledger (the poll
  1299. * endpoint's memory) is the guard, so the resume notice never
  1300. * repeats a verdict the running round already saw. The
  1301. * session's `answers_injected` count increments per verdict —
  1302. * a later park's resume notice summarises them instead of
  1303. * repeating the texts. Failures (transport, HTTP) re-offer
  1304. * the delivery for the next tick: an answer must never be
  1305. * dropped, and the daemon tolerates a duplicate as one extra
  1306. * user turn.
  1307. */
  1308. public async void deliver_pending_answers(TaskService tasks,
  1309. AnswerDeliveryHub hub) throws Error {
  1310. foreach (var delivery in hub.drain()) {
  1311. var session = yield tasks.open_session_for_task(delivery.task_id);
  1312. if (session == null) {
  1313. // No running round: the resume path owns delivery
  1314. // (the verdict rides the next resuming
  1315. // assignment's notice). Drop the queue entry —
  1316. // the answer is safe in its block row.
  1317. continue;
  1318. }
  1319. var saddle = yield get_saddle(((!)session).saddle_id);
  1320. if (saddle == null) {
  1321. hub.offer(delivery);
  1322. continue;
  1323. }
  1324. var payload = new Json.Builder();
  1325. payload.begin_object();
  1326. payload.set_member_name("kind");
  1327. payload.add_string_value("answer");
  1328. payload.set_member_name("text");
  1329. payload.add_string_value(delivery.notice);
  1330. payload.set_member_name("task_id");
  1331. payload.add_int_value(delivery.task_id);
  1332. payload.end_object();
  1333. Json.Node? root = null;
  1334. bool handed = false;
  1335. try {
  1336. int status_code = yield request_saddle(((!)saddle).host, "POST",
  1337. "/task-update", (owned) payload, out root);
  1338. // 404/405: a daemon without the answer kind keeps
  1339. // the delivery pending — re-offer for the next
  1340. // tick (an upgrade may land under us). Everything
  1341. // else that fails re-offers too: never drop an
  1342. // answer.
  1343. handed = status_code == 200 || status_code == 204;
  1344. if (!handed) {
  1345. warning("[ClankerSaddle] Answer task-update to saddle $(((!)saddle).id) returned HTTP %d",
  1346. status_code);
  1347. hub.offer(delivery);
  1348. continue;
  1349. }
  1350. } catch (Error e) {
  1351. warning("[ClankerSaddle] Answer task-update to saddle $(((!)saddle).id) failed: %s",
  1352. e.message);
  1353. hub.offer(delivery);
  1354. continue;
  1355. }
  1356. poll_state.mark_delivered(delivery.block_id);
  1357. ((!)session).answers_injected += 1;
  1358. yield db.update_async<ClankerSaddle.Data.Session>((!)session);
  1359. yield task_service.add_system_comment(delivery.task_id,
  1360. @"verdict delivered into the running session (block #$(delivery.block_id))");
  1361. yield pulse.bump(delivery.task_id, "answer delivered mid-session");
  1362. }
  1363. }
  1364. /**
  1365. * The goal id when `task_id` is a goal agent's run task (kind
  1366. * 'goal_run'), else 0 — the discriminator for the goal-context
  1367. * verb handling above.
  1368. */
  1369. /**
  1370. * True when the goal owns at least one WORKER task that could
  1371. * still fire a terminal transition (the goal-run park's wake
  1372. * source): every status but done/failed counts — a queued,
  1373. * assigned, working, blocked, review-requested or suspended
  1374. * row will end its life in a transition that wakes the run.
  1375. * DRAFT rows do NOT count: a draft nobody ever enqueued
  1376. * produces no transition, ever (the goal-2 stall class).
  1377. */
  1378. private async bool goal_has_wakeable_task(int64 goal_id) throws Error {
  1379. foreach (var task in yield db.query<TaskEntity>()
  1380. .where(expr("goal_id == $0 && kind == $1",
  1381. elem<int64?>(goal_id), elem(TASK_KIND_TASK)))
  1382. .materialise_async()) {
  1383. if (task.status != STATUS_DONE && task.status != STATUS_FAILED
  1384. && task.status != STATUS_DRAFT) {
  1385. return true;
  1386. }
  1387. }
  1388. return false;
  1389. }
  1390. /**
  1391. * The parking guard's refusal: the park mechanics run (status
  1392. * 'suspended', the round's session closed 'parked',
  1393. * suspended_at stamped) and the ONE-SHOT
  1394. * {@link TaskEntity.pending_resume_notice} is set — which
  1395. * makes the run resume-offerable IMMEDIATELY (no terminal
  1396. * event or coalescing window). The daemon asks again the
  1397. * moment its round ends; that ask claims the run back and the
  1398. * refusal text rides the assignment as its resume_notice, so
  1399. * the agent is told to finish (submit its report) or cancel
  1400. * instead of sleeping forever. Keeping the session open
  1401. * instead would dead-end: an open session on an assigned task
  1402. * marks the saddle busy, so the next ask would 204 and the
  1403. * daemon would sleep with the workspace wedged.
  1404. */
  1405. private async void refuse_park(int64 task_id, string notice) throws Error {
  1406. yield task_service.set_parked(task_id);
  1407. var task = yield db.query<TaskEntity>()
  1408. .where(expr("id == $0", elem<int64?>(task_id)))
  1409. .first_async();
  1410. if (task == null || ((!)task).pending_resume_notice != null) {
  1411. // The first refusal stands (an already-pending notice
  1412. // keeps its original wording).
  1413. return;
  1414. }
  1415. ((!)task).pending_resume_notice = notice;
  1416. yield db.update_async<TaskEntity>((!)task);
  1417. var refused = new Json.Object();
  1418. refused.set_string_member("text", notice);
  1419. yield event_service.append(EVENT_STATE_CHANGE, EVENT_ACTOR_SYSTEM,
  1420. EVENT_SCOPE_TASK, task_id, refused);
  1421. yield task_service.add_system_comment(task_id, notice);
  1422. yield pulse.bump(task_id, "park refused (nothing to wait on)");
  1423. }
  1424. private async int64 goal_id_of_run(int64 task_id) throws Error {
  1425. var task = yield db.query<TaskEntity>()
  1426. .where(expr("id == $0", elem<int64?>(task_id)))
  1427. .first_async();
  1428. return task != null && ((!)task).kind == TASK_KIND_GOAL_RUN
  1429. ? ((!)task).goal_id : 0;
  1430. }
  1431. /**
  1432. * Raises the daemon's pushed blocking prompt as a `task_blocks`
  1433. * row exactly once: a not-yet-answered row, or an answered row
  1434. * whose answer has not been delivered to the daemon yet, means the
  1435. * prompt is already represented. The results push's `block`
  1436. * object carries the question plus the block type, options and
  1437. * markdown body (the `/status` field names — `block_question`
  1438. * &c — are accepted as a fallback). `saddle block rfc` carries
  1439. * no question text, so rfc prompts surface under a stable
  1440. * fallback title.
  1441. */
  1442. private async void ensure_block(TaskService tasks, int64 task_id, Json.Object block) throws Error {
  1443. string question = block_question_from_payload(block);
  1444. if (question.length == 0) {
  1445. return;
  1446. }
  1447. string incoming_type = block_string_or(block, "type", "block_type");
  1448. foreach (var existing in yield tasks.list_blocks(task_id)) {
  1449. if (existing.answer == null || !poll_state.is_delivered(existing.id)) {
  1450. // A pending review-family block of EITHER kind
  1451. // (review/completion) swallows the other's push: the
  1452. // daemon's done park and the web's completion raise
  1453. // are the SAME review raised from two paths — one
  1454. // row, whichever landed first.
  1455. if ((incoming_type == BLOCK_REVIEW || incoming_type == BLOCK_COMPLETION)
  1456. && (existing.block_type == BLOCK_REVIEW
  1457. || existing.block_type == BLOCK_COMPLETION)) {
  1458. return;
  1459. }
  1460. }
  1461. if (existing.question != question) {
  1462. continue;
  1463. }
  1464. if (existing.answer == null || !poll_state.is_delivered(existing.id)) {
  1465. return;
  1466. }
  1467. }
  1468. yield tasks.add_block(task_id, question,
  1469. block_string_or(block, "type", "block_type"),
  1470. EntryJson.string_array_member(block, block.has_member("options") ? "options" : "block_options"),
  1471. EntryJson.optional_string_member(block, block.has_member("body") ? "body" : "block_body"));
  1472. }
  1473. /** The first non-empty of two string members (empty when both are). */
  1474. private static string block_string_or(Json.Object obj, string primary, string fallback) {
  1475. string value = EntryJson.string_member(obj, primary);
  1476. return value.length > 0 ? value : EntryJson.string_member(obj, fallback);
  1477. }
  1478. /**
  1479. * The question a daemon block is recognised by: `question`
  1480. * (or the `/status`-style `block_question`) verbatim, or the
  1481. * rfc fallback when the daemon raised an RFC (the
  1482. * `saddle block rfc` shape carries its document as the body and
  1483. * no question operand). Both the block-raising and answer-
  1484. * delivery paths key blocks by this string.
  1485. */
  1486. private static string block_question_from_payload(Json.Object block) {
  1487. string question = block_string_or(block, "question", "block_question");
  1488. if (question.length > 0) {
  1489. return question;
  1490. }
  1491. return block_string_or(block, "type", "block_type") == BLOCK_RFC
  1492. ? "Request for comments" : "";
  1493. }
  1494. /**
  1495. * Applies a pushed state transition (`POST /api/saddle/results`,
  1496. * plan 10 phase 2) — the same task/saddle bookkeeping the
  1497. * deleted /status poll switch ran, driven by the daemon's push:
  1498. * `done`/`failed` complete or fail the task with the pushed
  1499. * summary (closing the session through the release owner) —
  1500. * except on a goal-run session, where they submit the
  1501. * completion report or fail the goal (plan 12); `blocked`
  1502. * raises the block payload as an answerable block on the
  1503. * goal run's OWN task row (plan 15 C1: the goal agent's
  1504. * blocks ride the run task — no mirrored second row exists)
  1505. * and parks the round (the raise closes its session 'parked',
  1506. * D4); `suspended`
  1507. * parks the same way — for a goal agent the resume is driven
  1508. * by the goal's task-terminal events; and `working` flips an
  1509. * ASSIGNED task to working (a parked task keeps its park —
  1510. * the verdict arrives as a resuming allocation, never as a
  1511. * state push; plan 16 D4). No saddle-row write remains
  1512. * (plan 16 item 23): busy/idle derive from the sessions.
  1513. *
  1514. * Plan 13 B2: a push against a task that is already terminal —
  1515. * or a goal-context push against a finished goal — is an
  1516. * outbox redelivery and is consumed silently, whatever state
  1517. * it carries; terminal transitions and their narrative fire
  1518. * once, ever.
  1519. */
  1520. public async void apply_result_state(ClankerSaddle.Data.Session session,
  1521. string state, Json.Object? block, string? summary, TaskService tasks) throws Error {
  1522. int64 task_id = session.task_id;
  1523. int64 goal_id = yield goal_id_of_run(task_id);
  1524. bool terminal = yield task_is_terminal(task_id);
  1525. if (!terminal && goal_id > 0) {
  1526. terminal = yield goal_is_terminal(goal_id);
  1527. }
  1528. if (terminal) {
  1529. return;
  1530. }
  1531. switch (state) {
  1532. case "done":
  1533. if (goal_id > 0) {
  1534. // Goal context: `saddle done` submits the
  1535. // completion report and enters completion
  1536. // review — the run parks on the mirrored
  1537. // block (submit_completion's raise already
  1538. // closed the session 'parked'), it does not
  1539. // complete.
  1540. yield goal_service.submit_completion(goal_id,
  1541. summary != null && summary.length > 0 ? summary
  1542. : "The goal agent submitted its completion report");
  1543. break;
  1544. }
  1545. yield tasks.complete_task(task_id, summary, OUTCOME_DONE);
  1546. break;
  1547. case "failed":
  1548. string explanation = summary ?? "";
  1549. if (goal_id > 0) {
  1550. yield goal_service.fail_goal(goal_id,
  1551. explanation.length > 0 ? explanation : null);
  1552. break;
  1553. }
  1554. yield tasks.fail_task(task_id, explanation.length > 0 ? explanation : null);
  1555. break;
  1556. case "blocked":
  1557. if (goal_id > 0) {
  1558. // Goal context: the block lands on the run
  1559. // task itself (plan 15 C1) — a plan payload
  1560. // enters plan review, anything else raises a
  1561. // goal question/RFC; the run parks on that
  1562. // very row for the verdict resume. A BLOCKED
  1563. // push WITHOUT a
  1564. // block payload (the park-unwinding round-end
  1565. // push, its trailing entries) raises nothing —
  1566. // the payload-carrying transition already did,
  1567. // and a synthesised fallback row would dupe it.
  1568. if (block == null) {
  1569. yield tasks.set_status(task_id, STATUS_BLOCKED);
  1570. yield tasks.park_open_sessions(task_id);
  1571. break;
  1572. }
  1573. string question = block_question_from_payload((!)block);
  1574. if (question.length == 0) {
  1575. question = "The goal agent raised a question";
  1576. }
  1577. if (block_string_or((!)block, "type", "block_type") == BLOCK_PLAN) {
  1578. yield goal_service.submit_plan(goal_id,
  1579. EntryJson.optional_string_member((!)block, "body") ?? question);
  1580. } else if (block_string_or((!)block, "type", "block_type") == BLOCK_REVIEW) {
  1581. // The daemon's submit_report done-park: a
  1582. // review-kind block on the goal run IS the
  1583. // completion submission (submit_completion
  1584. // adopts the raised row — one row, whichever
  1585. // path landed first — and drives the
  1586. // completion_review transition).
  1587. yield goal_service.submit_completion(goal_id,
  1588. EntryJson.optional_string_member((!)block, "body") ?? question);
  1589. } else {
  1590. yield goal_service.raise_agent_block(goal_id, question,
  1591. block_string_or((!)block, "type", "block_type"),
  1592. EntryJson.string_array_member((!)block,
  1593. (!)block.has_member("options") ? "options" : "block_options"),
  1594. EntryJson.optional_string_member((!)block, "body"));
  1595. }
  1596. yield tasks.set_status(task_id, STATUS_BLOCKED);
  1597. break;
  1598. }
  1599. yield ensure_block(tasks, task_id, block != null ? (!)block : new Json.Object());
  1600. {
  1601. // A review-kind block parks as review_requested
  1602. // (the banner state); every other kind as
  1603. // blocked.
  1604. bool review = block != null
  1605. && block_string_or((!)block, "type", "block_type") == BLOCK_REVIEW;
  1606. yield tasks.set_status(task_id,
  1607. review ? STATUS_REVIEW_REQUESTED : STATUS_BLOCKED);
  1608. }
  1609. // The daemon parks the task's workspace while
  1610. // blocked, freeing the saddle for other work
  1611. // (plan 16 item 23: the raise already closed the
  1612. // round's session 'parked' and the task's blocked
  1613. // status IS the pause — no saddle-row write
  1614. // remains; busy/idle derive from the sessions).
  1615. break;
  1616. case "suspended":
  1617. // `saddle suspend`: the same parking as a block, but
  1618. // the web drives resumption — a wake of the affine
  1619. // saddle once a goal event lands, the resume
  1620. // assignment carrying the coalesced report.
  1621. // (set_parked closed the round's session 'parked';
  1622. // the freed saddle is the derived view's reading.)
  1623. //
  1624. // Goal 27 refinements — the PARKING GUARD: a goal
  1625. // run parks so its OWNED tasks' terminal
  1626. // transitions can wake it. When the goal owns no
  1627. // task whose terminal transition could ever fire
  1628. // (no non-terminal worker — only done/failed rows
  1629. // and drafts nobody enqueued), the park would be a
  1630. // dead sleep: nobody but a human force-wake could
  1631. // end it. Refuse it instead — the park mechanics
  1632. // run but the one-shot refusal notice makes the
  1633. // run resume-offerable IMMEDIATELY, so the
  1634. // daemon's next ask hands it "Parking refused — …"
  1635. // instead of a sleep nothing will ever wake. A
  1636. // live task anywhere under the goal keeps the
  1637. // normal park.
  1638. if (goal_id > 0 && !(yield goal_has_wakeable_task(goal_id))) {
  1639. yield refuse_park(task_id,
  1640. "Parking refused: the goal owns no live tasks to wait on —"
  1641. + " finish the run (submit your report) or cancel.");
  1642. break;
  1643. }
  1644. // A PLAIN task parking is ALLOWED (no guard): it
  1645. // has no wake source either way, but its park is a
  1646. // human-facing pause by design — the task page's
  1647. // force-wake is the resume path a goal run lacks.
  1648. yield tasks.set_parked(task_id);
  1649. break;
  1650. case "working":
  1651. // A working push only advances a task that is
  1652. // actually running — assigned (the claim's flip)
  1653. // or a legacy never-allocated queued row. A
  1654. // PARKED task (blocked / review / suspended, its
  1655. // session closed at the raise — plan 16 D4) keeps
  1656. // its park: the verdict arrives as a resuming
  1657. // allocation, never as a state push, so a
  1658. // park-unwinding or late round push can never
  1659. // resurrect the workspace as "working" out from
  1660. // under its review banner.
  1661. var advance = connection.create_command(
  1662. "UPDATE tasks SET status = 'working' WHERE id = :id"
  1663. + " AND status IN ('assigned', 'queued')"
  1664. + " AND status != 'working'");
  1665. advance.with_parameter<Invercargill.Element>("id",
  1666. elem<int64?>(task_id));
  1667. bool flipped = (yield advance.execute_non_query_async()) > 0;
  1668. var current = yield db.query<TaskEntity>()
  1669. .where(expr("id == $0", elem<int64?>(task_id)))
  1670. .first_async();
  1671. if (flipped) {
  1672. var moved = new Json.Object();
  1673. moved.set_string_member("from", STATUS_ASSIGNED);
  1674. moved.set_string_member("to", STATUS_WORKING);
  1675. moved.set_string_member("text", "Status → working");
  1676. yield event_service.append(EVENT_STATE_CHANGE,
  1677. EVENT_ACTOR_SYSTEM, EVENT_SCOPE_TASK, task_id, moved);
  1678. }
  1679. // started_at (and the first-start push) only stamp
  1680. // for work that never allocated; the guarded
  1681. // advance above means the re-read task is working
  1682. // exactly when the flip landed or it already ran.
  1683. if (current != null && ((!)current).status == STATUS_WORKING
  1684. && ((!)current).started_at == null) {
  1685. ((!)current).started_at = new DateTime.now_local();
  1686. yield db.update_async<TaskEntity>((!)current);
  1687. // The first-start push lives at the allocation
  1688. // (where started_at is stamped for dispatched
  1689. // tasks); this path only covers legacy status
  1690. // pushes for never-allocated work.
  1691. push.notify_opted_in(task_id, PushService.lifecycle_notification(
  1692. ((!)current).title, task_id, "started", null));
  1693. }
  1694. break;
  1695. default:
  1696. break;
  1697. }
  1698. }
  1699. /** True when the task's status is done or failed. */
  1700. private async bool task_is_terminal(int64 task_id) throws Error {
  1701. var task = yield db.query<TaskEntity>()
  1702. .where(expr("id == $0", elem<int64?>(task_id)))
  1703. .first_async();
  1704. return task != null && (((!)task).status == STATUS_DONE
  1705. || ((!)task).status == STATUS_FAILED);
  1706. }
  1707. /** True when the goal's status is complete, failed or cancelled. */
  1708. private async bool goal_is_terminal(int64 goal_id) throws Error {
  1709. var goal = yield db.query<Goal>()
  1710. .where(expr("id == $0", elem<int64?>(goal_id)))
  1711. .first_async();
  1712. return goal != null && (((!)goal).status == GOAL_COMPLETE
  1713. || ((!)goal).status == GOAL_FAILED || ((!)goal).status == GOAL_CANCELLED);
  1714. }
  1715. /**
  1716. * The running-liveness probe (plan 10 phase 2): a busy saddle's
  1717. * session has seen neither chunks nor results for the staleness
  1718. * window, so ONE exceptional `GET /status` decides its fate —
  1719. * an answer means the daemon is alive (the health snapshot is
  1720. * refreshed and the task keeps running; the silence episode
  1721. * gets no second probe), a failure requeues the task with an agent
  1722. * comment and marks the saddle offline (the normal unreachable
  1723. * bookkeeping). The probe flag was taken by the caller; the
  1724. * next push resets it.
  1725. */
  1726. public async void probe_stale_saddle(Saddle saddle, TaskService tasks,
  1727. int64 running_task_id) throws Error {
  1728. try {
  1729. var status = yield request_saddle_json(saddle.host, "GET", "/status", null);
  1730. yield record_status_probe(saddle, status);
  1731. // An answer proves the DAEMON lives - not that it is
  1732. // running THIS task. A daemon that crashed and
  1733. // restarted answers from its idle loop (task_id null,
  1734. // or another task it won since): the stale session is
  1735. // a crash orphan, and its task requeues through the
  1736. // same guarded bookkeeping as the unreachable path
  1737. // (the comment names the restart). The saddle itself
  1738. // stays as-is - it is demonstrably alive and asking
  1739. // for work; stamping it offline would only block the
  1740. // requeued task's next claim until another ask.
  1741. var answered_task = status.has_member("task_id")
  1742. && status.get_member("task_id").get_node_type() == Json.NodeType.VALUE
  1743. && status.get_member("task_id").get_value_type() == typeof(int64)
  1744. ? status.get_int_member("task_id") : 0;
  1745. if (answered_task != running_task_id) {
  1746. var flip = connection.create_command(
  1747. "UPDATE tasks SET status = 'queued' WHERE id = :id"
  1748. + " AND status IN ('assigned', 'working')");
  1749. flip.with_parameter<Invercargill.Element>("id",
  1750. elem<int64?>(running_task_id));
  1751. if ((yield flip.execute_non_query_async()) > 0) {
  1752. yield tasks.add_system_comment(running_task_id,
  1753. @"Saddle '$(saddle.name)' restarted mid-run (its status probe answered without the task); task requeued");
  1754. yield tasks.release_task(running_task_id, OUTCOME_STOPPED, false);
  1755. }
  1756. yield pulse.bump(0, "crash orphan requeued at probe");
  1757. print(@"[ClankerSaddle] Stale saddle '$(saddle.name)' answered its probe without task $(running_task_id); requeued\n");
  1758. return;
  1759. }
  1760. print(@"[ClankerSaddle] Stale saddle '$(saddle.name)' answered its exceptional status probe; task $(running_task_id) keeps running\n");
  1761. } catch (Error e) {
  1762. warning(@"[ClankerSaddle] Saddle '$(saddle.name)' sent neither chunks nor results for $(stale_after_seconds())s and its status probe failed ($(e.message)); marking offline");
  1763. yield mark_saddle_unreachable(saddle.id, tasks,
  1764. @"no chunks or results for $(stale_after_seconds())s; status probe failed ($(e.message))");
  1765. }
  1766. }
  1767. /**
  1768. * Performs an HTTP request against a saddle daemon and parses the
  1769. * response body as JSON; transport failures propagate to the
  1770. * caller (the dispatcher marks the saddle unreachable). Transport
  1771. * blips — typically a pooled keep-alive connection closed by the
  1772. * daemon at the same instant the request reused it — are retried
  1773. * once before giving up. Unlike {@link request_saddle_json} the
  1774. * HTTP status is returned rather than thrown, and array bodies
  1775. * parse too.
  1776. */
  1777. private async int request_saddle(string host, string method, string path,
  1778. owned Json.Builder? payload, out Json.Node? root) throws Error {
  1779. root = null;
  1780. GLib.Error? failure = null;
  1781. for (var attempt = 0; attempt < 2; attempt++) {
  1782. try {
  1783. root = null;
  1784. int status = yield request_saddle_once(host, method, path, payload, out root);
  1785. return status;
  1786. } catch (Error e) {
  1787. failure = e;
  1788. }
  1789. if (((!)failure).code != 13 && ((!)failure).code != 14) {
  1790. break;
  1791. }
  1792. GLib.Timeout.add(250, request_saddle.callback);
  1793. yield;
  1794. }
  1795. throw (!)failure;
  1796. }
  1797. private async int request_saddle_once(string host, string method, string path,
  1798. Json.Builder? payload, out Json.Node? root) throws Error {
  1799. root = null;
  1800. string base_url = host.has_prefix("http") ? host : @"http://$(host)";
  1801. var message = new Soup.Message(method, @"$(base_url)$(path)");
  1802. if (payload != null) {
  1803. var generator = new Json.Generator();
  1804. generator.set_root(((!)payload).get_root());
  1805. message.set_request_body_from_bytes("application/json",
  1806. new Bytes(generator.to_data(null).data));
  1807. }
  1808. InputStream input;
  1809. try {
  1810. input = yield http.send_async(message, Priority.DEFAULT, null);
  1811. } catch (Error e) {
  1812. throw new Error(Quark.from_string("clanker-saddle"), 13,
  1813. @"Saddle request to $(base_url)$(path) failed: $(e.message)");
  1814. }
  1815. var output = new MemoryOutputStream.resizable();
  1816. try {
  1817. yield output.splice_async(input, OutputStreamSpliceFlags.CLOSE_SOURCE,
  1818. Priority.DEFAULT, null);
  1819. yield output.close_async(Priority.DEFAULT, null);
  1820. } catch (Error e) {
  1821. throw new Error(Quark.from_string("clanker-saddle"), 14,
  1822. @"Failed reading saddle response: $(e.message)");
  1823. }
  1824. var written = output.get_data_size();
  1825. uint8[] data = output.steal_data();
  1826. // steal_data's array length is -1 and the buffer is sized to
  1827. // the data — there is no NUL terminator to rely on — so slice
  1828. // by the written byte count before reading it as a string.
  1829. string response_text = written > 0 && data != null && data[0] != '\0'
  1830. ? ((string) data).substring(0, (long) written) : "{}";
  1831. var parser = new Json.Parser();
  1832. try {
  1833. parser.load_from_data(response_text);
  1834. } catch (Error e) {
  1835. throw new Error(Quark.from_string("clanker-saddle"), 16,
  1836. @"Saddle returned invalid JSON: $(e.message)");
  1837. }
  1838. root = parser.get_root();
  1839. return (int) message.status_code;
  1840. }
  1841. /**
  1842. * Performs an HTTP request against a saddle daemon and returns the
  1843. * parsed JSON object body (an empty object for non-object bodies);
  1844. * HTTP statuses of 400 and above throw.
  1845. */
  1846. private async Json.Object request_saddle_json(string host, string method, string path,
  1847. owned Json.Builder? payload) throws Error {
  1848. Json.Node? root = null;
  1849. int status_code = yield request_saddle(host, method, path, (owned) payload, out root);
  1850. if (status_code == 409) {
  1851. // Machine-readable per the wire contract: the body (state,
  1852. // running_task_id) rides in the message so the dispatch
  1853. // path can distinguish zombie conflicts from transport
  1854. // failures and recover.
  1855. string body_text = "{}";
  1856. if (root != null && root.get_node_type() == Json.NodeType.OBJECT) {
  1857. var generator = new Json.Generator();
  1858. generator.set_root(root);
  1859. body_text = generator.to_data(null);
  1860. }
  1861. throw new Error(Quark.from_string("clanker-saddle"), 409,
  1862. @"busy: $(body_text)");
  1863. }
  1864. if (status_code >= 400) {
  1865. throw new Error(Quark.from_string("clanker-saddle"), 15,
  1866. @"Saddle request returned HTTP $(status_code)");
  1867. }
  1868. if (root == null || root.get_node_type() != Json.NodeType.OBJECT) {
  1869. return new Json.Object();
  1870. }
  1871. return root.get_object();
  1872. }
  1873. /**
  1874. * A `{"name", "git_url", "path", "branch"}` project reference in
  1875. * the saddle daemon's wire shape: `git_url` is the clone source,
  1876. * and the checkout lives at
  1877. * `SADDLE_PROJECTS_ROOT/task-<task_id>/<name>` inside the
  1878. * saddle's container — one workspace per task, so concurrent
  1879. * tasks never share a checkout.
  1880. */
  1881. /**
  1882. * The projects root assignment paths are addressed against: the
  1883. * ASKING saddle's registered `workspace_root` (the daemon's own
  1884. * SADDLE_WORKSPACE_ROOT, reported at /register — goal 8 task 6,
  1885. * so a fleet no longer assumes one shared /workspace mount),
  1886. * else the `SADDLE_PROJECTS_ROOT` environment override, else
  1887. * the historical "/workspace" default. Paths are ADVISORY: the
  1888. * daemon owns its filesystem and rewrites them onto its own
  1889. * workspace root.
  1890. */
  1891. private static string projects_root_for_saddle(Saddle? saddle) {
  1892. var registered = saddle != null ? ((!)saddle).workspace_root : null;
  1893. if (registered != null && ((!)registered).strip().length > 0) {
  1894. return ((!)registered).strip();
  1895. }
  1896. var root_var = (Environment.get_variable("SADDLE_PROJECTS_ROOT") ?? "").strip();
  1897. return root_var.length > 0 ? root_var : "/workspace";
  1898. }
  1899. private static Json.Node project_node(Project project, int64 task_id, bool is_reference = false,
  1900. Saddle? saddle = null) {
  1901. string projects_root = projects_root_for_saddle(saddle);
  1902. var builder = new Json.Builder();
  1903. builder.begin_object();
  1904. builder.set_member_name("name");
  1905. builder.add_string_value(project.name);
  1906. builder.set_member_name("git_url");
  1907. builder.add_string_value(project.git_url);
  1908. builder.set_member_name("path");
  1909. builder.add_string_value(is_reference
  1910. ? @"$(projects_root)/task-$(task_id)/reference/$(project.name)"
  1911. : @"$(projects_root)/task-$(task_id)/$(project.name)");
  1912. builder.set_member_name("branch");
  1913. builder.add_string_value(project.default_branch);
  1914. builder.end_object();
  1915. return builder.get_root();
  1916. }
  1917. /**
  1918. * The answer as the saddle daemon expects it over /answer (or in a
  1919. * resume payload): verbatim. The daemon applies its own
  1920. * BlockRequest formatting, so this only strips the "Human
  1921. * selected: …"/"Human answered (other): …" prefixes still stored
  1922. * by answers recorded before the daemon took over formatting.
  1923. */
  1924. private static string raw_answer_for_saddle(string answer) {
  1925. if (answer.has_prefix("Human selected: ")) {
  1926. return answer.substring("Human selected: ".length);
  1927. }
  1928. if (answer.has_prefix("Human answered (other): ")) {
  1929. return answer.substring("Human answered (other): ".length);
  1930. }
  1931. return answer;
  1932. }
  1933. }
  1934. }