SaddleService.vala 96 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999
  1. using Invercargill;
  2. using Invercargill.DataStructures;
  3. using Invercargill.Expressions;
  4. using InvercargillSql;
  5. using InvercargillSql.Orm;
  6. using Inversion;
  7. using Soup;
  8. using ClankerSaddle.Data;
  9. namespace ClankerSaddle.Services {
  10. /** Caps of the `project_notes` array in the /start payload. */
  11. public const int64 PROJECT_NOTES_MAX_ENTRIES = 30;
  12. public const int64 PROJECT_NOTES_MAX_CHARS = 20000;
  13. /** A block answer awaiting collection by one saddle's next poll. */
  14. public class PendingAnswer : Object {
  15. public int64 block_id { get; set; }
  16. public string answer { get; set; default = ""; }
  17. public bool branch_advanced { get; set; }
  18. }
  19. /** A human comment awaiting delivery to one saddle's conversation. */
  20. public class PendingComment : Object {
  21. public int64 comment_id { get; set; }
  22. public string author { get; set; default = ""; }
  23. public string body { get; set; default = ""; }
  24. public string quote { get; set; default = ""; }
  25. }
  26. /** A saddle registration plus the API token generated for it. */
  27. public class SaddleRegistration : Object {
  28. public Saddle saddle { get; set; }
  29. public string token { get; set; default = ""; }
  30. }
  31. /**
  32. * The OpenAI-compatible endpoint every saddle assignment is sent to,
  33. * read from the environment: `SADDLE_LLM_BASE_URL` (required — dispatch
  34. * pauses without it), `SADDLE_LLM_API_KEY` and `SADDLE_LLM_MODEL`.
  35. */
  36. public class LlmEndpoint : Object {
  37. public string base_url { get; set; default = ""; }
  38. public string api_key { get; set; default = ""; }
  39. public string model { get; set; default = ""; }
  40. /** The configured endpoint, or null when the base URL is unset. */
  41. public static LlmEndpoint? from_environment() {
  42. var base_url = (Environment.get_variable("SADDLE_LLM_BASE_URL") ?? "").strip();
  43. if (base_url.length == 0) {
  44. return null;
  45. }
  46. return new LlmEndpoint() {
  47. base_url = base_url,
  48. api_key = (Environment.get_variable("SADDLE_LLM_API_KEY") ?? "").strip(),
  49. model = (Environment.get_variable("SADDLE_LLM_MODEL") ?? "").strip()
  50. };
  51. }
  52. }
  53. /**
  54. * Saddle registry and control-plane client (plan 10 phase 2): registers
  55. * containers, finds idle capacity for the dispatcher's wake
  56. * computation, wakes sleeping daemons, and ingests the telemetry the
  57. * daemons PUSH — raw PTY chunks (`POST /api/saddle/chunks`) and round
  58. * results with state transitions (`POST /api/saddle/results`) — reusing
  59. * the storage logic the deleted `/output` + `/log` pollers ran. GET
  60. * /status against a daemon is exceptional only: the staleness probe
  61. * for a silent runner and the manual refresh button. Block answers are
  62. * handed out exactly once (delivered ids are remembered in memory; a
  63. * restart re-delivers and the daemon deduplicates).
  64. */
  65. public class SaddleService : Object {
  66. private OrmSession db = inject<OrmSession>();
  67. private Connection connection = inject<Connection>();
  68. /**
  69. * Every daemon request shares this session and therefore its
  70. * timeouts: without them a single hung request (a daemon wedged
  71. * mid-response over the keep-alive pool) froze the dispatcher's
  72. * tick latch forever. `timeout` bounds each request
  73. * (SADDLE_HTTP_TIMEOUT_S, default 30s); `idle_timeout` reaps
  74. * pooled connections so a peer that vanished between requests is
  75. * not dialed again.
  76. */
  77. private Soup.Session http = new Soup.Session() {
  78. timeout = http_timeout_seconds(),
  79. idle_timeout = 10
  80. };
  81. private static uint http_timeout_seconds() {
  82. var raw = Environment.get_variable("SADDLE_HTTP_TIMEOUT_S") ?? "";
  83. int64 parsed = int64.parse(raw);
  84. return parsed > 0 ? (uint) parsed : 30;
  85. }
  86. private SaddlePollState poll_state = inject<SaddlePollState>();
  87. private StatePulse pulse = inject<StatePulse>();
  88. private PushService push = inject<PushService>();
  89. private RuleService rules = inject<RuleService>();
  90. private CapabilityService capability_service = inject<CapabilityService>();
  91. private TaskService task_service = inject<TaskService>();
  92. private GoalService goal_service = inject<GoalService>();
  93. private Astralis.WebConfig config = inject<Astralis.WebConfig>();
  94. /**
  95. * Registers a saddle container: inserts the row with a freshly
  96. * minted per-saddle token (plan 10) and hands the credential to
  97. * the daemon via `POST /register {saddle_token, web_base_url}`
  98. * (best effort — an older daemon without the route stays
  99. * registered; the operator copies the token from the saddles
  100. * page instead). `host` is host:port of its control server; the
  101. * returned {@link SaddleRegistration} carries the token the
  102. * daemon must present as `Authorization: Bearer …` on
  103. * `/api/saddle/*` calls.
  104. */
  105. public async SaddleRegistration register_saddle(string name, string host) throws Error {
  106. var saddle = new Saddle() {
  107. name = name,
  108. host = host,
  109. status = "idle",
  110. token = mint_saddle_token()
  111. };
  112. yield db.insert_async<Saddle>(saddle);
  113. yield register_daemon(saddle);
  114. yield pulse.bump(0, "saddle registered");
  115. return new SaddleRegistration() { saddle = saddle, token = saddle.token ?? "" };
  116. }
  117. /** A fresh per-saddle bearer token (`saddle-<random>`). */
  118. private static string mint_saddle_token() {
  119. return @"saddle-$(Uuid.string_random().replace("-", ""))";
  120. }
  121. /**
  122. * Delivers the per-saddle token to the daemon's
  123. * `POST /register` endpoint (wire contract): the daemon
  124. * persists it for its calls home and answers
  125. * `{ok, ssh_public_key, version}` — the key is stored on the
  126. * row the moment the daemon reports one. A daemon without the
  127. * route (HTTP 404/405) or unreachable at registration time is
  128. * tolerated: the row keeps the token, the operator can rety
  129. * via a refresh or copy the token manually.
  130. */
  131. public async void register_daemon(Saddle saddle) throws Error {
  132. var payload = new Json.Builder();
  133. payload.begin_object();
  134. payload.set_member_name("saddle_token");
  135. payload.add_string_value(saddle.token ?? "");
  136. payload.set_member_name("web_base_url");
  137. payload.add_string_value(callback_base_url());
  138. payload.end_object();
  139. try {
  140. var response = yield request_saddle_json(saddle.host, "POST", "/register",
  141. (owned) payload);
  142. string reported_key = string_member(response, "ssh_public_key");
  143. if (reported_key.length > 0 && reported_key != (saddle.ssh_public_key ?? "")) {
  144. saddle.ssh_public_key = reported_key;
  145. yield db.update_async<Saddle>(saddle);
  146. }
  147. print(@"[ClankerSaddle] Registered with saddle '$(saddle.name)'s daemon (token delivered)\n");
  148. } catch (Error e) {
  149. print(@"[ClankerSaddle] Saddle '$(saddle.name)'s daemon did not accept /register ($(e.message)); token stays on the row\n");
  150. }
  151. }
  152. /**
  153. * Startup migration of the per-saddle token: every saddle row
  154. * gets its canonical token — the latest legacy `api_tokens`
  155. * value it already holds (daemons registered before plan 10
  156. * keep their credential), or a freshly minted one. Existing
  157. * rows with a token are left alone.
  158. */
  159. public async void ensure_saddle_tokens() throws Error {
  160. foreach (var saddle in yield list_saddles()) {
  161. if ((saddle.token ?? "").length > 0) {
  162. continue;
  163. }
  164. string? legacy = yield token_for_saddle(saddle.id);
  165. saddle.token = legacy != null && ((!)legacy).length > 0
  166. ? (!)legacy : mint_saddle_token();
  167. yield db.update_async<Saddle>(saddle);
  168. print(@"[ClankerSaddle] Saddle '$(saddle.name)' token ready ($(saddle.token.substring(0, 12))…)\n");
  169. }
  170. }
  171. /**
  172. * Generates and stores a fresh API token for `saddle_id`,
  173. * replacing any earlier one on the wire (older rows remain valid
  174. * until removed; in practice each saddle holds one).
  175. */
  176. public async string generate_token(int64 saddle_id) throws Error {
  177. var row = new ApiToken() {
  178. saddle_id = saddle_id,
  179. token = @"saddle-$(Uuid.string_random().replace("-", ""))",
  180. created_at = new DateTime.now_local()
  181. };
  182. yield db.insert_async<ApiToken>(row);
  183. return row.token;
  184. }
  185. /**
  186. * True when `token` matches the credential of `saddle_id` — its
  187. * per-saddle row token (plan 10), or any legacy `api_tokens`
  188. * row still held for it (tokens minted before the column
  189. * existed keep validating) — the credential check for the
  190. * `/api/saddle/*` endpoints.
  191. */
  192. public async bool validate_token(int64 saddle_id, string token) throws Error {
  193. if (token.length == 0) {
  194. return false;
  195. }
  196. var saddle = yield get_saddle(saddle_id);
  197. if (saddle != null && ((!)saddle).token == token) {
  198. return true;
  199. }
  200. var rows = yield db.query<ApiToken>()
  201. .where(expr("saddle_id == $0", elem<int64?>(saddle_id)))
  202. .materialise_async();
  203. foreach (var row in rows) {
  204. if (row.token == token) {
  205. return true;
  206. }
  207. }
  208. return false;
  209. }
  210. /**
  211. * The saddle a bearer token belongs to, or null — the
  212. * credential check for the id-less `/api/saddle/*` endpoints
  213. * (`next-task`, `note`, `task`): the row token
  214. * first, then any legacy `api_tokens` row (mapped to its
  215. * saddle). Removing the saddle revokes both.
  216. */
  217. public async Saddle? saddle_for_token(string token) throws Error {
  218. if (token.length == 0) {
  219. return null;
  220. }
  221. var by_row = yield db.query<Saddle>()
  222. .where(expr("token == $0", elem(token)))
  223. .first_async();
  224. if (by_row != null) {
  225. return by_row;
  226. }
  227. var legacy = yield db.query<ApiToken>()
  228. .where(expr("token == $0", elem(token)))
  229. .first_async();
  230. if (legacy == null) {
  231. return null;
  232. }
  233. return yield get_saddle(((!)legacy).saddle_id);
  234. }
  235. /**
  236. * The most recent API token issued for `saddle_id`, or null: the
  237. * row token first, then the latest legacy `api_tokens` row.
  238. */
  239. public async string? token_for_saddle(int64 saddle_id) throws Error {
  240. var saddle = yield get_saddle(saddle_id);
  241. if (saddle != null && (((!)saddle).token ?? "").length > 0) {
  242. return ((!)saddle).token;
  243. }
  244. ApiToken? latest = null;
  245. var rows = yield db.query<ApiToken>()
  246. .where(expr("saddle_id == $0", elem<int64?>(saddle_id)))
  247. .order_by(expr("id"))
  248. .materialise_async();
  249. foreach (var row in rows) {
  250. latest = row;
  251. }
  252. return latest != null ? ((!)latest).token : null;
  253. }
  254. /**
  255. * This app's externally reachable base URL — what daemons use to
  256. * POST notes back — read from the `saddle_callback_base_url` key
  257. * of web-config.json, falling back to the
  258. * `SADDLE_CALLBACK_BASE_URL` environment variable and then to
  259. * `http://localhost:8080`. The daemon typically runs in a
  260. * container or on another host, so its view of this app differs
  261. * from the browser's; set the key to whatever reaches this
  262. * process from there.
  263. */
  264. public string callback_base_url() {
  265. string configured = config.get_string("saddle_callback_base_url", "").strip();
  266. if (configured.length > 0) {
  267. return configured;
  268. }
  269. var from_env = (Environment.get_variable("SADDLE_CALLBACK_BASE_URL") ?? "").strip();
  270. if (from_env.length > 0) {
  271. return from_env;
  272. }
  273. return "http://localhost:8080";
  274. }
  275. /**
  276. * Removes a saddle registration; tasks it was running are
  277. * re-queued. Revocation: the row (and its per-saddle token)
  278. * dies here, and every legacy `api_tokens` row held for the
  279. * saddle is deleted — nothing authorises `/api/saddle/*` for
  280. * it afterwards.
  281. */
  282. public async void unregister_saddle(int64 id) throws Error {
  283. var saddle = yield get_saddle(id);
  284. if (saddle == null) {
  285. return;
  286. }
  287. if (((!)saddle).current_task_id > 0) {
  288. var task = yield db.query<TaskEntity>()
  289. .where(expr("id == $0", elem<int64?>(((!)saddle).current_task_id)))
  290. .first_async();
  291. if (task != null && (((!)task).status == STATUS_ASSIGNED
  292. || ((!)task).status == STATUS_WORKING || ((!)task).status == STATUS_BLOCKED
  293. || ((!)task).status == STATUS_REVIEW_REQUESTED)) {
  294. ((!)task).status = STATUS_QUEUED;
  295. yield db.update_async<TaskEntity>((!)task);
  296. }
  297. }
  298. foreach (var row in yield db.query<ApiToken>()
  299. .where(expr("saddle_id == $0", elem<int64?>(id)))
  300. .materialise_async()) {
  301. yield db.delete_async<ApiToken>(row);
  302. }
  303. yield db.delete_async<Saddle>((!)saddle);
  304. yield pulse.bump(0, "saddle removed");
  305. }
  306. /** Loads the saddle identified by `id`, or null. */
  307. public async Saddle? get_saddle(int64 id) throws Error {
  308. return yield db.query<Saddle>()
  309. .where(expr("id == $0", elem<int64?>(id)))
  310. .first_async();
  311. }
  312. /** Lists every saddle ordered by name. */
  313. public async ImmutableLot<Saddle> list_saddles() throws Error {
  314. return yield db.query<Saddle>()
  315. .order_by(expr("name"))
  316. .materialise_async();
  317. }
  318. /**
  319. * Lists saddles with status 'busy' (the dispatcher's poll
  320. * candidates).
  321. */
  322. public async ImmutableLot<Saddle> list_busy_saddles() throws Error {
  323. return yield db.query<Saddle>()
  324. .where(expr("status == $0", elem("busy")))
  325. .materialise_async();
  326. }
  327. /**
  328. * Records a /status observation into the health snapshot,
  329. * tolerating older daemons without the uptime/activity fields.
  330. */
  331. /**
  332. * Persists the saddle's git identity — what its daemon commits
  333. * as (`git_identity` in every /start). Empty/null clears the
  334. * fields so the daemon's fallback applies.
  335. */
  336. public async void update_git_identity(int64 id, string? git_name,
  337. string? git_email) throws Error {
  338. var saddle = yield get_saddle(id);
  339. if (saddle == null) {
  340. throw new Error(Quark.from_string("clanker-saddle"), 17,
  341. @"No saddle exists with id $(id)");
  342. }
  343. ((!)saddle).git_name = git_name != null && git_name.length > 0 ? git_name : null;
  344. ((!)saddle).git_email = git_email != null && git_email.length > 0 ? git_email : null;
  345. yield db.update_async<Saddle>((!)saddle);
  346. yield pulse.bump(0, "saddle identity saved");
  347. }
  348. /**
  349. * Persists the daemon's reported `ssh_public_key` onto the saddle
  350. * row (the plan-9 contract: the key travels on /status, empty
  351. * until the daemon generates one; the saddles page shows it for
  352. * adding as a deploy key). A single-column UPDATE — the
  353. * full-row write could stomp a concurrent ask claim. Only writes
  354. * when it changed.
  355. */
  356. private async void store_reported_ssh_key(Saddle saddle, Json.Object status) throws Error {
  357. if (!status.has_member("ssh_public_key")) {
  358. return;
  359. }
  360. string reported = status.get_string_member_with_default("ssh_public_key", "");
  361. string stored = saddle.ssh_public_key ?? "";
  362. if (reported == stored) {
  363. return;
  364. }
  365. var command = connection.create_command(
  366. "UPDATE saddles SET ssh_public_key = :key WHERE id = :id");
  367. command.with_parameter<Invercargill.Element>("key", elem(reported.length > 0 ? reported : null));
  368. command.with_parameter<Invercargill.Element>("id", elem<int64?>(saddle.id));
  369. yield command.execute_non_query_async();
  370. }
  371. /**
  372. * Records a /status observation: the health snapshot, the
  373. * daemon-reported ssh key, and — because the daemon surfaces
  374. * `resumed` on the FIRST status after a workspace restore,
  375. * whichever probe happens to receive it (the dispatcher's poll,
  376. * a heal pass, a manual refresh) — the once-per-session resume
  377. * progress entry.
  378. */
  379. private async void record_status_probe(Saddle saddle, Json.Object status) throws Error {
  380. string state = status.has_member("state")
  381. ? status.get_string_member_with_default("state", "") : "";
  382. int64 uptime = status.has_member("uptime_seconds")
  383. && status.get_member("uptime_seconds").get_value_type() == typeof(int64)
  384. ? status.get_int_member("uptime_seconds") : 0;
  385. int64 activity = status.has_member("last_activity_epoch")
  386. && status.get_member("last_activity_epoch").get_value_type() == typeof(int64)
  387. ? status.get_int_member("last_activity_epoch") : 0;
  388. poll_state.record_probe(saddle.id, state, uptime, activity);
  389. yield store_reported_ssh_key(saddle, status);
  390. // Plan 12: a daemon running (or parking — the workspace
  391. // stays while the saddle frees) a goal agent exposes the
  392. // push state of the goal branches on /status; cache it on
  393. // the goal row for the completion-review UI.
  394. if (status.has_member("goal_branches")) {
  395. var goal_branches = status.get_member("goal_branches");
  396. if (goal_branches.get_node_type() == Json.NodeType.ARRAY
  397. || goal_branches.get_node_type() == Json.NodeType.OBJECT) {
  398. var generator = new Json.Generator();
  399. generator.set_root(goal_branches);
  400. string raw = generator.to_data(null);
  401. int64 push_goal_id = 0;
  402. if (saddle.current_task_id > 0) {
  403. push_goal_id = yield goal_id_of_run(saddle.current_task_id);
  404. } else {
  405. // Parked: the open session still names the
  406. // goal agent's workspace.
  407. var open_session = yield open_session_on_saddle(saddle.id);
  408. if (open_session != null) {
  409. push_goal_id = yield goal_id_of_run(((!)open_session).task_id);
  410. }
  411. }
  412. if (push_goal_id > 0) {
  413. yield goal_service.store_branch_push_state(push_goal_id, raw);
  414. }
  415. }
  416. }
  417. // The "interrupted -> resumed" timeline arc. Absent on older
  418. // daemons; marked once per session in SaddlePollState.
  419. if (status.has_member("resumed")
  420. && status.get_member("resumed").get_value_type() == typeof(bool)
  421. && status.get_boolean_member("resumed")
  422. && saddle.current_task_id > 0) {
  423. var session = yield task_service.open_session_for_task(saddle.current_task_id);
  424. if (session != null && poll_state.mark_resumed(((!)session).id)) {
  425. yield task_service.add_progress(saddle.current_task_id,
  426. "Saddle restored the session context; task resumed");
  427. }
  428. }
  429. }
  430. /**
  431. * Wakes a sleeping daemon (plan 10 wire contract): `POST /wake`
  432. * `{}` — a capable task is ready; sleeping daemons ask for work,
  433. * busy ones ignore it, idle-awake ones re-ask. Idempotent. The
  434. * caller records the pending wake for the lost-wake guard.
  435. */
  436. public async void wake_saddle(Saddle saddle) throws Error {
  437. var payload = new Json.Builder();
  438. payload.begin_object();
  439. payload.end_object();
  440. Json.Node? root = null;
  441. int status_code = yield request_saddle(saddle.host, "POST", "/wake",
  442. (owned) payload, out root);
  443. if (status_code >= 400) {
  444. throw new Error(Quark.from_string("clanker-saddle"), 49,
  445. @"wake returned HTTP $(status_code)");
  446. }
  447. }
  448. /**
  449. * Cancelling / requeueing / stopping a task with a live session
  450. * reaches its daemon (wire contract): `POST /task-update
  451. * {kind: "cancel"}` — the authoritative stop through the daemon's
  452. * abort machinery, the workspace kept on disk per the park
  453. * contract. Success is narrated as "<verb>; daemon aborted"; a
  454. * transport failure marks the saddle offline (the daemon's next
  455. * ask proves it alive again and flips the row back to idle). The
  456. * daemon must never keep running a task the web has moved on
  457. * from as a zombie.
  458. */
  459. public async void abort_live_session(TaskService tasks, int64 task_id,
  460. string verb) throws Error {
  461. var session = yield tasks.open_session_for_task(task_id);
  462. if (session == null) {
  463. return;
  464. }
  465. yield abort_session_on_its_saddle(tasks, (!)session, verb);
  466. }
  467. /**
  468. * SADDLE_STALE_S (default 90s): the silence window after which a
  469. * running daemon owes an exceptional GET /status probe.
  470. */
  471. public static uint stale_after_seconds() {
  472. var raw = Environment.get_variable("SADDLE_STALE_S") ?? "";
  473. int64 parsed = int64.parse(raw);
  474. return parsed > 0 ? (uint) parsed : 90;
  475. }
  476. /**
  477. * {@link abort_live_session} keyed by saddle instead of task —
  478. * the unregister path. The cancel is delivered to the saddle
  479. * whose own open session was found (NOT re-resolved through the
  480. * task's lowest-id open session, which a stale fixture row on
  481. * another saddle would redirect to the wrong daemon).
  482. */
  483. public async void abort_saddle_session(TaskService tasks, int64 saddle_id,
  484. string verb) throws Error {
  485. var session = yield open_session_on_saddle(saddle_id);
  486. if (session == null) {
  487. return;
  488. }
  489. yield abort_session_on_its_saddle(tasks, (!)session, verb);
  490. }
  491. /**
  492. * The cancel delivery core shared by the task-keyed and
  493. * saddle-keyed abort paths: `POST /task-update {kind: cancel}`
  494. * to the daemon holding THIS session.
  495. */
  496. private async void abort_session_on_its_saddle(TaskService tasks,
  497. ClankerSaddle.Data.Session session, string verb) throws Error {
  498. var saddle = yield get_saddle(session.saddle_id);
  499. if (saddle == null) {
  500. return;
  501. }
  502. var payload = new Json.Builder();
  503. payload.begin_object();
  504. payload.set_member_name("kind");
  505. payload.add_string_value("cancel");
  506. payload.end_object();
  507. Json.Node? root = null;
  508. try {
  509. int status_code = yield request_saddle(((!)saddle).host, "POST",
  510. "/task-update", (owned) payload, out root);
  511. if (status_code >= 400) {
  512. throw new Error(Quark.from_string("clanker-saddle"), 45,
  513. @"task-update cancel returned HTTP $(status_code)");
  514. }
  515. yield tasks.add_comment(session.task_id, "agent", @"$(verb); daemon aborted");
  516. } catch (Error e) {
  517. warning(@"[ClankerSaddle] Abort for task $(session.task_id) failed: $(e.message); marking saddle '$(((!)saddle).name)' offline");
  518. yield mark_saddle_unreachable(((!)saddle).id, tasks,
  519. @"abort delivery failed ($(e.message))");
  520. yield tasks.add_comment(session.task_id, "agent",
  521. @"$(verb); daemon unreachable — saddle marked offline");
  522. }
  523. }
  524. /**
  525. * Requeues tasks caught running on saddles that have since gone
  526. * offline (a manual status refresh, a restart) — without this
  527. * pass they would wait forever for a daemon that will not answer.
  528. * Only `assigned`/`working` tasks move: parked tasks
  529. * (blocked/review/suspended) keep their sessions, which the
  530. * resume machinery needs. Each move lands a agent comment so the
  531. * timeline shows why the task restarted.
  532. */
  533. public async void requeue_tasks_on_offline_saddles(TaskService tasks) throws Error {
  534. foreach (var status in new string[] { STATUS_ASSIGNED, STATUS_WORKING }) {
  535. foreach (var task in yield db.query<TaskEntity>()
  536. .where(expr("status == $0", elem(status)))
  537. .materialise_async()) {
  538. var session = yield tasks.open_session_for_task(task.id);
  539. if (session == null) {
  540. continue;
  541. }
  542. if (yield tasks.newest_block_answered(task.id)) {
  543. // Parked-with-answered: a verdict awaits the
  544. // resume machinery — never requeue it.
  545. continue;
  546. }
  547. var saddle = yield get_saddle(((!)session).saddle_id);
  548. if (saddle == null || ((!)saddle).status != "offline") {
  549. continue;
  550. }
  551. print(@"[ClankerSaddle] Task $(task.id) requeued: its saddle went offline\n");
  552. yield tasks.requeue_task(task.id);
  553. yield tasks.add_comment(task.id, "agent",
  554. "saddle went offline; task requeued");
  555. }
  556. }
  557. }
  558. /**
  559. * Parks a saddle (idle, no task) only when it still runs
  560. * `task_id` — a single conditional UPDATE, atomic against the
  561. * ask allocation claiming a NEW task for the saddle between the
  562. * poller's read and its write (the poller's "blocked"/"suspended"
  563. * parking races endpoint-side claims otherwise, leaving the
  564. * freshly-claimed task assigned to a busy@0 saddle nothing
  565. * polls).
  566. */
  567. public async void park_if_current(int64 saddle_id, int64 task_id) throws Error {
  568. var command = connection.create_command(
  569. "UPDATE saddles SET status = 'idle', current_task_id = 0, allocated_at = NULL"
  570. + " WHERE id = :id AND current_task_id = :task");
  571. command.with_parameter<Invercargill.Element>("id", elem<int64?>(saddle_id));
  572. command.with_parameter<Invercargill.Element>("task", elem<int64?>(task_id));
  573. yield command.execute_non_query_async();
  574. yield pulse.bump(0, "saddle parked");
  575. }
  576. /**
  577. * Every saddle with status 'idle', registration order — the
  578. * dispatcher's capability-matching candidate list.
  579. */
  580. public async ImmutableLot<Saddle> list_idle_saddles() throws Error {
  581. return yield db.query<Saddle>()
  582. .where(expr("status == $0", elem("idle")))
  583. .order_by(expr("id"))
  584. .materialise_async();
  585. }
  586. /**
  587. * Marks a saddle offline when its control server cannot be
  588. * reached, releasing the task it was holding back to 'queued'
  589. * and closing that task's open session (outcome 'stopped'). The
  590. * requeue lands as a agent comment on the task, so the timeline
  591. * shows the interruption; combined with the daemon's
  592. * crash-resume, the next dispatch restores the workspace and
  593. * reports `resumed` on /status.
  594. */
  595. public async void mark_saddle_unreachable(int64 id, TaskService tasks,
  596. string? reason = null) throws Error {
  597. var saddle = yield get_saddle(id);
  598. if (saddle == null) {
  599. return;
  600. }
  601. int64 held_task_at_entry = ((!)saddle).current_task_id;
  602. if (((!)saddle).current_task_id > 0) {
  603. int64 task_id = ((!)saddle).current_task_id;
  604. bool keep_parked = false;
  605. var task = yield db.query<TaskEntity>()
  606. .where(expr("id == $0", elem<int64?>(task_id)))
  607. .first_async();
  608. if (task != null && (((!)task).status == STATUS_ASSIGNED
  609. || ((!)task).status == STATUS_WORKING || ((!)task).status == STATUS_BLOCKED
  610. || ((!)task).status == STATUS_REVIEW_REQUESTED)) {
  611. keep_parked = yield tasks.newest_block_answered(task_id);
  612. if (keep_parked) {
  613. // Parked-with-answered: the verdict must survive
  614. // for the resume (answer delivery, or the
  615. // affinity allocation when the daemon asks
  616. // again) — the session stays open and the task
  617. // stays parked.
  618. print(@"[ClankerSaddle] Task $(task_id) keeps its parked session: its answered block awaits delivery\n");
  619. } else {
  620. ((!)task).status = STATUS_QUEUED;
  621. yield db.update_async<TaskEntity>((!)task);
  622. string unreachable_text = reason != null && reason.length > 0
  623. ? @"Saddle became unreachable ($(reason)); task requeued"
  624. : "Saddle became unreachable; task requeued";
  625. yield tasks.add_comment(task_id, "agent", unreachable_text);
  626. }
  627. }
  628. if (!keep_parked) {
  629. var sessions = yield db.query<ClankerSaddle.Data.Session>()
  630. .where(expr("task_id == $0", elem<int64?>(task_id)))
  631. .materialise_async();
  632. foreach (var session in sessions) {
  633. if (session.ended_at != null) {
  634. continue;
  635. }
  636. session.ended_at = new DateTime.now_local();
  637. session.outcome = "stopped";
  638. yield db.update_async<ClankerSaddle.Data.Session>(session);
  639. }
  640. }
  641. }
  642. var offline_write = connection.create_command(
  643. "UPDATE saddles SET status = 'offline', current_task_id = 0, allocated_at = NULL"
  644. + " WHERE id = :id AND current_task_id = :seen");
  645. offline_write.with_parameter<Invercargill.Element>("id", elem<int64?>(id));
  646. offline_write.with_parameter<Invercargill.Element>("seen",
  647. elem<int64?>(held_task_at_entry));
  648. yield offline_write.execute_non_query_async();
  649. // An offline saddle is neither sleeping nor wake-awaiting;
  650. // a restarted daemon re-establishes both by asking.
  651. poll_state.clear_ask(id);
  652. yield pulse.bump(0, "saddle unreachable");
  653. }
  654. /**
  655. * Assembles the assignment payload shared by the /start push and
  656. * the `POST /api/saddle/next-task` response — `{task, kind,
  657. * mode, creator, goal?, rules, project_notes, callback,
  658. * llm_config, session_id}` — with the project, links (the
  659. * goal's own links for goal work, else the project's), branch
  660. * fields and recent comments inside the task object, plus any
  661. * block answers not yet handed over (a resumed workspace picks
  662. * its pending answers up from the assignment itself), the
  663. * effective rule set ({@link RuleService.effective_rules}) as
  664. * the top-level `rules` array, the capped `project_notes` of
  665. * the primary and co-developed projects, the note-creation
  666. * `callback` (base URL + the saddle's own bearer token), the
  667. * saddle's `git_identity`, a `capabilities` context (plan 10
  668. * phase 3: `saddle_tags` — the allocated saddle's own tags —
  669. * and `fleet_tags` — every tag name known across the fleet),
  670. * the plan-12 goal context (`goal` for goal work; the
  671. * worker's `ephemeral_branch`) and — for resumed workspaces —
  672. * the `resume_notice`/`branch_advanced` context. `session_id`
  673. * (the web-side run row the daemon reports against) rides
  674. * along so pushed telemetry can key on it. Project checkouts
  675. * are addressed by the `SADDLE_PROJECTS_ROOT` convention
  676. * (`root/task-<task id>/name`) and cloned from each project's
  677. * `git_url`; the LLM endpoint by the {@link LlmEndpoint}
  678. * environment.
  679. */
  680. public async Json.Object build_assignment_payload(Saddle saddle, TaskEntity task,
  681. int64 session_id, ProjectService projects, string? resume_notice = null,
  682. bool? branch_advanced = null,
  683. string? context_update = null) throws Error {
  684. var endpoint = LlmEndpoint.from_environment();
  685. if (endpoint == null) {
  686. throw new Error(Quark.from_string("clanker-saddle"), 24,
  687. "No LLM endpoint configured (set SADDLE_LLM_BASE_URL)");
  688. }
  689. var project = yield db.query<Project>()
  690. .where(expr("id == $0", elem<int64?>(task.project_id)))
  691. .first_async();
  692. if (project == null) {
  693. throw new Error(Quark.from_string("clanker-saddle"), 12,
  694. @"Task $(task.id) has no project $(task.project_id)");
  695. }
  696. // Plan 12: goal work (the agent's own run and its worker
  697. // tasks) scopes its co-developed/reference set through the
  698. // GOAL's links; standalone tasks through the project's.
  699. Goal? goal = null;
  700. if (task.goal_id > 0) {
  701. goal = yield db.query<Goal>()
  702. .where(expr("id == $0", elem<int64?>(task.goal_id)))
  703. .first_async();
  704. }
  705. var co_developed = new Vector<Project>();
  706. var references = new Vector<Project>();
  707. if (goal != null) {
  708. foreach (var link in yield goal_service.list_links(task.goal_id)) {
  709. var linked = yield db.query<Project>()
  710. .where(expr("id == $0", elem<int64?>(link.project_id)))
  711. .first_async();
  712. if (linked == null) {
  713. continue;
  714. }
  715. if (link.relationship == LINK_CO_DEVELOPED) {
  716. co_developed.add((!)linked);
  717. } else if (link.relationship == LINK_REFERENCE) {
  718. references.add((!)linked);
  719. }
  720. }
  721. } else {
  722. foreach (var link in yield projects.list_links(task.project_id)) {
  723. var linked = yield db.query<Project>()
  724. .where(expr("id == $0", elem<int64?>(link.linked_project_id)))
  725. .first_async();
  726. if (linked == null) {
  727. continue;
  728. }
  729. if (link.relationship == LINK_CO_DEVELOPED) {
  730. co_developed.add((!)linked);
  731. } else if (link.relationship == LINK_REFERENCE) {
  732. references.add((!)linked);
  733. }
  734. }
  735. }
  736. var payload = new Json.Builder();
  737. payload.begin_object();
  738. payload.set_member_name("task");
  739. payload.begin_object();
  740. payload.set_member_name("id");
  741. payload.add_int_value(task.id);
  742. payload.set_member_name("title");
  743. payload.add_string_value(task.title);
  744. payload.set_member_name("description");
  745. payload.add_string_value(task.description ?? "");
  746. payload.set_member_name("project_name");
  747. payload.add_string_value(((!)project).name);
  748. payload.set_member_name("project_branch");
  749. payload.add_string_value(((!)project).default_branch);
  750. payload.set_member_name("task_branch");
  751. payload.add_string_value(task.task_branch ?? "");
  752. payload.set_member_name("branch_from");
  753. payload.add_string_value(task.branch_from ?? ((!)project).default_branch);
  754. payload.set_member_name("co_developed");
  755. payload.begin_array();
  756. payload.add_value(project_node(((!)project), task.id));
  757. foreach (var linked in co_developed) {
  758. payload.add_value(project_node(linked, task.id));
  759. }
  760. payload.end_array();
  761. payload.set_member_name("reference");
  762. payload.begin_array();
  763. foreach (var linked in references) {
  764. payload.add_value(project_node(linked, task.id, true));
  765. }
  766. payload.end_array();
  767. payload.set_member_name("comments");
  768. payload.begin_array();
  769. var comments = yield db.query<TaskComment>()
  770. .where(expr("task_id == $0", elem<int64?>(task.id)))
  771. .order_by(expr("id"))
  772. .materialise_async();
  773. foreach (var comment in comments) {
  774. payload.begin_object();
  775. payload.set_member_name("author");
  776. payload.add_string_value(comment.author);
  777. payload.set_member_name("body");
  778. payload.add_string_value(comment.body);
  779. if (comment.quote != null) {
  780. payload.set_member_name("quote");
  781. payload.add_string_value(((!)comment.quote));
  782. }
  783. payload.end_object();
  784. }
  785. payload.end_array();
  786. payload.set_member_name("answers");
  787. payload.begin_array();
  788. foreach (var pending in yield take_pending_answers_for_task(task.id)) {
  789. payload.begin_object();
  790. payload.set_member_name("block_id");
  791. payload.add_int_value(pending.block_id);
  792. payload.set_member_name("answer");
  793. payload.add_string_value(raw_answer_for_saddle(pending.answer));
  794. if (branch_advanced != null) {
  795. payload.set_member_name("branch_advanced");
  796. payload.add_boolean_value((!)branch_advanced);
  797. }
  798. payload.end_object();
  799. }
  800. payload.end_array();
  801. payload.end_object();
  802. // The plan-12 wire additions: kind ("task"|"goal_run"),
  803. // mode and creator ride the top level beside the task.
  804. payload.set_member_name("kind");
  805. payload.add_string_value(task.kind);
  806. payload.set_member_name("mode");
  807. payload.add_string_value(task.mode);
  808. payload.set_member_name("creator");
  809. payload.add_string_value(task.creator);
  810. // The goal context (goal work only): the goal agent gets
  811. // the full brief + approved plan; a goal worker gets the
  812. // same object as read-only reference (the daemon adds the
  813. // stick-to-mandate shaping) plus its ephemeral branch.
  814. if (goal != null) {
  815. payload.set_member_name("goal");
  816. payload.begin_object();
  817. payload.set_member_name("id");
  818. payload.add_int_value(((!)goal).id);
  819. payload.set_member_name("branch");
  820. payload.add_string_value(((!)goal).branch_name);
  821. payload.set_member_name("base_branch");
  822. payload.add_string_value(((!)goal).base_branch
  823. ?? ((!)project).default_branch);
  824. payload.set_member_name("brief");
  825. payload.add_string_value(((!)goal).brief);
  826. payload.set_member_name("plan");
  827. payload.add_string_value(((!)goal).plan ?? "");
  828. payload.end_object();
  829. if (task.kind == TASK_KIND_TASK) {
  830. payload.set_member_name("ephemeral_branch");
  831. payload.add_string_value(@"task/$(task.goal_id)-$(task.id)");
  832. }
  833. }
  834. // The wire contract's top-level `rules` addition: the
  835. // effective rule set (task ∪ primary ∪ co-developed, deduped,
  836. // stable order — see RuleService.effective_rules). Goal work
  837. // expands co-developed through the goal's links.
  838. payload.set_member_name("rules");
  839. payload.begin_array();
  840. foreach (var rule in yield rules.effective_rules_for(task, projects,
  841. goal != null ? task.goal_id : 0)) {
  842. payload.begin_object();
  843. payload.set_member_name("name");
  844. payload.add_string_value(rule.name);
  845. payload.set_member_name("body");
  846. payload.add_string_value(rule.body);
  847. payload.end_object();
  848. }
  849. payload.end_array();
  850. // The wire contract's `git_identity` addition (plan 9): the
  851. // name/email the daemon commits as, from the saddle row.
  852. // Omitted entirely when unset so the daemon's fallback
  853. // identity applies.
  854. string identity_name = saddle.git_name ?? "";
  855. string identity_email = saddle.git_email ?? "";
  856. if (identity_name.length > 0 || identity_email.length > 0) {
  857. payload.set_member_name("git_identity");
  858. payload.begin_object();
  859. payload.set_member_name("name");
  860. payload.add_string_value(identity_name);
  861. payload.set_member_name("email");
  862. payload.add_string_value(identity_email);
  863. payload.end_object();
  864. }
  865. // The wire contract's `capabilities` context (plan 10
  866. // phase 3): the allocated saddle's own tags plus every tag
  867. // name known across the fleet (the operator-managed
  868. // catalogue) — the daemon renders both into its system
  869. // prompt and validates `task create --capability`.
  870. payload.set_member_name("capabilities");
  871. payload.begin_object();
  872. payload.set_member_name("saddle_tags");
  873. payload.begin_array();
  874. foreach (var carried in yield capability_service.list_for_saddle(saddle.id)) {
  875. payload.add_string_value(carried.name);
  876. }
  877. payload.end_array();
  878. payload.set_member_name("fleet_tags");
  879. payload.begin_array();
  880. foreach (var known in yield capability_service.list_capabilities()) {
  881. payload.add_string_value(known.name);
  882. }
  883. payload.end_array();
  884. payload.end_object();
  885. // The wire contract's `resume_notice` addition: on a
  886. // workspace-adopting start (a suspended parent's resume), the
  887. // daemon injects it as the next user turn instead of the
  888. // opening prompt.
  889. if (resume_notice != null && resume_notice.length > 0) {
  890. payload.set_member_name("resume_notice");
  891. payload.add_string_value(resume_notice);
  892. }
  893. // The wire's `branch_advanced` (resume pushes): whether the
  894. // shared branch moved while this task was parked — false or
  895. // absent means the daemon omits the pull instruction entirely.
  896. if (branch_advanced != null) {
  897. payload.set_member_name("branch_advanced");
  898. payload.add_boolean_value((!)branch_advanced);
  899. }
  900. // The wire's `context_update` (resume pushes): a short
  901. // markdown delta of the instruction context that changed
  902. // while the workspace was parked — the goal's plan
  903. // approved since the run's last round, rules created,
  904. // edited or attached, or new project notes. The daemon
  905. // appends it as a TURN (prompt-cache friendly) instead of
  906. // refreshing the system prompt; absent when nothing
  907. // changed.
  908. if (context_update != null && context_update.length > 0) {
  909. payload.set_member_name("context_update");
  910. payload.add_string_value(context_update);
  911. }
  912. // The wire contract's `project_notes` addition: notes of the
  913. // primary and co-developed projects, oldest first, capped at
  914. // 30 entries / 20 000 chars (head kept).
  915. var note_project_names = new Dictionary<string, string>();
  916. var project_notes_rows = new Vector<ClankerSaddle.Data.ProjectNote>();
  917. note_project_names.set(((!)project).id.to_string(), ((!)project).name);
  918. foreach (var note in yield projects.list_notes(((!)project).id)) {
  919. project_notes_rows.add(note);
  920. }
  921. foreach (var linked in co_developed) {
  922. note_project_names.set(linked.id.to_string(), linked.name);
  923. foreach (var note in yield projects.list_notes(linked.id)) {
  924. project_notes_rows.add(note);
  925. }
  926. }
  927. var project_notes = project_notes_rows.sort((a, b) => (int)(a.id - b.id));
  928. payload.set_member_name("project_notes");
  929. payload.begin_array();
  930. int64 note_entries = 0;
  931. int64 note_chars = 0;
  932. foreach (var note in project_notes) {
  933. if (note_entries >= PROJECT_NOTES_MAX_ENTRIES
  934. || note_chars >= PROJECT_NOTES_MAX_CHARS) {
  935. break;
  936. }
  937. string note_body = note.body;
  938. if (note_chars + note_body.length > PROJECT_NOTES_MAX_CHARS) {
  939. note_body = note_body.substring(0,
  940. (int)(PROJECT_NOTES_MAX_CHARS - note_chars));
  941. }
  942. payload.begin_object();
  943. payload.set_member_name("project");
  944. payload.add_string_value(note_project_names.get_or_default(note.project_id.to_string()) ?? "");
  945. payload.set_member_name("body");
  946. payload.add_string_value(note_body);
  947. payload.end_object();
  948. note_entries++;
  949. note_chars += note_body.length;
  950. }
  951. payload.end_array();
  952. // The wire contract's `callback` addition: this app's
  953. // externally reachable base URL plus the bearer token the
  954. // daemon presents when posting back — its own per-saddle
  955. // credential (plan 10), which every `/api/saddle/*` call
  956. // carries.
  957. string callback_token = saddle.token ?? "";
  958. if (callback_token.length == 0) {
  959. string? legacy = yield token_for_saddle(saddle.id);
  960. callback_token = legacy ?? "";
  961. }
  962. payload.set_member_name("callback");
  963. payload.begin_object();
  964. payload.set_member_name("base_url");
  965. payload.add_string_value(callback_base_url());
  966. payload.set_member_name("token");
  967. payload.add_string_value(callback_token);
  968. payload.end_object();
  969. payload.set_member_name("llm_config");
  970. payload.begin_object();
  971. payload.set_member_name("base_url");
  972. payload.add_string_value(((!)endpoint).base_url);
  973. payload.set_member_name("api_key");
  974. payload.add_string_value(((!)endpoint).api_key);
  975. payload.set_member_name("model");
  976. payload.add_string_value(((!)endpoint).model);
  977. payload.end_object();
  978. // The web-side run row the daemon's pushed telemetry keys on
  979. // (plan-10 phase-1 wire addition: it rides every assignment
  980. // so /start consumers can ignore it harmlessly).
  981. payload.set_member_name("session_id");
  982. payload.add_int_value(session_id);
  983. payload.end_object();
  984. var root = payload.get_root().copy();
  985. return root.get_object();
  986. }
  987. /**
  988. * Probes a saddle's `GET /status` and refreshes its registered
  989. * status: 'idle' or 'busy' from the daemon state, 'offline' when
  990. * the daemon cannot be reached. The write is guarded on the
  991. * assignment seen at load — an ask allocation may claim the
  992. * saddle while the probe is in flight, and must not be stomped.
  993. */
  994. public async void refresh_saddle_status(int64 id) throws Error {
  995. var saddle = yield get_saddle(id);
  996. if (saddle == null) {
  997. throw new Error(Quark.from_string("clanker-saddle"), 17,
  998. @"No saddle exists with id $(id)");
  999. }
  1000. int64 seen_task = ((!)saddle).current_task_id;
  1001. string next_status = "idle";
  1002. int64 next_task = 0;
  1003. DateTime? next_allocated = null;
  1004. try {
  1005. var status = yield request_saddle_json(((!)saddle).host, "GET", "/status", null);
  1006. string state = status.has_member("state") ? status.get_string_member_with_default("state", "") : "";
  1007. yield record_status_probe(((!)saddle), status);
  1008. if (state == "working") {
  1009. // Reconcile with the web's own records: busy only when
  1010. // a live session truly holds this saddle.
  1011. var session = yield open_session_on_saddle(id);
  1012. next_status = "busy";
  1013. next_task = session != null ? ((!)session).task_id : 0;
  1014. next_allocated = session != null ? ((!)session).started_at : null;
  1015. }
  1016. // Idle AND the parked states (blocked/suspended, plus
  1017. // terminal and unknown states): the workspace is
  1018. // parked, so the saddle is logically free — marking it
  1019. // busy here used to wedge assignment against a daemon
  1020. // that would never poll.
  1021. } catch (Error e) {
  1022. next_status = "offline";
  1023. }
  1024. var command = connection.create_command(
  1025. "UPDATE saddles SET status = :status, current_task_id = :task,"
  1026. + " allocated_at = :allocated WHERE id = :id AND current_task_id = :seen");
  1027. command.with_parameter<Invercargill.Element>("status", elem(next_status));
  1028. command.with_parameter<Invercargill.Element>("task", elem<int64?>(next_task));
  1029. command.with_parameter<Invercargill.Element>("allocated", elem<DateTime?>(next_allocated));
  1030. command.with_parameter<Invercargill.Element>("id", elem<int64?>(id));
  1031. command.with_parameter<Invercargill.Element>("seen", elem<int64?>(seen_task));
  1032. yield command.execute_non_query_async();
  1033. yield pulse.bump(0, "saddle status refreshed");
  1034. }
  1035. /**
  1036. * The saddle's open (live or parked) session, or null — the
  1037. * reconciliation source for whether a daemon is genuinely
  1038. * running one of our tasks.
  1039. */
  1040. private async ClankerSaddle.Data.Session? open_session_on_saddle(int64 saddle_id) throws Error {
  1041. var sessions = yield db.query<ClankerSaddle.Data.Session>()
  1042. .where(expr("saddle_id == $0", elem<int64?>(saddle_id)))
  1043. .order_by(expr("id"))
  1044. .materialise_async();
  1045. ClankerSaddle.Data.Session? newest_open = null;
  1046. foreach (var session in sessions) {
  1047. if (session.ended_at == null) {
  1048. newest_open = session;
  1049. }
  1050. }
  1051. return newest_open;
  1052. }
  1053. /**
  1054. * Answered blocks of the saddle's current task not yet handed over
  1055. * the poll endpoint; marks them delivered.
  1056. */
  1057. public async ImmutableLot<PendingAnswer> take_pending_answers(int64 saddle_id) throws Error {
  1058. var saddle = yield get_saddle(saddle_id);
  1059. if (saddle == null || ((!)saddle).current_task_id == 0) {
  1060. return new Vector<PendingAnswer>().to_immutable_buffer();
  1061. }
  1062. return yield take_pending_answers_for_task(((!)saddle).current_task_id);
  1063. }
  1064. /**
  1065. * Answered blocks of `task_id` not yet handed over anywhere;
  1066. * marks them delivered. Used both by the poll endpoint and by
  1067. * {@link push_task_to_saddle} (a resumed assignment carries its
  1068. * pending answers in the payload).
  1069. */
  1070. public async ImmutableLot<PendingAnswer> take_pending_answers_for_task(int64 task_id) throws Error {
  1071. var blocks = yield db.query<TaskBlock>()
  1072. .where(expr("task_id == $0", elem<int64?>(task_id)))
  1073. .materialise_async();
  1074. var rows = new Vector<PendingAnswer>();
  1075. foreach (var block in blocks) {
  1076. if (block.answer == null) {
  1077. continue;
  1078. }
  1079. if (!poll_state.mark_delivered(block.id)) {
  1080. continue;
  1081. }
  1082. rows.add(new PendingAnswer() {
  1083. block_id = block.id,
  1084. answer = raw_answer_for_saddle(((!)block).answer ?? "")
  1085. });
  1086. }
  1087. return rows.to_immutable_buffer();
  1088. }
  1089. /**
  1090. * Human comments of the saddle's current task not yet delivered to
  1091. * the daemon; marks them delivered (the poll response hands them
  1092. * over exactly once).
  1093. */
  1094. public async ImmutableLot<PendingComment> take_pending_comments(int64 saddle_id) throws Error {
  1095. var saddle = yield get_saddle(saddle_id);
  1096. if (saddle == null || ((!)saddle).current_task_id == 0) {
  1097. return new Vector<PendingComment>().to_immutable_buffer();
  1098. }
  1099. var comments = yield db.query<TaskComment>()
  1100. .where(expr("task_id == $0", elem<int64?>(((!)saddle).current_task_id)))
  1101. .order_by(expr("id"))
  1102. .materialise_async();
  1103. var rows = new Vector<PendingComment>();
  1104. foreach (var comment in comments) {
  1105. if (!comment.author.has_prefix("user:")
  1106. || poll_state.is_comment_delivered(comment.id)) {
  1107. continue;
  1108. }
  1109. poll_state.mark_comment_delivered(comment.id);
  1110. rows.add(new PendingComment() {
  1111. comment_id = comment.id,
  1112. author = comment.author,
  1113. body = comment.body,
  1114. quote = comment.quote ?? ""
  1115. });
  1116. }
  1117. return rows.to_immutable_buffer();
  1118. }
  1119. /**
  1120. * Pushes the task's undelivered human comments to the RUNNING
  1121. * daemon's `POST /task-update {kind: "comment"}` endpoint so
  1122. * they land in the live conversation. Best effort: a daemon
  1123. * without the route (HTTP 404/405) keeps the comments pending,
  1124. * and other failures are logged and retried on the next
  1125. * dispatcher tick.
  1126. */
  1127. public async void deliver_pending_comments(int64 saddle_id, int64 task_id) throws Error {
  1128. var saddle = yield get_saddle(saddle_id);
  1129. if (saddle == null) {
  1130. return;
  1131. }
  1132. var comments = yield db.query<TaskComment>()
  1133. .where(expr("task_id == $0", elem<int64?>(task_id)))
  1134. .order_by(expr("id"))
  1135. .materialise_async();
  1136. foreach (var comment in comments) {
  1137. if (!comment.author.has_prefix("user:")
  1138. || poll_state.is_comment_delivered(comment.id)) {
  1139. continue;
  1140. }
  1141. var payload = new Json.Builder();
  1142. payload.begin_object();
  1143. payload.set_member_name("kind");
  1144. payload.add_string_value("comment");
  1145. payload.set_member_name("text");
  1146. payload.add_string_value(comment.body);
  1147. payload.set_member_name("quote");
  1148. payload.add_string_value(comment.quote ?? "");
  1149. payload.end_object();
  1150. Json.Node? root = null;
  1151. int status_code = yield request_saddle(((!)saddle).host, "POST",
  1152. "/task-update", (owned) payload, out root);
  1153. if (status_code == 200 || status_code == 204) {
  1154. poll_state.mark_comment_delivered(comment.id);
  1155. } else if (status_code != 404 && status_code != 405) {
  1156. warning(@"[ClankerSaddle] Comment task-update to saddle $(saddle_id) returned HTTP $(status_code)");
  1157. }
  1158. }
  1159. }
  1160. /**
  1161. * Marks the task's human comments as delivered — they were handed
  1162. * over inside an assignment payload, so neither the poll endpoint
  1163. * nor `/notify` should repeat them.
  1164. */
  1165. public async void mark_task_comments_delivered(int64 task_id) throws Error {
  1166. var comments = yield db.query<TaskComment>()
  1167. .where(expr("task_id == $0", elem<int64?>(task_id)))
  1168. .materialise_async();
  1169. foreach (var comment in comments) {
  1170. if (comment.author.has_prefix("user:")) {
  1171. poll_state.mark_comment_delivered(comment.id);
  1172. }
  1173. }
  1174. }
  1175. /**
  1176. * Projects the agent's conversation-level events onto the task:
  1177. * `saddle comment` lines become agent comments, `saddle plan`
  1178. * lines update the plan, `saddle progress-update` lines append
  1179. * progress, `saddle review` raises the review banner, the
  1180. * `saddle done` entry completes the task with the summary the
  1181. * agent gave, and the `saddle failure` entry fails it with the
  1182. * explanation the agent piped in. `saddle yeehaw` entries are
  1183. * handled explicitly but project nothing — the stored
  1184. * `session_commands` row IS the celebration the card, badge
  1185. * and timeline read back. Returns true once a done or failed
  1186. * event finished the task.
  1187. *
  1188. * Plan 12: on a GOAL-RUN session the plan/done/failure verbs
  1189. * are goal-context — a plan submission enters plan review, a
  1190. * done submits the completion report (the goal-run task parks
  1191. * instead of completing), a failure fails the goal — and
  1192. * comments also land on the goal's thread.
  1193. *
  1194. * Plan 13 B2: the projections key on the DEDUPED entry stream
  1195. * ({@link store_pushed_entries} claims every id once per
  1196. * saddle, ever) and stop at terminal state — a task already
  1197. * done/failed, a goal already finished, projects nothing
  1198. * further. The live failure's 115-event/187-comment fan-out is
  1199. * structurally impossible: replays never reach this function,
  1200. * and the honest burst of DISTINCT failure ids dies at the
  1201. * first one.
  1202. */
  1203. public async bool apply_entry_events(int64 task_id, Vector<Json.Object> entries,
  1204. TaskService tasks) throws Error {
  1205. var goal_id = yield goal_id_of_run(task_id);
  1206. bool terminal = yield task_is_terminal(task_id);
  1207. if (!terminal && goal_id > 0) {
  1208. terminal = yield goal_is_terminal(goal_id);
  1209. }
  1210. if (terminal) {
  1211. return false;
  1212. }
  1213. foreach (var entry in entries) {
  1214. string command = string_member(entry, "command");
  1215. string output = string_member(entry, "output");
  1216. switch (string_member(entry, "type")) {
  1217. case "comment":
  1218. // Plan 13 B4: body-first — the entry's output IS
  1219. // the comment body; the command echo (the
  1220. // `saddle comment` invocation, heredoc shapes
  1221. // included) must never become the projected
  1222. // text. The parsed command stays as a legacy
  1223. // fallback for pushes without output.
  1224. string body = output;
  1225. if (body.length == 0) {
  1226. body = strip_quotes(strip_verb(command, "saddle comment"));
  1227. }
  1228. if (body.length > 0) {
  1229. yield tasks.add_comment(task_id, "agent", body);
  1230. if (goal_id > 0) {
  1231. yield goal_service.add_comment(goal_id, "agent", body);
  1232. yield goal_service.record_event(goal_id, "comment",
  1233. snippet_text(body));
  1234. }
  1235. }
  1236. break;
  1237. case "plan":
  1238. // Plan 12: plans are goal-context only — a
  1239. // goal-run's `saddle plan` enters plan review;
  1240. // plain task runs carry no plan member.
  1241. if (goal_id > 0) {
  1242. string markdown = output.length > 0 ? output
  1243. : strip_quotes(strip_verb(command, "saddle plan"));
  1244. if (markdown.length > 0) {
  1245. yield goal_service.submit_plan(goal_id, markdown);
  1246. }
  1247. }
  1248. break;
  1249. case "progress":
  1250. string summary = strip_verb(command, "saddle progress-update");
  1251. if (summary.length == 0) {
  1252. summary = output;
  1253. }
  1254. if (summary.length > 0) {
  1255. yield tasks.add_progress(task_id, strip_quotes(summary));
  1256. }
  1257. break;
  1258. case "review":
  1259. yield tasks.request_review(task_id,
  1260. output.length > 0 ? output
  1261. : strip_quotes(strip_verb(command, "saddle review")));
  1262. break;
  1263. case "done":
  1264. if (goal_id > 0) {
  1265. string report = output.length > 0 ? output
  1266. : strip_quotes(strip_verb(command, "saddle done"));
  1267. yield goal_service.submit_completion(goal_id,
  1268. report.length > 0 ? report : "The goal agent submitted its completion report");
  1269. return true;
  1270. }
  1271. {
  1272. // the DONE entry's output IS the report
  1273. // (the command is just "saddle done") —
  1274. // falling back to it keeps the summary when
  1275. // this entry path beats the transition push
  1276. string report = output.length > 0 ? output
  1277. : strip_quotes(strip_verb(command, "saddle done"));
  1278. yield tasks.complete_task(task_id,
  1279. report.length > 0 ? report : null, null, "done");
  1280. }
  1281. return true;
  1282. case "failed":
  1283. string explanation = output.length > 0 ? output
  1284. : strip_quotes(strip_verb(command, "saddle failure"));
  1285. if (goal_id > 0) {
  1286. yield goal_service.fail_goal(goal_id,
  1287. explanation.length > 0 ? explanation : null);
  1288. return true;
  1289. }
  1290. yield tasks.fail_task(task_id, explanation);
  1291. return true;
  1292. case "yeehaw":
  1293. // The celebration needs no task-side projection:
  1294. // store_pushed_entries already persisted the row
  1295. // the yeehaw card, badge and timeline read from.
  1296. break;
  1297. default:
  1298. break;
  1299. }
  1300. }
  1301. return false;
  1302. }
  1303. /**
  1304. * The goal id when `task_id` is a goal agent's run task (kind
  1305. * 'goal_run'), else 0 — the discriminator for the goal-context
  1306. * verb handling above.
  1307. */
  1308. private async int64 goal_id_of_run(int64 task_id) throws Error {
  1309. var task = yield db.query<TaskEntity>()
  1310. .where(expr("id == $0", elem<int64?>(task_id)))
  1311. .first_async();
  1312. return task != null && ((!)task).kind == TASK_KIND_GOAL_RUN
  1313. ? ((!)task).goal_id : 0;
  1314. }
  1315. /** A one-line snippet of a body (narrative event text). */
  1316. private static string snippet_text(string body) {
  1317. string flat = body.replace("\n", " ").strip();
  1318. return flat.length > 160 ? flat.substring(0, 160) + "…" : flat;
  1319. }
  1320. /**
  1321. * Raises the daemon's pushed blocking prompt as a `task_blocks`
  1322. * row exactly once: a not-yet-answered row, or an answered row
  1323. * whose answer has not been delivered to the daemon yet, means the
  1324. * prompt is already represented. The results push's `block`
  1325. * object carries the question plus the block type, options and
  1326. * markdown body (the `/status` field names — `block_question`
  1327. * &c — are accepted as a fallback). `saddle block rfc` carries
  1328. * no question text, so rfc prompts surface under a stable
  1329. * fallback title.
  1330. */
  1331. private async void ensure_block(TaskService tasks, int64 task_id, Json.Object block) throws Error {
  1332. string question = block_question_from_payload(block);
  1333. if (question.length == 0) {
  1334. return;
  1335. }
  1336. foreach (var existing in yield tasks.list_blocks(task_id)) {
  1337. if (existing.question != question) {
  1338. continue;
  1339. }
  1340. if (existing.answer == null || !poll_state.is_delivered(existing.id)) {
  1341. return;
  1342. }
  1343. }
  1344. yield tasks.add_block(task_id, question,
  1345. block_string_or(block, "type", "block_type"),
  1346. string_array_member(block, block.has_member("options") ? "options" : "block_options"),
  1347. optional_string_member(block, block.has_member("body") ? "body" : "block_body"));
  1348. }
  1349. /** The first non-empty of two string members (empty when both are). */
  1350. private static string block_string_or(Json.Object obj, string primary, string fallback) {
  1351. string value = string_member(obj, primary);
  1352. return value.length > 0 ? value : string_member(obj, fallback);
  1353. }
  1354. /**
  1355. * The question a daemon block is recognised by: `question`
  1356. * (or the `/status`-style `block_question`) verbatim, or the
  1357. * rfc fallback when the daemon raised an RFC (the
  1358. * `saddle block rfc` shape carries its document as the body and
  1359. * no question operand). Both the block-raising and answer-
  1360. * delivery paths key blocks by this string.
  1361. */
  1362. private static string block_question_from_payload(Json.Object block) {
  1363. string question = block_string_or(block, "question", "block_question");
  1364. if (question.length > 0) {
  1365. return question;
  1366. }
  1367. return block_string_or(block, "type", "block_type") == "rfc"
  1368. ? "Request for comments" : "";
  1369. }
  1370. /**
  1371. * Stores a pushed batch of raw PTY chunks (`POST
  1372. * /api/saddle/chunks`, plan 10 phase 2) as the open session's
  1373. * `session_raw_log` rows — the same storage logic the deleted
  1374. * `GET /log` poller ran, driven by the daemon's flusher instead
  1375. * of the web's cursor. Idempotent on the unique
  1376. * (session_id, seq) pair: a chunk whose row already exists (an
  1377. * outbox redelivery after a lost response) is consumed, not an
  1378. * error; anything else stops the pass so the daemon's retry
  1379. * re-sends the tail. A truly empty text stores no row (it
  1380. * renders as nothing); whitespace-only chunks DO store. Returns
  1381. * the count of rows newly stored.
  1382. */
  1383. public async int64 store_pushed_chunks(int64 session_id, Json.Array chunks) throws Error {
  1384. int64 stored = 0;
  1385. foreach (var element in chunks.get_elements()) {
  1386. if (element.get_node_type() != Json.NodeType.OBJECT) {
  1387. continue;
  1388. }
  1389. var chunk = element.get_object();
  1390. int64 seq = chunk.has_member("seq")
  1391. && chunk.get_member("seq").get_value_type() == typeof(int64)
  1392. ? chunk.get_int_member("seq") : 0;
  1393. if (seq <= 0) {
  1394. continue;
  1395. }
  1396. string text = string_member(chunk, "text");
  1397. if (text.length == 0) {
  1398. continue;
  1399. }
  1400. try {
  1401. yield db.insert_async<SessionRawLog>(new SessionRawLog() {
  1402. session_id = session_id,
  1403. seq = seq,
  1404. chunk = text
  1405. });
  1406. stored++;
  1407. } catch (Error insert_error) {
  1408. // The unique (session_id, seq) pair is the idempotency
  1409. // guarantee: a row already there means the push is a
  1410. // redelivery — consume it. Any other failure is real.
  1411. var existing = yield db.query<SessionRawLog>()
  1412. .where(expr("session_id == $0 && seq == $1",
  1413. elem<int64?>(session_id), elem<int64?>(seq)))
  1414. .first_async();
  1415. if (existing == null) {
  1416. throw insert_error;
  1417. }
  1418. }
  1419. }
  1420. return stored;
  1421. }
  1422. /**
  1423. * Stores a pushed batch of round entries (`POST
  1424. * /api/saddle/results`, plan 10 phase 2) as `session_commands`
  1425. * rows — the mapping the deleted `GET /output` poller ran. Plan
  1426. * 13 B1: the dedup key is the DAEMON-GLOBAL (saddle, entry id)
  1427. * pair ({@link SaddleEntryDedup}), persisted in the database so
  1428. * it survives web restarts and ignores session churn entirely —
  1429. * a redelivered entry is consumed silently under whatever
  1430. * session id the payload carries, and a push may only CREATE
  1431. * rows for entries whose ids have not already landed on this
  1432. * saddle. Entries without an id always store. Returns the
  1433. * entries stored, oldest first, for {@link apply_entry_events}
  1434. * — the deduped entry stream every projection keys on (B2).
  1435. */
  1436. public async Vector<Json.Object> store_pushed_entries(Saddle saddle, int64 session_id,
  1437. Json.Array entries) throws Error {
  1438. var stored = new Vector<Json.Object>();
  1439. foreach (var element in entries.get_elements()) {
  1440. if (element.get_node_type() != Json.NodeType.OBJECT) {
  1441. continue;
  1442. }
  1443. var entry = element.get_object();
  1444. int64 entry_id = entry.has_member("id")
  1445. && entry.get_member("id").get_value_type() == typeof(int64)
  1446. ? entry.get_int_member("id") : 0;
  1447. if (entry_id > 0 && !(yield claim_entry_id(saddle.id, entry_id))) {
  1448. // A redelivery under any session binding — consumed,
  1449. // never duplicated, never projected.
  1450. continue;
  1451. }
  1452. var command = new SessionCommand() {
  1453. session_id = session_id,
  1454. command_text = string_member(entry, "command"),
  1455. output_text = entry.has_member("output")
  1456. && entry.get_member("output").get_string() != null
  1457. ? entry.get_string_member("output") : null,
  1458. exit_code = entry.has_member("exit_code")
  1459. && entry.get_member("exit_code").get_value_type() == typeof(int64)
  1460. ? entry.get_int_member("exit_code") : 0,
  1461. entry_type = string_member(entry, "type").length > 0
  1462. ? string_member(entry, "type") : "command",
  1463. piped_input = optional_string_member(entry, "input"),
  1464. started_at = parse_iso(entry, "started_at") ?? parse_iso(entry, "timestamp"),
  1465. completed_at = parse_iso(entry, "completed_at"),
  1466. duration_ms = entry.has_member("duration_ms")
  1467. && entry.get_member("duration_ms").get_value_type() == typeof(int64)
  1468. ? entry.get_int_member("duration_ms") : 0
  1469. };
  1470. yield db.insert_async<SessionCommand>(command);
  1471. stored.add(entry);
  1472. }
  1473. return stored;
  1474. }
  1475. /**
  1476. * Claims `(saddle_id, entry_id)` for ingestion: true when the
  1477. * id is fresh on this saddle, false when it already landed (an
  1478. * outbox redelivery — consumed silently regardless of the
  1479. * payload's session binding). The dedup row is inserted BEFORE
  1480. * the entry stores, so a mid-pass failure can never re-project
  1481. * an entry the daemon retries (exactly-once projections beat
  1482. * at-least-once storage).
  1483. */
  1484. private async bool claim_entry_id(int64 saddle_id, int64 entry_id) throws Error {
  1485. try {
  1486. yield db.insert_async<SaddleEntryDedup>(new SaddleEntryDedup() {
  1487. saddle_id = saddle_id,
  1488. entry_id = entry_id
  1489. });
  1490. return true;
  1491. } catch (Error insert_error) {
  1492. var existing = yield db.query<SaddleEntryDedup>()
  1493. .where(expr("saddle_id == $0 && entry_id == $1",
  1494. elem<int64?>(saddle_id), elem<int64?>(entry_id)))
  1495. .first_async();
  1496. if (existing == null) {
  1497. throw insert_error;
  1498. }
  1499. return false;
  1500. }
  1501. }
  1502. /**
  1503. * Applies a pushed state transition (`POST /api/saddle/results`,
  1504. * plan 10 phase 2) — the same task/saddle bookkeeping the
  1505. * deleted /status poll switch ran, driven by the daemon's push:
  1506. * `done`/`failed` complete or fail the task with the pushed
  1507. * summary (freeing the saddle and closing the session) —
  1508. * except on a goal-run session, where they submit the
  1509. * completion report or fail the goal (plan 12); `blocked`
  1510. * raises the block payload as an answerable block on the
  1511. * goal run's OWN task row (plan 15 C1: the goal agent's
  1512. * blocks ride the run task — no mirrored second row exists)
  1513. * and parks the saddle; `suspended`
  1514. * parks the same way — for a goal agent the resume is driven
  1515. * by the goal's task-terminal events; and `working` flips the
  1516. * task to working (never downgrading a pending review).
  1517. *
  1518. * Plan 13 B2: a push against a task that is already terminal —
  1519. * or a goal-context push against a finished goal — is an
  1520. * outbox redelivery and is consumed silently, whatever state
  1521. * it carries; terminal transitions and their narrative fire
  1522. * once, ever.
  1523. */
  1524. public async void apply_result_state(Saddle saddle, ClankerSaddle.Data.Session session,
  1525. string state, Json.Object? block, string? summary, TaskService tasks) throws Error {
  1526. int64 task_id = session.task_id;
  1527. int64 goal_id = yield goal_id_of_run(task_id);
  1528. bool terminal = yield task_is_terminal(task_id);
  1529. if (!terminal && goal_id > 0) {
  1530. terminal = yield goal_is_terminal(goal_id);
  1531. }
  1532. if (terminal) {
  1533. return;
  1534. }
  1535. switch (state) {
  1536. case "done":
  1537. if (goal_id > 0) {
  1538. // Goal context: `saddle done` submits the
  1539. // completion report and enters completion
  1540. // review — the run parks on the mirrored
  1541. // block, it does not complete.
  1542. yield goal_service.submit_completion(goal_id,
  1543. summary != null && summary.length > 0 ? summary
  1544. : "The goal agent submitted its completion report");
  1545. yield park_if_current(saddle.id, task_id);
  1546. break;
  1547. }
  1548. yield tasks.complete_task(task_id, summary, null, "done");
  1549. break;
  1550. case "failed":
  1551. string explanation = summary ?? "";
  1552. if (goal_id > 0) {
  1553. yield goal_service.fail_goal(goal_id,
  1554. explanation.length > 0 ? explanation : null);
  1555. break;
  1556. }
  1557. yield tasks.fail_task(task_id, explanation.length > 0 ? explanation : null);
  1558. break;
  1559. case "blocked":
  1560. if (goal_id > 0) {
  1561. // Goal context: the block lands on the run
  1562. // task itself (plan 15 C1) — a plan payload
  1563. // enters plan review, anything else raises a
  1564. // goal question/RFC; the run parks on that
  1565. // very row for the verdict resume. A BLOCKED
  1566. // push WITHOUT a
  1567. // block payload (the park-unwinding round-end
  1568. // push, its trailing entries) raises nothing —
  1569. // the payload-carrying transition already did,
  1570. // and a synthesised fallback row would dupe it.
  1571. if (block == null) {
  1572. yield tasks.set_status(task_id, STATUS_BLOCKED);
  1573. yield park_if_current(saddle.id, task_id);
  1574. break;
  1575. }
  1576. string question = block_question_from_payload((!)block);
  1577. if (question.length == 0) {
  1578. question = "The goal agent raised a question";
  1579. }
  1580. if (block_string_or((!)block, "type", "block_type") == BLOCK_PLAN) {
  1581. yield goal_service.submit_plan(goal_id,
  1582. optional_string_member((!)block, "body") ?? question);
  1583. } else {
  1584. yield goal_service.raise_agent_block(goal_id, question,
  1585. block_string_or((!)block, "type", "block_type"),
  1586. string_array_member((!)block,
  1587. (!)block.has_member("options") ? "options" : "block_options"),
  1588. optional_string_member((!)block, "body"));
  1589. }
  1590. yield tasks.set_status(task_id, STATUS_BLOCKED);
  1591. yield park_if_current(saddle.id, task_id);
  1592. break;
  1593. }
  1594. yield ensure_block(tasks, task_id, block != null ? (!)block : new Json.Object());
  1595. {
  1596. // A review-kind block parks as review_requested
  1597. // (the banner state); every other kind as
  1598. // blocked.
  1599. bool review = block != null
  1600. && block_string_or((!)block, "type", "block_type") == BLOCK_REVIEW;
  1601. yield tasks.set_status(task_id,
  1602. review ? STATUS_REVIEW_REQUESTED : STATUS_BLOCKED);
  1603. }
  1604. // The daemon parks the task's workspace while blocked,
  1605. // freeing the saddle for other work. The blocked task
  1606. // plus its open session is the pause record the
  1607. // affinity allocation resumes from once the block is
  1608. // answered.
  1609. yield park_if_current(saddle.id, task_id);
  1610. break;
  1611. case "suspended":
  1612. // `saddle suspend`: the same parking as a block, but
  1613. // the web drives resumption — a wake of the affine
  1614. // saddle once a goal event lands, the resume
  1615. // assignment carrying the coalesced report.
  1616. yield tasks.suspend_task(task_id);
  1617. yield park_if_current(saddle.id, task_id);
  1618. break;
  1619. case "working":
  1620. // A pending review keeps its banner: started_at (and
  1621. // the first-start push) only stamp for work that
  1622. // never allocated.
  1623. var current = yield tasks.set_status(task_id, STATUS_WORKING);
  1624. if (current != null && ((!)current).status != STATUS_REVIEW_REQUESTED
  1625. && ((!)current).started_at == null) {
  1626. ((!)current).started_at = new DateTime.now_local();
  1627. yield db.update_async<TaskEntity>((!)current);
  1628. // The first-start push lives at the allocation
  1629. // (where started_at is stamped for dispatched
  1630. // tasks); this path only covers legacy status
  1631. // pushes for never-allocated work.
  1632. push.notify_opted_in(task_id, PushService.lifecycle_notification(
  1633. ((!)current).title, task_id, "started", null));
  1634. }
  1635. break;
  1636. default:
  1637. break;
  1638. }
  1639. }
  1640. /** True when the task's status is done or failed. */
  1641. private async bool task_is_terminal(int64 task_id) throws Error {
  1642. var task = yield db.query<TaskEntity>()
  1643. .where(expr("id == $0", elem<int64?>(task_id)))
  1644. .first_async();
  1645. return task != null && (((!)task).status == STATUS_DONE
  1646. || ((!)task).status == STATUS_FAILED);
  1647. }
  1648. /** True when the goal's status is complete, failed or cancelled. */
  1649. private async bool goal_is_terminal(int64 goal_id) throws Error {
  1650. var goal = yield db.query<Goal>()
  1651. .where(expr("id == $0", elem<int64?>(goal_id)))
  1652. .first_async();
  1653. return goal != null && (((!)goal).status == GOAL_COMPLETE
  1654. || ((!)goal).status == GOAL_FAILED || ((!)goal).status == GOAL_CANCELLED);
  1655. }
  1656. /**
  1657. * The running-liveness probe (plan 10 phase 2): a busy saddle's
  1658. * session has seen neither chunks nor results for the staleness
  1659. * window, so ONE exceptional `GET /status` decides its fate —
  1660. * an answer means the daemon is alive (the health snapshot is
  1661. * refreshed and the task keeps running; the silence episode
  1662. * gets no second probe), a failure requeues the task with an agent
  1663. * comment and marks the saddle offline (the normal unreachable
  1664. * bookkeeping). The probe flag was taken by the caller; the
  1665. * next push resets it.
  1666. */
  1667. public async void probe_stale_saddle(Saddle saddle, TaskService tasks) throws Error {
  1668. try {
  1669. var status = yield request_saddle_json(saddle.host, "GET", "/status", null);
  1670. yield record_status_probe(saddle, status);
  1671. print(@"[ClankerSaddle] Stale saddle '$(saddle.name)' answered its exceptional status probe; task $(saddle.current_task_id) keeps running\n");
  1672. } catch (Error e) {
  1673. warning(@"[ClankerSaddle] Saddle '$(saddle.name)' sent neither chunks nor results for $(stale_after_seconds())s and its status probe failed ($(e.message)); marking offline");
  1674. yield mark_saddle_unreachable(saddle.id, tasks,
  1675. @"no chunks or results for $(stale_after_seconds())s; status probe failed ($(e.message))");
  1676. }
  1677. }
  1678. /**
  1679. * Performs an HTTP request against a saddle daemon and parses the
  1680. * response body as JSON; transport failures propagate to the
  1681. * caller (the dispatcher marks the saddle unreachable). Transport
  1682. * blips — typically a pooled keep-alive connection closed by the
  1683. * daemon at the same instant the request reused it — are retried
  1684. * once before giving up. Unlike {@link request_saddle_json} the
  1685. * HTTP status is returned rather than thrown, and array bodies
  1686. * parse too.
  1687. */
  1688. private async int request_saddle(string host, string method, string path,
  1689. owned Json.Builder? payload, out Json.Node? root) throws Error {
  1690. root = null;
  1691. GLib.Error? failure = null;
  1692. for (var attempt = 0; attempt < 2; attempt++) {
  1693. try {
  1694. root = null;
  1695. int status = yield request_saddle_once(host, method, path, payload, out root);
  1696. return status;
  1697. } catch (Error e) {
  1698. failure = e;
  1699. }
  1700. if (((!)failure).code != 13 && ((!)failure).code != 14) {
  1701. break;
  1702. }
  1703. GLib.Timeout.add(250, request_saddle.callback);
  1704. yield;
  1705. }
  1706. throw (!)failure;
  1707. }
  1708. private async int request_saddle_once(string host, string method, string path,
  1709. Json.Builder? payload, out Json.Node? root) throws Error {
  1710. root = null;
  1711. string base_url = host.has_prefix("http") ? host : @"http://$(host)";
  1712. var message = new Soup.Message(method, @"$(base_url)$(path)");
  1713. if (payload != null) {
  1714. var generator = new Json.Generator();
  1715. generator.set_root(((!)payload).get_root());
  1716. message.set_request_body_from_bytes("application/json",
  1717. new Bytes(generator.to_data(null).data));
  1718. }
  1719. InputStream input;
  1720. try {
  1721. input = yield http.send_async(message, Priority.DEFAULT, null);
  1722. } catch (Error e) {
  1723. throw new Error(Quark.from_string("clanker-saddle"), 13,
  1724. @"Saddle request to $(base_url)$(path) failed: $(e.message)");
  1725. }
  1726. var output = new MemoryOutputStream.resizable();
  1727. try {
  1728. yield output.splice_async(input, OutputStreamSpliceFlags.CLOSE_SOURCE,
  1729. Priority.DEFAULT, null);
  1730. yield output.close_async(Priority.DEFAULT, null);
  1731. } catch (Error e) {
  1732. throw new Error(Quark.from_string("clanker-saddle"), 14,
  1733. @"Failed reading saddle response: $(e.message)");
  1734. }
  1735. var written = output.get_data_size();
  1736. uint8[] data = output.steal_data();
  1737. // steal_data's array length is -1 and the buffer is sized to
  1738. // the data — there is no NUL terminator to rely on — so slice
  1739. // by the written byte count before reading it as a string.
  1740. string response_text = written > 0 && data != null && data[0] != '\0'
  1741. ? ((string) data).substring(0, (long) written) : "{}";
  1742. var parser = new Json.Parser();
  1743. try {
  1744. parser.load_from_data(response_text);
  1745. } catch (Error e) {
  1746. throw new Error(Quark.from_string("clanker-saddle"), 16,
  1747. @"Saddle returned invalid JSON: $(e.message)");
  1748. }
  1749. root = parser.get_root();
  1750. return (int) message.status_code;
  1751. }
  1752. /**
  1753. * Performs an HTTP request against a saddle daemon and returns the
  1754. * parsed JSON object body (an empty object for non-object bodies);
  1755. * HTTP statuses of 400 and above throw.
  1756. */
  1757. private async Json.Object request_saddle_json(string host, string method, string path,
  1758. owned Json.Builder? payload) throws Error {
  1759. Json.Node? root = null;
  1760. int status_code = yield request_saddle(host, method, path, (owned) payload, out root);
  1761. if (status_code == 409) {
  1762. // Machine-readable per the wire contract: the body (state,
  1763. // running_task_id) rides in the message so the dispatch
  1764. // path can distinguish zombie conflicts from transport
  1765. // failures and recover.
  1766. string body_text = "{}";
  1767. if (root != null && root.get_node_type() == Json.NodeType.OBJECT) {
  1768. var generator = new Json.Generator();
  1769. generator.set_root(root);
  1770. body_text = generator.to_data(null);
  1771. }
  1772. throw new Error(Quark.from_string("clanker-saddle"), 409,
  1773. @"busy: $(body_text)");
  1774. }
  1775. if (status_code >= 400) {
  1776. throw new Error(Quark.from_string("clanker-saddle"), 15,
  1777. @"Saddle request returned HTTP $(status_code)");
  1778. }
  1779. if (root == null || root.get_node_type() != Json.NodeType.OBJECT) {
  1780. return new Json.Object();
  1781. }
  1782. return root.get_object();
  1783. }
  1784. /**
  1785. * A `{"name", "git_url", "path", "branch"}` project reference in
  1786. * the saddle daemon's wire shape: `git_url` is the clone source,
  1787. * and the checkout lives at
  1788. * `SADDLE_PROJECTS_ROOT/task-<task_id>/<name>` inside the
  1789. * saddle's container — one workspace per task, so concurrent
  1790. * tasks never share a checkout.
  1791. */
  1792. private static Json.Node project_node(Project project, int64 task_id, bool is_reference = false) {
  1793. var root_var = (Environment.get_variable("SADDLE_PROJECTS_ROOT") ?? "").strip();
  1794. string projects_root = root_var.length > 0 ? root_var : "/workspace";
  1795. var builder = new Json.Builder();
  1796. builder.begin_object();
  1797. builder.set_member_name("name");
  1798. builder.add_string_value(project.name);
  1799. builder.set_member_name("git_url");
  1800. builder.add_string_value(project.git_url);
  1801. builder.set_member_name("path");
  1802. builder.add_string_value(is_reference
  1803. ? @"$(projects_root)/task-$(task_id)/reference/$(project.name)"
  1804. : @"$(projects_root)/task-$(task_id)/$(project.name)");
  1805. builder.set_member_name("branch");
  1806. builder.add_string_value(project.default_branch);
  1807. builder.end_object();
  1808. return builder.get_root();
  1809. }
  1810. /** `command` minus a leading `verb` and its separating whitespace. */
  1811. private static string strip_verb(string command, string verb) {
  1812. var trimmed = command.strip();
  1813. if (trimmed == verb) {
  1814. return "";
  1815. }
  1816. return trimmed.has_prefix(verb + " ") ? trimmed.substring(verb.length + 1).strip() : trimmed;
  1817. }
  1818. /**
  1819. * The answer as the saddle daemon expects it over /answer (or in a
  1820. * resume payload): verbatim. The daemon applies its own
  1821. * BlockRequest formatting, so this only strips the "Human
  1822. * selected: …"/"Human answered (other): …" prefixes still stored
  1823. * by answers recorded before the daemon took over formatting.
  1824. */
  1825. private static string raw_answer_for_saddle(string answer) {
  1826. if (answer.has_prefix("Human selected: ")) {
  1827. return answer.substring("Human selected: ".length);
  1828. }
  1829. if (answer.has_prefix("Human answered (other): ")) {
  1830. return answer.substring("Human answered (other): ".length);
  1831. }
  1832. return answer;
  1833. }
  1834. /** Strips one layer of matching double quotes, when present. */
  1835. private static string strip_quotes(string text) {
  1836. if (text.length >= 2 && text.has_prefix("\"") && text.has_suffix("\"")) {
  1837. return text.substring(1, text.length - 2);
  1838. }
  1839. return text;
  1840. }
  1841. private static string string_member(Json.Object obj, string name) {
  1842. return obj.has_member(name) && obj.get_member(name).get_string() != null
  1843. ? obj.get_string_member(name) : "";
  1844. }
  1845. /** A nullable string member: null when absent or empty. */
  1846. private static string? optional_string_member(Json.Object obj, string name) {
  1847. string value = string_member(obj, name);
  1848. return value.length > 0 ? value : null;
  1849. }
  1850. /**
  1851. * A string-array member: native JSON arrays are read element-wise;
  1852. * a JSON-encoded string is parsed; anything else is empty.
  1853. */
  1854. private static string[] string_array_member(Json.Object obj, string name) {
  1855. if (!obj.has_member(name)) {
  1856. return new string[0];
  1857. }
  1858. var member = obj.get_member(name);
  1859. if (member.get_node_type() == Json.NodeType.ARRAY) {
  1860. var values = new Vector<string>();
  1861. foreach (var element in member.get_array().get_elements()) {
  1862. if (element.get_node_type() == Json.NodeType.VALUE
  1863. && element.get_string() != null) {
  1864. values.add(element.get_string());
  1865. }
  1866. }
  1867. return values.to_array();
  1868. }
  1869. string raw = string_member(obj, name);
  1870. if (raw.length == 0) {
  1871. return new string[0];
  1872. }
  1873. try {
  1874. var parser = new Json.Parser();
  1875. parser.load_from_data(raw);
  1876. var root = parser.get_root();
  1877. if (root != null && root.get_node_type() == Json.NodeType.ARRAY) {
  1878. var values = new Vector<string>();
  1879. foreach (var element in root.get_array().get_elements()) {
  1880. if (element.get_node_type() == Json.NodeType.VALUE
  1881. && element.get_string() != null) {
  1882. values.add(element.get_string());
  1883. }
  1884. }
  1885. return values.to_array();
  1886. }
  1887. } catch (Error e) {
  1888. warning("[ClankerSaddle] Ignoring malformed options array: %s", e.message);
  1889. }
  1890. return new string[0];
  1891. }
  1892. private static DateTime? parse_iso(Json.Object obj, string name) {
  1893. if (!obj.has_member(name)) {
  1894. return null;
  1895. }
  1896. var value = obj.get_member(name);
  1897. if (value.get_node_type() != Json.NodeType.VALUE || value.get_string() == null) {
  1898. return null;
  1899. }
  1900. return new DateTime.from_iso8601(value.get_string(), new TimeZone.utc());
  1901. }
  1902. }
  1903. }