SaddleService.vala 89 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811
  1. using Invercargill;
  2. using Invercargill.DataStructures;
  3. using Invercargill.Expressions;
  4. using InvercargillSql;
  5. using InvercargillSql.Orm;
  6. using Inversion;
  7. using Soup;
  8. using ClankerSaddle.Data;
  9. namespace ClankerSaddle.Services {
  10. /**
  11. * Valid saddle statuses (the `saddles.status` wire values, plan 16
  12. * R2 / arch item 26 — the raw literals died with these). The row
  13. * default and the migration seed keep their literals (the Data
  14. * layer does not reference the services).
  15. */
  16. public const string SADDLE_IDLE = "idle";
  17. public const string SADDLE_BUSY = "busy";
  18. public const string SADDLE_OFFLINE = "offline";
  19. /**
  20. * Session outcomes (the `sessions.outcome` values the park, the
  21. * ONE release path and the terminal pushes write): `parked` (a
  22. * round that parked on a block or a suspend — the row closes at
  23. * park time and the workspace awaits its resuming allocation;
  24. * plan 16 D4, aligned with the daemon's file-log brackets),
  25. * `stopped` (a cancelled/requeued/unreachable release),
  26. * `done`/`failed` (the terminal pushes).
  27. */
  28. public const string OUTCOME_STOPPED = "stopped";
  29. public const string OUTCOME_PARKED = "parked";
  30. public const string OUTCOME_DONE = "done";
  31. public const string OUTCOME_FAILED = "failed";
  32. /** Caps of the `project_notes` array in the /start payload. */
  33. public const int64 PROJECT_NOTES_MAX_ENTRIES = 30;
  34. public const int64 PROJECT_NOTES_MAX_CHARS = 20000;
  35. /**
  36. * Cap of the `understanding` member in the /start payload: the
  37. * primary project's current understanding document (goal 24).
  38. * The whole document rides the payload and renders into the
  39. * system prompt next to the project notes, so an over-grown
  40. * revision is head-clamped rather than bloating every prompt.
  41. */
  42. public const int64 UNDERSTANDING_MAX_CHARS = 16000;
  43. /**
  44. * Upper bound of a saddle's prompt_note served to the daemon (goal 8
  45. * task 5): the note rides EVERY assignment payload, so a fat note
  46. * would bloat every system prompt and cache.
  47. */
  48. public const int64 PROMPT_NOTE_MAX_CHARS = 2000;
  49. /** A block answer awaiting collection by one saddle's next poll. */
  50. public class PendingAnswer : Object {
  51. public int64 block_id { get; set; }
  52. public string answer { get; set; default = ""; }
  53. public bool branch_advanced { get; set; }
  54. }
  55. /** A saddle registration plus the API token generated for it. */
  56. public class SaddleRegistration : Object {
  57. public Saddle saddle { get; set; }
  58. public string token { get; set; default = ""; }
  59. }
  60. /**
  61. * A saddle's current work as the cards render it: the running
  62. * task's title, plus the GOAL it stands for when that task is a
  63. * goal agent's internal run.
  64. */
  65. public class CurrentWork : Object {
  66. public int64 task_id { get; set; }
  67. public string task_title { get; set; default = ""; }
  68. /** Set when the current task is a goal agent's run: the card links the goal. */
  69. public int64 goal_id { get; set; }
  70. public string goal_title { get; set; default = ""; }
  71. /** The running session the derivation keyed on (plan 16 item 23). */
  72. public int64 session_id { get; set; }
  73. /** When that session started — the "allocated" display stamp. */
  74. public DateTime? started_at { get; set; default = null; }
  75. }
  76. /**
  77. * The OpenAI-compatible endpoint every saddle assignment is sent to,
  78. * read from the environment: `SADDLE_LLM_BASE_URL` (required — dispatch
  79. * pauses without it), `SADDLE_LLM_API_KEY` and `SADDLE_LLM_MODEL`.
  80. */
  81. public class LlmEndpoint : Object {
  82. public string base_url { get; set; default = ""; }
  83. public string api_key { get; set; default = ""; }
  84. public string model { get; set; default = ""; }
  85. /** The configured endpoint, or null when the base URL is unset. */
  86. public static LlmEndpoint? from_environment() {
  87. var base_url = (Environment.get_variable("SADDLE_LLM_BASE_URL") ?? "").strip();
  88. if (base_url.length == 0) {
  89. return null;
  90. }
  91. return new LlmEndpoint() {
  92. base_url = base_url,
  93. api_key = (Environment.get_variable("SADDLE_LLM_API_KEY") ?? "").strip(),
  94. model = (Environment.get_variable("SADDLE_LLM_MODEL") ?? "").strip()
  95. };
  96. }
  97. }
  98. /**
  99. * Saddle registry and control-plane client (plan 10 phase 2): registers
  100. * containers, derives idle capacity for the dispatcher's wake
  101. * computation (plan 16 item 23: busy iff an open session on an
  102. * assigned/working task exists — {@link current_work_view} is the ONE
  103. * truth accessor; the saddle row keeps only the OFFLINE probe state),
  104. * wakes sleeping daemons, and ingests the telemetry the daemons PUSH —
  105. * raw PTY chunks (`POST /api/saddle/chunks`) and round results with
  106. * state transitions (`POST /api/saddle/results`) — reusing the storage
  107. * logic the deleted `/output` + `/log` pollers ran. GET /status
  108. * against a daemon is exceptional only: the staleness probe for a
  109. * silent runner and the manual refresh button. Block answers are
  110. * handed out exactly once (delivered ids are remembered in memory; a
  111. * restart re-delivers and the daemon deduplicates).
  112. */
  113. public class SaddleService : Object {
  114. private OrmSession db = inject<OrmSession>();
  115. private Connection connection = inject<Connection>();
  116. /**
  117. * Every daemon request shares this session and therefore its
  118. * timeouts: without them a single hung request (a daemon wedged
  119. * mid-response over the keep-alive pool) froze the dispatcher's
  120. * tick latch forever. `timeout` bounds each request
  121. * (SADDLE_HTTP_TIMEOUT_S, default 30s); `idle_timeout` reaps
  122. * pooled connections so a peer that vanished between requests is
  123. * not dialed again.
  124. */
  125. private Soup.Session http = new Soup.Session() {
  126. timeout = http_timeout_seconds(),
  127. idle_timeout = 10
  128. };
  129. private static uint http_timeout_seconds() {
  130. var raw = Environment.get_variable("SADDLE_HTTP_TIMEOUT_S") ?? "";
  131. int64 parsed = int64.parse(raw);
  132. return parsed > 0 ? (uint) parsed : 30;
  133. }
  134. private SaddlePollState poll_state = inject<SaddlePollState>();
  135. private StatePulse pulse = inject<StatePulse>();
  136. private PushService push = inject<PushService>();
  137. private RuleService rules = inject<RuleService>();
  138. private CapabilityService capability_service = inject<CapabilityService>();
  139. private TaskService task_service = inject<TaskService>();
  140. private GoalService goal_service = inject<GoalService>();
  141. private EventService event_service = inject<EventService>();
  142. private UnderstandingService understandings = inject<UnderstandingService>();
  143. private Astralis.WebConfig config = inject<Astralis.WebConfig>();
  144. /**
  145. * Registers a saddle container: inserts the row with a freshly
  146. * minted per-saddle token (plan 10) and hands the credential to
  147. * the daemon via `POST /register {saddle_token, web_base_url}`
  148. * (best effort — an older daemon without the route stays
  149. * registered; the operator copies the token from the saddles
  150. * page instead). `host` is host:port of its control server; the
  151. * returned {@link SaddleRegistration} carries the token the
  152. * daemon must present as `Authorization: Bearer …` on
  153. * `/api/saddle/*` calls.
  154. */
  155. public async SaddleRegistration register_saddle(string name, string host) throws Error {
  156. var saddle = new Saddle() {
  157. name = name,
  158. host = host,
  159. status = SADDLE_IDLE,
  160. token = mint_saddle_token()
  161. };
  162. yield db.insert_async<Saddle>(saddle);
  163. yield register_daemon(saddle);
  164. yield pulse.bump(0, "saddle registered");
  165. return new SaddleRegistration() { saddle = saddle, token = saddle.token ?? "" };
  166. }
  167. /** A fresh per-saddle bearer token (`saddle-<random>`). */
  168. private static string mint_saddle_token() {
  169. return @"saddle-$(Uuid.string_random().replace("-", ""))";
  170. }
  171. /**
  172. * Delivers the per-saddle token to the daemon's
  173. * `POST /register` endpoint (wire contract): the daemon
  174. * persists it for its calls home and answers
  175. * `{ok, ssh_public_key, version}` — the key is stored on the
  176. * row the moment the daemon reports one. A daemon without the
  177. * route (HTTP 404/405) or unreachable at registration time is
  178. * tolerated: the row keeps the token, the operator can rety
  179. * via a refresh or copy the token manually.
  180. */
  181. public async void register_daemon(Saddle saddle) throws Error {
  182. var payload = new Json.Builder();
  183. payload.begin_object();
  184. payload.set_member_name("saddle_token");
  185. payload.add_string_value(saddle.token ?? "");
  186. payload.set_member_name("web_base_url");
  187. payload.add_string_value(callback_base_url());
  188. payload.end_object();
  189. try {
  190. var response = yield request_saddle_json(saddle.host, "POST", "/register",
  191. (owned) payload);
  192. string reported_key = EntryJson.string_member(response, "ssh_public_key");
  193. // goal 8 task 6: the daemon reports the workspace root it
  194. // mounts so assignment paths address the ASKING saddle's
  195. // filesystem; absent on older daemons (kept null → the
  196. // SADDLE_PROJECTS_ROOT fallback chain)
  197. string reported_root = EntryJson.string_member(response, "workspace_root");
  198. if ((reported_key.length > 0 && reported_key != (saddle.ssh_public_key ?? ""))
  199. || reported_root != (saddle.workspace_root ?? "")) {
  200. if (reported_key.length > 0) {
  201. saddle.ssh_public_key = reported_key;
  202. }
  203. saddle.workspace_root = reported_root.length > 0 ? reported_root : null;
  204. yield db.update_async<Saddle>(saddle);
  205. }
  206. print(@"[ClankerSaddle] Registered with saddle '$(saddle.name)'s daemon (token delivered)\n");
  207. } catch (Error e) {
  208. print(@"[ClankerSaddle] Saddle '$(saddle.name)'s daemon did not accept /register ($(e.message)); token stays on the row\n");
  209. }
  210. }
  211. /**
  212. * The saddle a bearer token belongs to, or null — the
  213. * credential check for the id-less `/api/saddle/*` endpoints
  214. * (`next-task`, `note`, `task`): the per-saddle row token,
  215. * the single credential store since plan 10 (arch 24 dropped
  216. * the legacy api_tokens fallback — nothing deployed predates
  217. * the row token). Removing the saddle revokes it.
  218. */
  219. public async Saddle? saddle_for_token(string token) throws Error {
  220. if (token.length == 0) {
  221. return null;
  222. }
  223. return yield db.query<Saddle>()
  224. .where(expr("token == $0", elem(token)))
  225. .first_async();
  226. }
  227. /**
  228. * This app's externally reachable base URL — what daemons use to
  229. * POST notes back — read from the `saddle_callback_base_url` key
  230. * of web-config.json, falling back to the
  231. * `SADDLE_CALLBACK_BASE_URL` environment variable and then to
  232. * `http://localhost:8080`. The daemon typically runs in a
  233. * container or on another host, so its view of this app differs
  234. * from the browser's; set the key to whatever reaches this
  235. * process from there.
  236. */
  237. public string callback_base_url() {
  238. string configured = config.get_string("saddle_callback_base_url", "").strip();
  239. if (configured.length > 0) {
  240. return configured;
  241. }
  242. var from_env = (Environment.get_variable("SADDLE_CALLBACK_BASE_URL") ?? "").strip();
  243. if (from_env.length > 0) {
  244. return from_env;
  245. }
  246. return "http://localhost:8080";
  247. }
  248. /**
  249. * Removes a saddle registration; the task it was running (the
  250. * DERIVED running work — plan 16 item 23: sessions are the
  251. * truth) releases through the ONE path — back to the queue,
  252. * its open sessions closed stopped (no orphan open session
  253. * outlives the row). Revocation: the row dies here, and its
  254. * per-saddle token with it — nothing authorises
  255. * `/api/saddle/*` for it afterwards.
  256. */
  257. public async void unregister_saddle(int64 id) throws Error {
  258. var saddle = yield get_saddle(id);
  259. if (saddle == null) {
  260. return;
  261. }
  262. var work = yield current_work_view((!)saddle);
  263. if (work != null) {
  264. yield task_service.release_task(((!)work).task_id, OUTCOME_STOPPED, true);
  265. }
  266. yield db.delete_async<Saddle>((!)saddle);
  267. yield pulse.bump(0, "saddle removed");
  268. }
  269. /** Loads the saddle identified by `id`, or null. */
  270. public async Saddle? get_saddle(int64 id) throws Error {
  271. return yield db.query<Saddle>()
  272. .where(expr("id == $0", elem<int64?>(id)))
  273. .first_async();
  274. }
  275. /** Lists every saddle ordered by name. */
  276. public async ImmutableLot<Saddle> list_saddles() throws Error {
  277. return yield db.query<Saddle>()
  278. .order_by(expr("name"))
  279. .materialise_async();
  280. }
  281. /**
  282. * The saddle's current work resolved from the SESSION records —
  283. * the ONE truth accessor every surface reads (plan 16 item 23:
  284. * sessions own "who runs what"; the saddle row's cache columns
  285. * are gone). The running session is the newest open session on
  286. * the saddle whose task is still assigned/working — busy iff it
  287. * exists, idle otherwise; a parked task's session closed
  288. * 'parked' at its park (D4), so parks read as free exactly
  289. * like the daemon's file-log brackets say. The task join also
  290. * makes the derivation drift-tolerant: a stale open row on a
  291. * task that has since moved on cannot wedge a saddle busy.
  292. * Null when the saddle holds no running work.
  293. */
  294. public async CurrentWork? current_work_view(Saddle saddle) throws Error {
  295. var session = yield running_session_on_saddle(saddle.id);
  296. if (session == null) {
  297. return null;
  298. }
  299. var work = new CurrentWork() {
  300. task_id = ((!)session).task_id,
  301. session_id = ((!)session).id,
  302. started_at = ((!)session).started_at
  303. };
  304. var task = yield db.query<TaskEntity>()
  305. .where(expr("id == $0", elem<int64?>(((!)session).task_id)))
  306. .first_async();
  307. if (task == null) {
  308. return work;
  309. }
  310. work.task_title = ((!)task).title;
  311. if (((!)task).kind == TASK_KIND_GOAL_RUN) {
  312. var goal = yield db.query<Goal>()
  313. .where(expr("id == $0", elem<int64?>(((!)task).goal_id)))
  314. .first_async();
  315. if (goal != null) {
  316. work.goal_id = ((!)goal).id;
  317. work.goal_title = ((!)goal).title;
  318. }
  319. }
  320. return work;
  321. }
  322. /**
  323. * The derived busy/idle/offline status label every saddle view
  324. * shares — 'busy' iff {@link current_work_view} holds running
  325. * work, else the row's stored OFFLINE probe state ('offline' or
  326. * 'idle').
  327. */
  328. public static string derived_status(Saddle saddle, CurrentWork? work) {
  329. if (work != null) {
  330. return SADDLE_BUSY;
  331. }
  332. return saddle.status == SADDLE_OFFLINE ? SADDLE_OFFLINE : SADDLE_IDLE;
  333. }
  334. /**
  335. * The saddle's RUNNING session — the newest open session whose
  336. * task is assigned/working (the null check runs in SQL: the
  337. * expression layer cannot compare columns against null). At
  338. * most one exists per saddle by construction: every park closes
  339. * its row (D4) and the claim's guard refuses a saddle that
  340. * already holds one.
  341. */
  342. private async ClankerSaddle.Data.Session? running_session_on_saddle(int64 saddle_id)
  343. throws Error {
  344. var command = connection.create_command(
  345. "SELECT s.id AS session_id, s.task_id AS task_id, s.started_at AS started"
  346. + " FROM sessions s JOIN tasks t ON t.id = s.task_id"
  347. + " WHERE s.saddle_id = :saddle AND s.ended_at IS NULL"
  348. + " AND t.status IN ('assigned', 'working')"
  349. + " ORDER BY s.id DESC LIMIT 1");
  350. command.with_parameter<Invercargill.Element>("saddle",
  351. elem<int64?>(saddle_id));
  352. ClankerSaddle.Data.Session? found = null;
  353. foreach (var row in yield command.execute_query_async()) {
  354. int64? session_id = 0;
  355. int64? task_id = 0;
  356. ((!)row.get("session_id")).try_get_as<int64?>(out session_id);
  357. ((!)row.get("task_id")).try_get_as<int64?>(out task_id);
  358. found = new ClankerSaddle.Data.Session() {
  359. id = session_id ?? 0,
  360. task_id = task_id ?? 0,
  361. saddle_id = saddle_id,
  362. started_at = raw_date_time_of(row.get("started"))
  363. };
  364. }
  365. return found;
  366. }
  367. /** A raw query column back as a DateTime (null when null/unreadable). */
  368. private static DateTime? raw_date_time_of(Invercargill.Element? element) {
  369. if (element == null) {
  370. return null;
  371. }
  372. DateTime value;
  373. return ((!)element).try_get_as<DateTime>(out value) ? value : null;
  374. }
  375. /**
  376. * The ids (decimal strings — the OR-chain batch convention) of
  377. * every saddle holding a running session — the derived busy
  378. * set the fleet lists and the offline stamp share.
  379. */
  380. private async Invercargill.DataStructures.HashSet<string> busy_saddle_ids()
  381. throws Error {
  382. var busy = new Invercargill.DataStructures.HashSet<string>();
  383. var command = connection.create_command(
  384. "SELECT DISTINCT s.saddle_id AS saddle_id FROM sessions s"
  385. + " JOIN tasks t ON t.id = s.task_id"
  386. + " WHERE s.ended_at IS NULL"
  387. + " AND t.status IN ('assigned', 'working')");
  388. foreach (var row in yield command.execute_query_async()) {
  389. int64? saddle_id = 0;
  390. ((!)row.get("saddle_id")).try_get_as<int64?>(out saddle_id);
  391. if (saddle_id != null) {
  392. busy.add(((!)saddle_id).to_string());
  393. }
  394. }
  395. return busy;
  396. }
  397. /**
  398. * Saddles holding a running session (the dispatcher's poll
  399. * candidates) — derived from the session records, registration
  400. * order.
  401. */
  402. public async ImmutableLot<Saddle> list_busy_saddles() throws Error {
  403. var busy = yield busy_saddle_ids();
  404. if (busy.count() == 0) {
  405. return (new Vector<Saddle>()).to_immutable_buffer();
  406. }
  407. var rows = new Vector<Saddle>();
  408. foreach (var saddle in yield db.query<Saddle>()
  409. .where(TaskService.id_in("id", busy))
  410. .order_by(expr("id"))
  411. .materialise_async()) {
  412. rows.add(saddle);
  413. }
  414. return rows.to_immutable_buffer();
  415. }
  416. /**
  417. * Persists the saddle's git identity — what its daemon commits
  418. * as (`git_identity` in every /start). Empty/null clears the
  419. * fields so the daemon's fallback applies.
  420. */
  421. public async void update_git_identity(int64 id, string? git_name,
  422. string? git_email) throws Error {
  423. var saddle = yield get_saddle(id);
  424. if (saddle == null) {
  425. throw new Error(Quark.from_string("clanker-saddle"), 17,
  426. @"No saddle exists with id $(id)");
  427. }
  428. ((!)saddle).git_name = git_name != null && git_name.length > 0 ? git_name : null;
  429. ((!)saddle).git_email = git_email != null && git_email.length > 0 ? git_email : null;
  430. yield db.update_async<Saddle>((!)saddle);
  431. yield pulse.bump(0, "saddle identity saved");
  432. }
  433. /**
  434. * Saves the saddle admin's standing prompt note (goal 8 task 5):
  435. * the message about THIS machine rendered into every
  436. * assignment's system prompt. Empty clears it (the daemon then
  437. * omits the section entirely); the value is clamped to
  438. * {@link PROMPT_NOTE_MAX_CHARS} so a fat note can never bloat
  439. * every prompt.
  440. */
  441. public async void update_prompt_note(int64 id, string? prompt_note) throws Error {
  442. var saddle = yield get_saddle(id);
  443. if (saddle == null) {
  444. throw new Error(Quark.from_string("clanker-saddle"), 17,
  445. @"No saddle exists with id $(id)");
  446. }
  447. var trimmed = prompt_note != null ? prompt_note.strip() : "";
  448. if (trimmed.length > PROMPT_NOTE_MAX_CHARS) {
  449. trimmed = trimmed.substring(0, (int) PROMPT_NOTE_MAX_CHARS);
  450. }
  451. ((!)saddle).prompt_note = trimmed.length > 0 ? trimmed : null;
  452. yield db.update_async<Saddle>((!)saddle);
  453. yield pulse.bump(0, "saddle prompt note saved");
  454. }
  455. /**
  456. * Persists the daemon's reported `ssh_public_key` onto the saddle
  457. * row (the plan-9 contract: the key travels on /status, empty
  458. * until the daemon generates one; the saddles page shows it for
  459. * adding as a deploy key). A single-column UPDATE — the
  460. * full-row write could stomp a concurrent ask claim. Only writes
  461. * when it changed.
  462. */
  463. private async void store_reported_ssh_key(Saddle saddle, Json.Object status) throws Error {
  464. if (!status.has_member("ssh_public_key")) {
  465. return;
  466. }
  467. string reported = status.get_string_member_with_default("ssh_public_key", "");
  468. string stored = saddle.ssh_public_key ?? "";
  469. if (reported == stored) {
  470. return;
  471. }
  472. var command = connection.create_command(
  473. "UPDATE saddles SET ssh_public_key = :key WHERE id = :id");
  474. command.with_parameter<Invercargill.Element>("key", elem(reported.length > 0 ? reported : null));
  475. command.with_parameter<Invercargill.Element>("id", elem<int64?>(saddle.id));
  476. yield command.execute_non_query_async();
  477. }
  478. /**
  479. * Records a /status observation: the health snapshot, the
  480. * daemon-reported ssh key, and the goal-branch push flags
  481. * (plan 15 C4). The `resumed` flag needs no projection — the
  482. * restore's record is the daemon's own system entry in the
  483. * raw terminal log (plan 16 S5/U2).
  484. */
  485. private async void record_status_probe(Saddle saddle, Json.Object status) throws Error {
  486. string state = status.has_member("state")
  487. ? status.get_string_member_with_default("state", "") : "";
  488. int64 uptime = status.has_member("uptime_seconds")
  489. && status.get_member("uptime_seconds").get_value_type() == typeof(int64)
  490. ? status.get_int_member("uptime_seconds") : 0;
  491. int64 activity = status.has_member("last_activity_epoch")
  492. && status.get_member("last_activity_epoch").get_value_type() == typeof(int64)
  493. ? status.get_int_member("last_activity_epoch") : 0;
  494. poll_state.record_probe(saddle.id, state, uptime, activity);
  495. yield store_reported_ssh_key(saddle, status);
  496. // Plan 15 C4: a daemon running (or parking — the
  497. // workspace stays while the saddle frees) a goal agent
  498. // exposes the per-repo push flags of the goal branches on
  499. // /status; they are recorded as LIVE probe state on the
  500. // saddle (never a stored snapshot on the goal row) and
  501. // the goal page reads them back at render time for its
  502. // advisory push chips. The push rule itself is enforced
  503. // daemon-side at the agent's completion attempt.
  504. if (status.has_member("goal_branches")) {
  505. var goal_branches = status.get_member("goal_branches");
  506. if (goal_branches.get_node_type() == Json.NodeType.ARRAY
  507. || goal_branches.get_node_type() == Json.NodeType.OBJECT) {
  508. var generator = new Json.Generator();
  509. generator.set_root(goal_branches);
  510. poll_state.record_goal_branches(saddle.id, generator.to_data(null));
  511. }
  512. }
  513. // Plan 16 S5 (U2): the daemon's own system entry (typed
  514. // 'system' in the results stream) is the raw-terminal
  515. // record of a workspace restore — the web no longer authors
  516. // a duplicate timeline row for it. The `resumed` flag
  517. // carries no further web-side projection.
  518. }
  519. /**
  520. * Wakes a sleeping daemon (plan 10 wire contract): `POST /wake`
  521. * `{}` — a capable task is ready; sleeping daemons ask for work,
  522. * busy ones ignore it, idle-awake ones re-ask. Idempotent. The
  523. * caller records the pending wake for the lost-wake guard.
  524. */
  525. public async void wake_saddle(Saddle saddle) throws Error {
  526. var payload = new Json.Builder();
  527. payload.begin_object();
  528. payload.end_object();
  529. Json.Node? root = null;
  530. int status_code = yield request_saddle(saddle.host, "POST", "/wake",
  531. (owned) payload, out root);
  532. if (status_code >= 400) {
  533. throw new Error(Quark.from_string("clanker-saddle"), 49,
  534. @"wake returned HTTP $(status_code)");
  535. }
  536. }
  537. /**
  538. * Cancelling / requeueing / stopping a task with a live session
  539. * reaches its daemon (wire contract): `POST /task-update
  540. * {kind: "cancel"}` — the authoritative stop through the daemon's
  541. * abort machinery, the workspace kept on disk per the park
  542. * contract. Success is narrated as "<verb>; daemon aborted"; a
  543. * transport failure marks the saddle offline (the daemon's next
  544. * ask proves it alive again and flips the row back to idle). The
  545. * daemon must never keep running a task the web has moved on
  546. * from as a zombie.
  547. */
  548. public async void abort_live_session(TaskService tasks, int64 task_id,
  549. string verb) throws Error {
  550. var session = yield tasks.open_session_for_task(task_id);
  551. if (session == null) {
  552. return;
  553. }
  554. yield abort_session_on_its_saddle(tasks, (!)session, verb);
  555. }
  556. /**
  557. * SADDLE_STALE_S (default 90s): the silence window after which a
  558. * running daemon owes an exceptional GET /status probe.
  559. */
  560. public static uint stale_after_seconds() {
  561. var raw = Environment.get_variable("SADDLE_STALE_S") ?? "";
  562. int64 parsed = int64.parse(raw);
  563. return parsed > 0 ? (uint) parsed : 90;
  564. }
  565. /**
  566. * {@link abort_live_session} keyed by saddle instead of task —
  567. * the unregister path. The cancel is delivered to the saddle's
  568. * RUNNING session (the derived record; plan 16 item 23) — not
  569. * re-resolved through a task's lowest-id open session, which a
  570. * stale fixture row on another saddle would redirect to the
  571. * wrong daemon.
  572. */
  573. public async void abort_saddle_session(TaskService tasks, int64 saddle_id,
  574. string verb) throws Error {
  575. var session = yield running_session_on_saddle(saddle_id);
  576. if (session == null) {
  577. return;
  578. }
  579. yield abort_session_on_its_saddle(tasks, (!)session, verb);
  580. }
  581. /**
  582. * The cancel delivery core shared by the task-keyed and
  583. * saddle-keyed abort paths: `POST /task-update {kind: cancel}`
  584. * to the daemon holding THIS session.
  585. */
  586. private async void abort_session_on_its_saddle(TaskService tasks,
  587. ClankerSaddle.Data.Session session, string verb) throws Error {
  588. var saddle = yield get_saddle(session.saddle_id);
  589. if (saddle == null) {
  590. return;
  591. }
  592. var payload = new Json.Builder();
  593. payload.begin_object();
  594. payload.set_member_name("kind");
  595. payload.add_string_value("cancel");
  596. payload.end_object();
  597. Json.Node? root = null;
  598. try {
  599. int status_code = yield request_saddle(((!)saddle).host, "POST",
  600. "/task-update", (owned) payload, out root);
  601. if (status_code >= 400) {
  602. throw new Error(Quark.from_string("clanker-saddle"), 45,
  603. @"task-update cancel returned HTTP $(status_code)");
  604. }
  605. yield tasks.add_system_comment(session.task_id, @"$(verb); daemon aborted");
  606. } catch (Error e) {
  607. warning(@"[ClankerSaddle] Abort for task $(session.task_id) failed: $(e.message); marking saddle '$(((!)saddle).name)' offline");
  608. yield mark_saddle_unreachable(((!)saddle).id, tasks,
  609. @"abort delivery failed ($(e.message))");
  610. yield tasks.add_system_comment(session.task_id,
  611. @"$(verb); daemon unreachable — saddle marked offline");
  612. }
  613. }
  614. /**
  615. * Requeues tasks caught running on saddles that have since gone
  616. * offline (a manual status refresh, a restart) — without this
  617. * pass they would wait forever for a daemon that will not answer.
  618. * Only `assigned`/`working` tasks move: parked tasks
  619. * (blocked/review/suspended, their sessions closed 'parked')
  620. * keep their records, which the resume machinery needs. Each
  621. * move is {@link TaskService.requeue_task} — the release owner
  622. * closes the session (the close IS the free, item 23) — plus a
  623. * system comment so the timeline shows why the task restarted.
  624. */
  625. public async void requeue_tasks_on_offline_saddles(TaskService tasks) throws Error {
  626. foreach (var status in new string[] { STATUS_ASSIGNED, STATUS_WORKING }) {
  627. foreach (var task in yield db.query<TaskEntity>()
  628. .where(expr("status == $0", elem(status)))
  629. .materialise_async()) {
  630. var session = yield tasks.open_session_for_task(task.id);
  631. if (session == null) {
  632. continue;
  633. }
  634. if (yield tasks.newest_block_answered(task.id)) {
  635. // Parked-with-answered: a verdict awaits the
  636. // resume machinery — never requeue it.
  637. continue;
  638. }
  639. var saddle = yield get_saddle(((!)session).saddle_id);
  640. if (saddle == null || ((!)saddle).status != SADDLE_OFFLINE) {
  641. continue;
  642. }
  643. print(@"[ClankerSaddle] Task $(task.id) requeued: its saddle went offline\n");
  644. yield tasks.requeue_task(task.id);
  645. yield tasks.add_system_comment(task.id,
  646. "saddle went offline; task requeued");
  647. }
  648. }
  649. }
  650. /**
  651. * Every saddle free for work — derived (plan 16 item 23): not
  652. * offline (the probe state) and holding no running session —
  653. * registration order, the dispatcher's capability-matching
  654. * candidate list. A parked task's closed session reads free
  655. * exactly like the freed daemon it brackets.
  656. */
  657. public async ImmutableLot<Saddle> list_idle_saddles() throws Error {
  658. var busy = yield busy_saddle_ids();
  659. var rows = new Vector<Saddle>();
  660. foreach (var saddle in yield db.query<Saddle>()
  661. .order_by(expr("id"))
  662. .materialise_async()) {
  663. if (saddle.status == SADDLE_OFFLINE
  664. || busy.contains(((int64) saddle.id).to_string())) {
  665. continue;
  666. }
  667. rows.add(saddle);
  668. }
  669. return rows.to_immutable_buffer();
  670. }
  671. /**
  672. * Marks a saddle offline when its control server cannot be
  673. * reached. The task it was RUNNING (the derived running
  674. * session — plan 16 item 23) releases back to 'queued' through
  675. * {@link TaskService.release_task} (the ONE owner; it closes
  676. * the session 'stopped'), with a system comment so the timeline
  677. * shows the interruption; combined with the daemon's
  678. * crash-resume, the next dispatch restores the workspace and
  679. * reports `resumed` on /status. A task that parked mid-flight
  680. * (its session already closed 'parked', the verdict awaiting
  681. * the resume) keeps its park. The offline stamp itself is the
  682. * ONE surviving guarded write — see {@link stamp_offline}.
  683. */
  684. public async void mark_saddle_unreachable(int64 id, TaskService tasks,
  685. string? reason = null) throws Error {
  686. var saddle = yield get_saddle(id);
  687. if (saddle == null) {
  688. return;
  689. }
  690. var session = yield running_session_on_saddle(id);
  691. int64 seen_session = session != null ? ((!)session).id : 0;
  692. int64 held_task = 0;
  693. if (session != null) {
  694. held_task = ((!)session).task_id;
  695. // The requeue flip is guarded: a park (or terminal)
  696. // interleaving this path keeps its state — the parked
  697. // verdict must survive for the resume.
  698. var flip = connection.create_command(
  699. "UPDATE tasks SET status = 'queued' WHERE id = :id"
  700. + " AND status IN ('assigned', 'working')");
  701. flip.with_parameter<Invercargill.Element>("id", elem<int64?>(held_task));
  702. if ((yield flip.execute_non_query_async()) > 0) {
  703. string unreachable_text = reason != null && reason.length > 0
  704. ? @"Saddle became unreachable ($(reason)); task requeued"
  705. : "Saddle became unreachable; task requeued";
  706. yield tasks.add_system_comment(held_task, unreachable_text);
  707. // Closes the held session FIRST — the offline
  708. // stamp's guard refuses to land over a running
  709. // session, and by then there is none.
  710. yield tasks.release_task(held_task, OUTCOME_STOPPED, false);
  711. } else {
  712. // Parked (or finished) mid-flight — the park closed
  713. // its session and the verdict must survive for the
  714. // resume; nothing to release.
  715. held_task = 0;
  716. }
  717. }
  718. yield stamp_offline(id, seen_session);
  719. // An offline saddle is neither sleeping nor wake-awaiting;
  720. // a restarted daemon re-establishes both by asking.
  721. poll_state.clear_ask(id);
  722. yield pulse.bump(0, "saddle unreachable");
  723. }
  724. /**
  725. * The ONE surviving guarded saddle write (plan 16 item 23):
  726. * the OFFLINE probe stamp. The guard is the sessions-truth
  727. * analogue of the old seen-assignment guard — the stamp lands
  728. * unless a running session the probe did NOT see at entry
  729. * appeared meanwhile (an ask claim is proof of life and must
  730. * not be stomped offline). The session observed at entry
  731. * (`seen_session`) never blocks its own stamp.
  732. */
  733. private async void stamp_offline(int64 saddle_id, int64 seen_session) throws Error {
  734. var command = connection.create_command(
  735. "UPDATE saddles SET status = '" + SADDLE_OFFLINE + "' WHERE id = :id"
  736. + " AND NOT EXISTS (SELECT 1 FROM sessions s"
  737. + " JOIN tasks t ON t.id = s.task_id"
  738. + " WHERE s.saddle_id = :id AND s.ended_at IS NULL"
  739. + " AND t.status IN ('assigned', 'working')"
  740. + " AND s.id != :seen)");
  741. command.with_parameter<Invercargill.Element>("id", elem<int64?>(saddle_id));
  742. command.with_parameter<Invercargill.Element>("seen", elem<int64?>(seen_session));
  743. yield command.execute_non_query_async();
  744. }
  745. /**
  746. * Assembles the assignment payload shared by the /start push and
  747. * the `POST /api/saddle/next-task` response — `{task, kind,
  748. * mode, creator, goal?, rules, project_notes, callback,
  749. * llm_config, session_id}` — with the project, links (the
  750. * goal's own links for goal work, else the project's), branch
  751. * fields and recent comments inside the task object, plus any
  752. * block answers not yet handed over (a resumed workspace picks
  753. * its pending answers up from the assignment itself), the
  754. * effective rule set ({@link RuleService.effective_rules}) as
  755. * the top-level `rules` array, the capped `project_notes` of
  756. * the primary and co-developed projects, the note-creation
  757. * `callback` (base URL + the saddle's own bearer token), the
  758. * saddle's `git_identity`, a `capabilities` context (plan 10
  759. * phase 3: `saddle_tags` — the allocated saddle's own tags —
  760. * and `fleet_tags` — every tag name known across the fleet),
  761. * the plan-12 goal context (`goal` for goal work; the
  762. * worker's `ephemeral_branch`) and — for resumed workspaces —
  763. * the `resume_notice`/`branch_advanced` context. `session_id`
  764. * (the web-side run row the daemon reports against) rides
  765. * along so pushed telemetry can key on it. Project checkouts
  766. * are addressed by the `SADDLE_PROJECTS_ROOT` convention
  767. * (`root/task-<task id>/name`) and cloned from each project's
  768. * `git_url`; the LLM endpoint by the {@link LlmEndpoint}
  769. * environment.
  770. */
  771. public async Json.Object build_assignment_payload(Saddle saddle, TaskEntity task,
  772. int64 session_id, ProjectService projects, string? resume_notice = null,
  773. bool? branch_advanced = null,
  774. string? context_update = null) throws Error {
  775. var endpoint = LlmEndpoint.from_environment();
  776. if (endpoint == null) {
  777. throw new Error(Quark.from_string("clanker-saddle"), 24,
  778. "No LLM endpoint configured (set SADDLE_LLM_BASE_URL)");
  779. }
  780. var project = task.project_id > 0
  781. ? yield db.query<Project>()
  782. .where(expr("id == $0", elem<int64?>(task.project_id)))
  783. .first_async()
  784. : null;
  785. // Goal 8 task 4: project_id 0 is a STANDALONE task — no
  786. // project row exists (and none is looked up), the
  787. // co-developed/reference arrays ship EMPTY and the daemon
  788. // renders the standalone prompt section. A task with a
  789. // non-zero project_id whose row is missing is still the
  790. // data-integrity error it always was.
  791. if (project == null && task.project_id > 0) {
  792. throw new Error(Quark.from_string("clanker-saddle"), 12,
  793. @"Task $(task.id) has no project $(task.project_id)");
  794. }
  795. // Plan 12: goal work (the agent's own run and its worker
  796. // tasks) scopes its co-developed/reference set through the
  797. // GOAL's links; standalone tasks through the project's.
  798. // Standalone tasks (project 0) own no links at all — the
  799. // simplest consistent shape: both arrays stay empty.
  800. Goal? goal = null;
  801. if (task.goal_id > 0) {
  802. goal = yield db.query<Goal>()
  803. .where(expr("id == $0", elem<int64?>(task.goal_id)))
  804. .first_async();
  805. }
  806. var co_developed = new Vector<Project>();
  807. var references = new Vector<Project>();
  808. if (goal != null) {
  809. foreach (var link in yield goal_service.list_links(task.goal_id)) {
  810. var linked = yield db.query<Project>()
  811. .where(expr("id == $0", elem<int64?>(link.project_id)))
  812. .first_async();
  813. if (linked == null) {
  814. continue;
  815. }
  816. if (link.relationship == LINK_CO_DEVELOPED) {
  817. co_developed.add((!)linked);
  818. } else if (link.relationship == LINK_REFERENCE) {
  819. references.add((!)linked);
  820. }
  821. }
  822. } else if (project != null) {
  823. foreach (var link in yield projects.list_links(task.project_id)) {
  824. var linked = yield db.query<Project>()
  825. .where(expr("id == $0", elem<int64?>(link.linked_project_id)))
  826. .first_async();
  827. if (linked == null) {
  828. continue;
  829. }
  830. if (link.relationship == LINK_CO_DEVELOPED) {
  831. co_developed.add((!)linked);
  832. } else if (link.relationship == LINK_REFERENCE) {
  833. references.add((!)linked);
  834. }
  835. }
  836. }
  837. var payload = new Json.Builder();
  838. payload.begin_object();
  839. payload.set_member_name("task");
  840. payload.begin_object();
  841. payload.set_member_name("id");
  842. payload.add_int_value(task.id);
  843. payload.set_member_name("title");
  844. payload.add_string_value(task.title);
  845. payload.set_member_name("description");
  846. payload.add_string_value(task.description ?? "");
  847. payload.set_member_name("project_name");
  848. payload.add_string_value(project != null ? ((!)project).name : "standalone");
  849. payload.set_member_name("project_branch");
  850. payload.add_string_value(project != null ? ((!)project).default_branch : "");
  851. payload.set_member_name("task_branch");
  852. payload.add_string_value(task.task_branch ?? "");
  853. payload.set_member_name("branch_from");
  854. payload.add_string_value(task.branch_from
  855. ?? (project != null ? ((!)project).default_branch : ""));
  856. payload.set_member_name("co_developed");
  857. payload.begin_array();
  858. if (project != null) {
  859. payload.add_value(project_node(((!)project), task.id, false, saddle));
  860. }
  861. foreach (var linked in co_developed) {
  862. payload.add_value(project_node(linked, task.id, false, saddle));
  863. }
  864. payload.end_array();
  865. payload.set_member_name("reference");
  866. payload.begin_array();
  867. foreach (var linked in references) {
  868. payload.add_value(project_node(linked, task.id, true, saddle));
  869. }
  870. payload.end_array();
  871. payload.set_member_name("comments");
  872. payload.begin_array();
  873. var comments = yield db.query<TaskComment>()
  874. .where(expr("task_id == $0", elem<int64?>(task.id)))
  875. .order_by(expr("id"))
  876. .materialise_async();
  877. foreach (var comment in comments) {
  878. payload.begin_object();
  879. payload.set_member_name("author");
  880. payload.add_string_value(comment.author);
  881. payload.set_member_name("body");
  882. payload.add_string_value(comment.body);
  883. if (comment.quote != null) {
  884. payload.set_member_name("quote");
  885. payload.add_string_value(((!)comment.quote));
  886. }
  887. payload.end_object();
  888. }
  889. payload.end_array();
  890. payload.set_member_name("answers");
  891. payload.begin_array();
  892. foreach (var pending in yield take_pending_answers_for_task(task.id)) {
  893. payload.begin_object();
  894. payload.set_member_name("block_id");
  895. payload.add_int_value(pending.block_id);
  896. payload.set_member_name("answer");
  897. payload.add_string_value(raw_answer_for_saddle(pending.answer));
  898. if (branch_advanced != null) {
  899. payload.set_member_name("branch_advanced");
  900. payload.add_boolean_value((!)branch_advanced);
  901. }
  902. payload.end_object();
  903. }
  904. payload.end_array();
  905. payload.end_object();
  906. // The plan-12 wire additions: kind ("task"|"goal_run"),
  907. // mode and creator ride the top level beside the task.
  908. payload.set_member_name("kind");
  909. payload.add_string_value(task.kind);
  910. payload.set_member_name("mode");
  911. payload.add_string_value(task.mode);
  912. payload.set_member_name("creator");
  913. payload.add_string_value(task.creator);
  914. // The goal context (goal work only): the goal agent gets
  915. // the full brief + approved plan; a goal worker gets the
  916. // same object as read-only reference (the daemon adds the
  917. // stick-to-mandate shaping) plus its ephemeral branch.
  918. if (goal != null) {
  919. payload.set_member_name("goal");
  920. payload.begin_object();
  921. payload.set_member_name("id");
  922. payload.add_int_value(((!)goal).id);
  923. payload.set_member_name("branch");
  924. payload.add_string_value(((!)goal).branch_name);
  925. payload.set_member_name("base_branch");
  926. payload.add_string_value(((!)goal).base_branch
  927. ?? (project != null ? ((!)project).default_branch : ""));
  928. payload.set_member_name("brief");
  929. payload.add_string_value(((!)goal).brief);
  930. payload.set_member_name("plan");
  931. payload.add_string_value(((!)goal).plan ?? "");
  932. payload.end_object();
  933. if (task.kind == TASK_KIND_TASK) {
  934. payload.set_member_name("ephemeral_branch");
  935. payload.add_string_value(@"task/$(task.goal_id)-$(task.id)");
  936. }
  937. }
  938. // The wire contract's top-level `rules` addition: the
  939. // effective rule set (task ∪ primary ∪ co-developed, deduped,
  940. // stable order — see RuleService.effective_rules). Goal work
  941. // expands co-developed through the goal's links.
  942. payload.set_member_name("rules");
  943. payload.begin_array();
  944. foreach (var rule in yield rules.effective_rules_for(task, projects,
  945. goal != null ? task.goal_id : 0)) {
  946. payload.begin_object();
  947. payload.set_member_name("name");
  948. payload.add_string_value(rule.name);
  949. payload.set_member_name("body");
  950. payload.add_string_value(rule.body);
  951. payload.end_object();
  952. }
  953. payload.end_array();
  954. // The wire contract's `git_identity` addition (plan 9): the
  955. // name/email the daemon commits as, from the saddle row.
  956. // Omitted entirely when unset so the daemon's fallback
  957. // identity applies.
  958. string identity_name = saddle.git_name ?? "";
  959. string identity_email = saddle.git_email ?? "";
  960. if (identity_name.length > 0 || identity_email.length > 0) {
  961. payload.set_member_name("git_identity");
  962. payload.begin_object();
  963. payload.set_member_name("name");
  964. payload.add_string_value(identity_name);
  965. payload.set_member_name("email");
  966. payload.add_string_value(identity_email);
  967. payload.end_object();
  968. }
  969. // The wire contract's `prompt_note` addition (goal 8 task
  970. // 5): the saddle admin's standing message about THIS
  971. // machine, served on EVERY assignment (the daemon renders
  972. // it verbatim under an admin heading). Empty string when
  973. // unset, clamped so a fat note can never bloat prompts.
  974. var prompt_note_text = saddle.prompt_note ?? "";
  975. if (prompt_note_text.length > PROMPT_NOTE_MAX_CHARS) {
  976. prompt_note_text = prompt_note_text.substring(0, (int) PROMPT_NOTE_MAX_CHARS);
  977. }
  978. payload.set_member_name("prompt_note");
  979. payload.add_string_value(prompt_note_text);
  980. // The wire contract's `capabilities` context (plan 10
  981. // phase 3): the allocated saddle's own tags plus every tag
  982. // name known across the fleet (the operator-managed
  983. // catalogue) — the daemon renders both into its system
  984. // prompt and validates `task create --capability`.
  985. payload.set_member_name("capabilities");
  986. payload.begin_object();
  987. payload.set_member_name("saddle_tags");
  988. payload.begin_array();
  989. foreach (var carried in yield capability_service.list_for_saddle(saddle.id)) {
  990. payload.add_string_value(carried.name);
  991. }
  992. payload.end_array();
  993. payload.set_member_name("fleet_tags");
  994. payload.begin_array();
  995. foreach (var known in yield capability_service.list_capabilities()) {
  996. payload.add_string_value(known.name);
  997. }
  998. payload.end_array();
  999. payload.end_object();
  1000. // The wire contract's `resume_notice` addition: on a
  1001. // workspace-adopting start (a suspended parent's resume), the
  1002. // daemon injects it as the next user turn instead of the
  1003. // opening prompt.
  1004. if (resume_notice != null && resume_notice.length > 0) {
  1005. payload.set_member_name("resume_notice");
  1006. payload.add_string_value(resume_notice);
  1007. }
  1008. // The wire's `branch_advanced` (resume pushes): whether the
  1009. // shared branch moved while this task was parked — false or
  1010. // absent means the daemon omits the pull instruction entirely.
  1011. if (branch_advanced != null) {
  1012. payload.set_member_name("branch_advanced");
  1013. payload.add_boolean_value((!)branch_advanced);
  1014. }
  1015. // The wire's `context_update` (resume pushes): a short
  1016. // markdown delta of the instruction context that changed
  1017. // while the workspace was parked — the goal's plan
  1018. // approved since the run's last round, rules created,
  1019. // edited or attached, or new project notes. The daemon
  1020. // appends it as a TURN (prompt-cache friendly) instead of
  1021. // refreshing the system prompt; absent when nothing
  1022. // changed.
  1023. if (context_update != null && context_update.length > 0) {
  1024. payload.set_member_name("context_update");
  1025. payload.add_string_value(context_update);
  1026. }
  1027. // The wire contract's `project_notes` addition: notes of the
  1028. // primary and co-developed projects, oldest first, capped at
  1029. // 30 entries / 20 000 chars (head kept).
  1030. var note_project_names = new Dictionary<string, string>();
  1031. var project_notes_rows = new Vector<ClankerSaddle.Data.ProjectNote>();
  1032. if (project != null) {
  1033. note_project_names.set(((!)project).id.to_string(), ((!)project).name);
  1034. foreach (var note in yield projects.list_notes(((!)project).id)) {
  1035. project_notes_rows.add(note);
  1036. }
  1037. }
  1038. foreach (var linked in co_developed) {
  1039. note_project_names.set(linked.id.to_string(), linked.name);
  1040. foreach (var note in yield projects.list_notes(linked.id)) {
  1041. project_notes_rows.add(note);
  1042. }
  1043. }
  1044. var project_notes = project_notes_rows.sort((a, b) => (int)(a.id - b.id));
  1045. payload.set_member_name("project_notes");
  1046. payload.begin_array();
  1047. int64 note_entries = 0;
  1048. int64 note_chars = 0;
  1049. foreach (var note in project_notes) {
  1050. if (note_entries >= PROJECT_NOTES_MAX_ENTRIES
  1051. || note_chars >= PROJECT_NOTES_MAX_CHARS) {
  1052. break;
  1053. }
  1054. string note_body = note.body;
  1055. if (note_chars + note_body.length > PROJECT_NOTES_MAX_CHARS) {
  1056. note_body = note_body.substring(0,
  1057. (int)(PROJECT_NOTES_MAX_CHARS - note_chars));
  1058. }
  1059. payload.begin_object();
  1060. payload.set_member_name("project");
  1061. payload.add_string_value(note_project_names.get_or_default(note.project_id.to_string()) ?? "");
  1062. payload.set_member_name("body");
  1063. payload.add_string_value(note_body);
  1064. payload.end_object();
  1065. note_entries++;
  1066. note_chars += note_body.length;
  1067. }
  1068. payload.end_array();
  1069. // The wire contract's `understanding` addition (goal 24):
  1070. // the PRIMARY project's current understanding document —
  1071. // a markdown summary of the project's structure and its
  1072. // unusual corners, stamped with the commit it was written
  1073. // against. Present but empty when the project has none;
  1074. // head-clamped at UNDERSTANDING_MAX_CHARS.
  1075. string understanding_body = "";
  1076. if (project != null) {
  1077. var current_understanding = yield understandings
  1078. .current_understanding(((!)project).id);
  1079. if (current_understanding != null) {
  1080. understanding_body = ((!)current_understanding).body;
  1081. if (understanding_body.length > UNDERSTANDING_MAX_CHARS) {
  1082. understanding_body = understanding_body.substring(0,
  1083. (int) UNDERSTANDING_MAX_CHARS);
  1084. }
  1085. }
  1086. }
  1087. payload.set_member_name("understanding");
  1088. payload.add_string_value(understanding_body);
  1089. // The wire contract's `callback` addition: this app's
  1090. // externally reachable base URL plus the bearer token the
  1091. // daemon presents when posting back — its own per-saddle
  1092. // credential (plan 10), which every `/api/saddle/*` call
  1093. // carries.
  1094. payload.set_member_name("callback");
  1095. payload.begin_object();
  1096. payload.set_member_name("base_url");
  1097. payload.add_string_value(callback_base_url());
  1098. payload.set_member_name("token");
  1099. payload.add_string_value(saddle.token ?? "");
  1100. payload.end_object();
  1101. payload.set_member_name("llm_config");
  1102. payload.begin_object();
  1103. payload.set_member_name("base_url");
  1104. payload.add_string_value(((!)endpoint).base_url);
  1105. payload.set_member_name("api_key");
  1106. payload.add_string_value(((!)endpoint).api_key);
  1107. payload.set_member_name("model");
  1108. payload.add_string_value(((!)endpoint).model);
  1109. payload.end_object();
  1110. // The web-side run row the daemon's pushed telemetry keys on
  1111. // (plan-10 phase-1 wire addition: it rides every assignment
  1112. // so /start consumers can ignore it harmlessly).
  1113. payload.set_member_name("session_id");
  1114. payload.add_int_value(session_id);
  1115. payload.end_object();
  1116. var root = payload.get_root().copy();
  1117. return root.get_object();
  1118. }
  1119. /**
  1120. * Probes a saddle's `GET /status` and records what a probe can
  1121. * tell: the health snapshot, the daemon-reported ssh key and
  1122. * the goal-branch push flags. Busy/idle need NO write anymore —
  1123. * they are derived from the session records (plan 16 item 23).
  1124. * The row's only remaining write is the OFFLINE probe state: a
  1125. * successful probe is proof of life and clears it; a failed
  1126. * probe stamps it through the guarded write (an ask claim that
  1127. * landed while the probe was in flight is a NEWER proof of
  1128. * life and must not be stomped).
  1129. */
  1130. public async void refresh_saddle_status(int64 id) throws Error {
  1131. var saddle = yield get_saddle(id);
  1132. if (saddle == null) {
  1133. throw new Error(Quark.from_string("clanker-saddle"), 17,
  1134. @"No saddle exists with id $(id)");
  1135. }
  1136. var session = yield running_session_on_saddle(id);
  1137. int64 seen_session = session != null ? ((!)session).id : 0;
  1138. try {
  1139. var status = yield request_saddle_json(((!)saddle).host, "GET", "/status", null);
  1140. yield record_status_probe(((!)saddle), status);
  1141. var clear = connection.create_command(
  1142. "UPDATE saddles SET status = '" + SADDLE_IDLE
  1143. + "' WHERE id = :id AND status = '" + SADDLE_OFFLINE + "'");
  1144. clear.with_parameter<Invercargill.Element>("id", elem<int64?>(id));
  1145. yield clear.execute_non_query_async();
  1146. } catch (Error e) {
  1147. yield stamp_offline(id, seen_session);
  1148. }
  1149. yield pulse.bump(0, "saddle status refreshed");
  1150. }
  1151. /**
  1152. * Answered blocks of `task_id` not yet handed over anywhere;
  1153. * marks them delivered — a resumed assignment carries its
  1154. * pending answers in the payload ({@link push_task_to_saddle}).
  1155. */
  1156. public async ImmutableLot<PendingAnswer> take_pending_answers_for_task(int64 task_id) throws Error {
  1157. var blocks = yield db.query<TaskBlock>()
  1158. .where(expr("task_id == $0", elem<int64?>(task_id)))
  1159. .materialise_async();
  1160. var rows = new Vector<PendingAnswer>();
  1161. foreach (var block in blocks) {
  1162. if (block.answer == null) {
  1163. continue;
  1164. }
  1165. if (!poll_state.mark_delivered(block.id)) {
  1166. continue;
  1167. }
  1168. rows.add(new PendingAnswer() {
  1169. block_id = block.id,
  1170. answer = raw_answer_for_saddle(((!)block).answer ?? "")
  1171. });
  1172. }
  1173. return rows.to_immutable_buffer();
  1174. }
  1175. /**
  1176. * Pushes the task's undelivered human comments to the RUNNING
  1177. * daemon's `POST /task-update {kind: "comment"}` endpoint so
  1178. * they land in the live conversation. Best effort: a daemon
  1179. * without the route (HTTP 404/405) keeps the comments pending,
  1180. * and other failures are logged and retried on the next
  1181. * dispatcher tick.
  1182. */
  1183. public async void deliver_pending_comments(int64 saddle_id, int64 task_id) throws Error {
  1184. var saddle = yield get_saddle(saddle_id);
  1185. if (saddle == null) {
  1186. return;
  1187. }
  1188. var comments = yield db.query<TaskComment>()
  1189. .where(expr("task_id == $0", elem<int64?>(task_id)))
  1190. .order_by(expr("id"))
  1191. .materialise_async();
  1192. foreach (var comment in comments) {
  1193. if (!comment.author.has_prefix("user:")
  1194. || poll_state.is_comment_delivered(comment.id)) {
  1195. continue;
  1196. }
  1197. var payload = new Json.Builder();
  1198. payload.begin_object();
  1199. payload.set_member_name("kind");
  1200. payload.add_string_value("comment");
  1201. payload.set_member_name("text");
  1202. payload.add_string_value(comment.body);
  1203. payload.set_member_name("quote");
  1204. payload.add_string_value(comment.quote ?? "");
  1205. payload.end_object();
  1206. Json.Node? root = null;
  1207. int status_code = yield request_saddle(((!)saddle).host, "POST",
  1208. "/task-update", (owned) payload, out root);
  1209. if (status_code == 200 || status_code == 204) {
  1210. poll_state.mark_comment_delivered(comment.id);
  1211. } else if (status_code != 404 && status_code != 405) {
  1212. warning(@"[ClankerSaddle] Comment task-update to saddle $(saddle_id) returned HTTP $(status_code)");
  1213. }
  1214. }
  1215. }
  1216. /**
  1217. * Marks the task's human comments as delivered — they were handed
  1218. * over inside an assignment payload, so neither the poll endpoint
  1219. * nor `/notify` should repeat them.
  1220. */
  1221. public async void mark_task_comments_delivered(int64 task_id) throws Error {
  1222. var comments = yield db.query<TaskComment>()
  1223. .where(expr("task_id == $0", elem<int64?>(task_id)))
  1224. .materialise_async();
  1225. foreach (var comment in comments) {
  1226. if (comment.author.has_prefix("user:")) {
  1227. poll_state.mark_comment_delivered(comment.id);
  1228. }
  1229. }
  1230. }
  1231. /**
  1232. * The goal id when `task_id` is a goal agent's run task (kind
  1233. * 'goal_run'), else 0 — the discriminator for the goal-context
  1234. * verb handling above.
  1235. */
  1236. private async int64 goal_id_of_run(int64 task_id) throws Error {
  1237. var task = yield db.query<TaskEntity>()
  1238. .where(expr("id == $0", elem<int64?>(task_id)))
  1239. .first_async();
  1240. return task != null && ((!)task).kind == TASK_KIND_GOAL_RUN
  1241. ? ((!)task).goal_id : 0;
  1242. }
  1243. /**
  1244. * Raises the daemon's pushed blocking prompt as a `task_blocks`
  1245. * row exactly once: a not-yet-answered row, or an answered row
  1246. * whose answer has not been delivered to the daemon yet, means the
  1247. * prompt is already represented. The results push's `block`
  1248. * object carries the question plus the block type, options and
  1249. * markdown body (the `/status` field names — `block_question`
  1250. * &c — are accepted as a fallback). `saddle block rfc` carries
  1251. * no question text, so rfc prompts surface under a stable
  1252. * fallback title.
  1253. */
  1254. private async void ensure_block(TaskService tasks, int64 task_id, Json.Object block) throws Error {
  1255. string question = block_question_from_payload(block);
  1256. if (question.length == 0) {
  1257. return;
  1258. }
  1259. string incoming_type = block_string_or(block, "type", "block_type");
  1260. foreach (var existing in yield tasks.list_blocks(task_id)) {
  1261. if (existing.answer == null || !poll_state.is_delivered(existing.id)) {
  1262. // A pending review-family block of EITHER kind
  1263. // (review/completion) swallows the other's push: the
  1264. // daemon's done park and the web's completion raise
  1265. // are the SAME review raised from two paths — one
  1266. // row, whichever landed first.
  1267. if ((incoming_type == BLOCK_REVIEW || incoming_type == BLOCK_COMPLETION)
  1268. && (existing.block_type == BLOCK_REVIEW
  1269. || existing.block_type == BLOCK_COMPLETION)) {
  1270. return;
  1271. }
  1272. }
  1273. if (existing.question != question) {
  1274. continue;
  1275. }
  1276. if (existing.answer == null || !poll_state.is_delivered(existing.id)) {
  1277. return;
  1278. }
  1279. }
  1280. yield tasks.add_block(task_id, question,
  1281. block_string_or(block, "type", "block_type"),
  1282. EntryJson.string_array_member(block, block.has_member("options") ? "options" : "block_options"),
  1283. EntryJson.optional_string_member(block, block.has_member("body") ? "body" : "block_body"));
  1284. }
  1285. /** The first non-empty of two string members (empty when both are). */
  1286. private static string block_string_or(Json.Object obj, string primary, string fallback) {
  1287. string value = EntryJson.string_member(obj, primary);
  1288. return value.length > 0 ? value : EntryJson.string_member(obj, fallback);
  1289. }
  1290. /**
  1291. * The question a daemon block is recognised by: `question`
  1292. * (or the `/status`-style `block_question`) verbatim, or the
  1293. * rfc fallback when the daemon raised an RFC (the
  1294. * `saddle block rfc` shape carries its document as the body and
  1295. * no question operand). Both the block-raising and answer-
  1296. * delivery paths key blocks by this string.
  1297. */
  1298. private static string block_question_from_payload(Json.Object block) {
  1299. string question = block_string_or(block, "question", "block_question");
  1300. if (question.length > 0) {
  1301. return question;
  1302. }
  1303. return block_string_or(block, "type", "block_type") == BLOCK_RFC
  1304. ? "Request for comments" : "";
  1305. }
  1306. /**
  1307. * Applies a pushed state transition (`POST /api/saddle/results`,
  1308. * plan 10 phase 2) — the same task/saddle bookkeeping the
  1309. * deleted /status poll switch ran, driven by the daemon's push:
  1310. * `done`/`failed` complete or fail the task with the pushed
  1311. * summary (closing the session through the release owner) —
  1312. * except on a goal-run session, where they submit the
  1313. * completion report or fail the goal (plan 12); `blocked`
  1314. * raises the block payload as an answerable block on the
  1315. * goal run's OWN task row (plan 15 C1: the goal agent's
  1316. * blocks ride the run task — no mirrored second row exists)
  1317. * and parks the round (the raise closes its session 'parked',
  1318. * D4); `suspended`
  1319. * parks the same way — for a goal agent the resume is driven
  1320. * by the goal's task-terminal events; and `working` flips an
  1321. * ASSIGNED task to working (a parked task keeps its park —
  1322. * the verdict arrives as a resuming allocation, never as a
  1323. * state push; plan 16 D4). No saddle-row write remains
  1324. * (plan 16 item 23): busy/idle derive from the sessions.
  1325. *
  1326. * Plan 13 B2: a push against a task that is already terminal —
  1327. * or a goal-context push against a finished goal — is an
  1328. * outbox redelivery and is consumed silently, whatever state
  1329. * it carries; terminal transitions and their narrative fire
  1330. * once, ever.
  1331. */
  1332. public async void apply_result_state(ClankerSaddle.Data.Session session,
  1333. string state, Json.Object? block, string? summary, TaskService tasks) throws Error {
  1334. int64 task_id = session.task_id;
  1335. int64 goal_id = yield goal_id_of_run(task_id);
  1336. bool terminal = yield task_is_terminal(task_id);
  1337. if (!terminal && goal_id > 0) {
  1338. terminal = yield goal_is_terminal(goal_id);
  1339. }
  1340. if (terminal) {
  1341. return;
  1342. }
  1343. switch (state) {
  1344. case "done":
  1345. if (goal_id > 0) {
  1346. // Goal context: `saddle done` submits the
  1347. // completion report and enters completion
  1348. // review — the run parks on the mirrored
  1349. // block (submit_completion's raise already
  1350. // closed the session 'parked'), it does not
  1351. // complete.
  1352. yield goal_service.submit_completion(goal_id,
  1353. summary != null && summary.length > 0 ? summary
  1354. : "The goal agent submitted its completion report");
  1355. break;
  1356. }
  1357. yield tasks.complete_task(task_id, summary, OUTCOME_DONE);
  1358. break;
  1359. case "failed":
  1360. string explanation = summary ?? "";
  1361. if (goal_id > 0) {
  1362. yield goal_service.fail_goal(goal_id,
  1363. explanation.length > 0 ? explanation : null);
  1364. break;
  1365. }
  1366. yield tasks.fail_task(task_id, explanation.length > 0 ? explanation : null);
  1367. break;
  1368. case "blocked":
  1369. if (goal_id > 0) {
  1370. // Goal context: the block lands on the run
  1371. // task itself (plan 15 C1) — a plan payload
  1372. // enters plan review, anything else raises a
  1373. // goal question/RFC; the run parks on that
  1374. // very row for the verdict resume. A BLOCKED
  1375. // push WITHOUT a
  1376. // block payload (the park-unwinding round-end
  1377. // push, its trailing entries) raises nothing —
  1378. // the payload-carrying transition already did,
  1379. // and a synthesised fallback row would dupe it.
  1380. if (block == null) {
  1381. yield tasks.set_status(task_id, STATUS_BLOCKED);
  1382. yield tasks.park_open_sessions(task_id);
  1383. break;
  1384. }
  1385. string question = block_question_from_payload((!)block);
  1386. if (question.length == 0) {
  1387. question = "The goal agent raised a question";
  1388. }
  1389. if (block_string_or((!)block, "type", "block_type") == BLOCK_PLAN) {
  1390. yield goal_service.submit_plan(goal_id,
  1391. EntryJson.optional_string_member((!)block, "body") ?? question);
  1392. } else if (block_string_or((!)block, "type", "block_type") == BLOCK_REVIEW) {
  1393. // The daemon's submit_report done-park: a
  1394. // review-kind block on the goal run IS the
  1395. // completion submission (submit_completion
  1396. // adopts the raised row — one row, whichever
  1397. // path landed first — and drives the
  1398. // completion_review transition).
  1399. yield goal_service.submit_completion(goal_id,
  1400. EntryJson.optional_string_member((!)block, "body") ?? question);
  1401. } else {
  1402. yield goal_service.raise_agent_block(goal_id, question,
  1403. block_string_or((!)block, "type", "block_type"),
  1404. EntryJson.string_array_member((!)block,
  1405. (!)block.has_member("options") ? "options" : "block_options"),
  1406. EntryJson.optional_string_member((!)block, "body"));
  1407. }
  1408. yield tasks.set_status(task_id, STATUS_BLOCKED);
  1409. break;
  1410. }
  1411. yield ensure_block(tasks, task_id, block != null ? (!)block : new Json.Object());
  1412. {
  1413. // A review-kind block parks as review_requested
  1414. // (the banner state); every other kind as
  1415. // blocked.
  1416. bool review = block != null
  1417. && block_string_or((!)block, "type", "block_type") == BLOCK_REVIEW;
  1418. yield tasks.set_status(task_id,
  1419. review ? STATUS_REVIEW_REQUESTED : STATUS_BLOCKED);
  1420. }
  1421. // The daemon parks the task's workspace while
  1422. // blocked, freeing the saddle for other work
  1423. // (plan 16 item 23: the raise already closed the
  1424. // round's session 'parked' and the task's blocked
  1425. // status IS the pause — no saddle-row write
  1426. // remains; busy/idle derive from the sessions).
  1427. break;
  1428. case "suspended":
  1429. // `saddle suspend`: the same parking as a block, but
  1430. // the web drives resumption — a wake of the affine
  1431. // saddle once a goal event lands, the resume
  1432. // assignment carrying the coalesced report.
  1433. // (set_parked closed the round's session 'parked';
  1434. // the freed saddle is the derived view's reading.)
  1435. yield tasks.set_parked(task_id);
  1436. break;
  1437. case "working":
  1438. // A working push only advances a task that is
  1439. // actually running — assigned (the claim's flip)
  1440. // or a legacy never-allocated queued row. A
  1441. // PARKED task (blocked / review / suspended, its
  1442. // session closed at the raise — plan 16 D4) keeps
  1443. // its park: the verdict arrives as a resuming
  1444. // allocation, never as a state push, so a
  1445. // park-unwinding or late round push can never
  1446. // resurrect the workspace as "working" out from
  1447. // under its review banner.
  1448. var advance = connection.create_command(
  1449. "UPDATE tasks SET status = 'working' WHERE id = :id"
  1450. + " AND status IN ('assigned', 'queued')"
  1451. + " AND status != 'working'");
  1452. advance.with_parameter<Invercargill.Element>("id",
  1453. elem<int64?>(task_id));
  1454. bool flipped = (yield advance.execute_non_query_async()) > 0;
  1455. var current = yield db.query<TaskEntity>()
  1456. .where(expr("id == $0", elem<int64?>(task_id)))
  1457. .first_async();
  1458. if (flipped) {
  1459. var moved = new Json.Object();
  1460. moved.set_string_member("from", STATUS_ASSIGNED);
  1461. moved.set_string_member("to", STATUS_WORKING);
  1462. moved.set_string_member("text", "Status → working");
  1463. yield event_service.append(EVENT_STATE_CHANGE,
  1464. EVENT_ACTOR_SYSTEM, EVENT_SCOPE_TASK, task_id, moved);
  1465. }
  1466. // started_at (and the first-start push) only stamp
  1467. // for work that never allocated; the guarded
  1468. // advance above means the re-read task is working
  1469. // exactly when the flip landed or it already ran.
  1470. if (current != null && ((!)current).status == STATUS_WORKING
  1471. && ((!)current).started_at == null) {
  1472. ((!)current).started_at = new DateTime.now_local();
  1473. yield db.update_async<TaskEntity>((!)current);
  1474. // The first-start push lives at the allocation
  1475. // (where started_at is stamped for dispatched
  1476. // tasks); this path only covers legacy status
  1477. // pushes for never-allocated work.
  1478. push.notify_opted_in(task_id, PushService.lifecycle_notification(
  1479. ((!)current).title, task_id, "started", null));
  1480. }
  1481. break;
  1482. default:
  1483. break;
  1484. }
  1485. }
  1486. /** True when the task's status is done or failed. */
  1487. private async bool task_is_terminal(int64 task_id) throws Error {
  1488. var task = yield db.query<TaskEntity>()
  1489. .where(expr("id == $0", elem<int64?>(task_id)))
  1490. .first_async();
  1491. return task != null && (((!)task).status == STATUS_DONE
  1492. || ((!)task).status == STATUS_FAILED);
  1493. }
  1494. /** True when the goal's status is complete, failed or cancelled. */
  1495. private async bool goal_is_terminal(int64 goal_id) throws Error {
  1496. var goal = yield db.query<Goal>()
  1497. .where(expr("id == $0", elem<int64?>(goal_id)))
  1498. .first_async();
  1499. return goal != null && (((!)goal).status == GOAL_COMPLETE
  1500. || ((!)goal).status == GOAL_FAILED || ((!)goal).status == GOAL_CANCELLED);
  1501. }
  1502. /**
  1503. * The running-liveness probe (plan 10 phase 2): a busy saddle's
  1504. * session has seen neither chunks nor results for the staleness
  1505. * window, so ONE exceptional `GET /status` decides its fate —
  1506. * an answer means the daemon is alive (the health snapshot is
  1507. * refreshed and the task keeps running; the silence episode
  1508. * gets no second probe), a failure requeues the task with an agent
  1509. * comment and marks the saddle offline (the normal unreachable
  1510. * bookkeeping). The probe flag was taken by the caller; the
  1511. * next push resets it.
  1512. */
  1513. public async void probe_stale_saddle(Saddle saddle, TaskService tasks,
  1514. int64 running_task_id) throws Error {
  1515. try {
  1516. var status = yield request_saddle_json(saddle.host, "GET", "/status", null);
  1517. yield record_status_probe(saddle, status);
  1518. // An answer proves the DAEMON lives - not that it is
  1519. // running THIS task. A daemon that crashed and
  1520. // restarted answers from its idle loop (task_id null,
  1521. // or another task it won since): the stale session is
  1522. // a crash orphan, and its task requeues through the
  1523. // same guarded bookkeeping as the unreachable path
  1524. // (the comment names the restart). The saddle itself
  1525. // stays as-is - it is demonstrably alive and asking
  1526. // for work; stamping it offline would only block the
  1527. // requeued task's next claim until another ask.
  1528. var answered_task = status.has_member("task_id")
  1529. && status.get_member("task_id").get_node_type() == Json.NodeType.VALUE
  1530. && status.get_member("task_id").get_value_type() == typeof(int64)
  1531. ? status.get_int_member("task_id") : 0;
  1532. if (answered_task != running_task_id) {
  1533. var flip = connection.create_command(
  1534. "UPDATE tasks SET status = 'queued' WHERE id = :id"
  1535. + " AND status IN ('assigned', 'working')");
  1536. flip.with_parameter<Invercargill.Element>("id",
  1537. elem<int64?>(running_task_id));
  1538. if ((yield flip.execute_non_query_async()) > 0) {
  1539. yield tasks.add_system_comment(running_task_id,
  1540. @"Saddle '$(saddle.name)' restarted mid-run (its status probe answered without the task); task requeued");
  1541. yield tasks.release_task(running_task_id, OUTCOME_STOPPED, false);
  1542. }
  1543. yield pulse.bump(0, "crash orphan requeued at probe");
  1544. print(@"[ClankerSaddle] Stale saddle '$(saddle.name)' answered its probe without task $(running_task_id); requeued\n");
  1545. return;
  1546. }
  1547. print(@"[ClankerSaddle] Stale saddle '$(saddle.name)' answered its exceptional status probe; task $(running_task_id) keeps running\n");
  1548. } catch (Error e) {
  1549. warning(@"[ClankerSaddle] Saddle '$(saddle.name)' sent neither chunks nor results for $(stale_after_seconds())s and its status probe failed ($(e.message)); marking offline");
  1550. yield mark_saddle_unreachable(saddle.id, tasks,
  1551. @"no chunks or results for $(stale_after_seconds())s; status probe failed ($(e.message))");
  1552. }
  1553. }
  1554. /**
  1555. * Performs an HTTP request against a saddle daemon and parses the
  1556. * response body as JSON; transport failures propagate to the
  1557. * caller (the dispatcher marks the saddle unreachable). Transport
  1558. * blips — typically a pooled keep-alive connection closed by the
  1559. * daemon at the same instant the request reused it — are retried
  1560. * once before giving up. Unlike {@link request_saddle_json} the
  1561. * HTTP status is returned rather than thrown, and array bodies
  1562. * parse too.
  1563. */
  1564. private async int request_saddle(string host, string method, string path,
  1565. owned Json.Builder? payload, out Json.Node? root) throws Error {
  1566. root = null;
  1567. GLib.Error? failure = null;
  1568. for (var attempt = 0; attempt < 2; attempt++) {
  1569. try {
  1570. root = null;
  1571. int status = yield request_saddle_once(host, method, path, payload, out root);
  1572. return status;
  1573. } catch (Error e) {
  1574. failure = e;
  1575. }
  1576. if (((!)failure).code != 13 && ((!)failure).code != 14) {
  1577. break;
  1578. }
  1579. GLib.Timeout.add(250, request_saddle.callback);
  1580. yield;
  1581. }
  1582. throw (!)failure;
  1583. }
  1584. private async int request_saddle_once(string host, string method, string path,
  1585. Json.Builder? payload, out Json.Node? root) throws Error {
  1586. root = null;
  1587. string base_url = host.has_prefix("http") ? host : @"http://$(host)";
  1588. var message = new Soup.Message(method, @"$(base_url)$(path)");
  1589. if (payload != null) {
  1590. var generator = new Json.Generator();
  1591. generator.set_root(((!)payload).get_root());
  1592. message.set_request_body_from_bytes("application/json",
  1593. new Bytes(generator.to_data(null).data));
  1594. }
  1595. InputStream input;
  1596. try {
  1597. input = yield http.send_async(message, Priority.DEFAULT, null);
  1598. } catch (Error e) {
  1599. throw new Error(Quark.from_string("clanker-saddle"), 13,
  1600. @"Saddle request to $(base_url)$(path) failed: $(e.message)");
  1601. }
  1602. var output = new MemoryOutputStream.resizable();
  1603. try {
  1604. yield output.splice_async(input, OutputStreamSpliceFlags.CLOSE_SOURCE,
  1605. Priority.DEFAULT, null);
  1606. yield output.close_async(Priority.DEFAULT, null);
  1607. } catch (Error e) {
  1608. throw new Error(Quark.from_string("clanker-saddle"), 14,
  1609. @"Failed reading saddle response: $(e.message)");
  1610. }
  1611. var written = output.get_data_size();
  1612. uint8[] data = output.steal_data();
  1613. // steal_data's array length is -1 and the buffer is sized to
  1614. // the data — there is no NUL terminator to rely on — so slice
  1615. // by the written byte count before reading it as a string.
  1616. string response_text = written > 0 && data != null && data[0] != '\0'
  1617. ? ((string) data).substring(0, (long) written) : "{}";
  1618. var parser = new Json.Parser();
  1619. try {
  1620. parser.load_from_data(response_text);
  1621. } catch (Error e) {
  1622. throw new Error(Quark.from_string("clanker-saddle"), 16,
  1623. @"Saddle returned invalid JSON: $(e.message)");
  1624. }
  1625. root = parser.get_root();
  1626. return (int) message.status_code;
  1627. }
  1628. /**
  1629. * Performs an HTTP request against a saddle daemon and returns the
  1630. * parsed JSON object body (an empty object for non-object bodies);
  1631. * HTTP statuses of 400 and above throw.
  1632. */
  1633. private async Json.Object request_saddle_json(string host, string method, string path,
  1634. owned Json.Builder? payload) throws Error {
  1635. Json.Node? root = null;
  1636. int status_code = yield request_saddle(host, method, path, (owned) payload, out root);
  1637. if (status_code == 409) {
  1638. // Machine-readable per the wire contract: the body (state,
  1639. // running_task_id) rides in the message so the dispatch
  1640. // path can distinguish zombie conflicts from transport
  1641. // failures and recover.
  1642. string body_text = "{}";
  1643. if (root != null && root.get_node_type() == Json.NodeType.OBJECT) {
  1644. var generator = new Json.Generator();
  1645. generator.set_root(root);
  1646. body_text = generator.to_data(null);
  1647. }
  1648. throw new Error(Quark.from_string("clanker-saddle"), 409,
  1649. @"busy: $(body_text)");
  1650. }
  1651. if (status_code >= 400) {
  1652. throw new Error(Quark.from_string("clanker-saddle"), 15,
  1653. @"Saddle request returned HTTP $(status_code)");
  1654. }
  1655. if (root == null || root.get_node_type() != Json.NodeType.OBJECT) {
  1656. return new Json.Object();
  1657. }
  1658. return root.get_object();
  1659. }
  1660. /**
  1661. * A `{"name", "git_url", "path", "branch"}` project reference in
  1662. * the saddle daemon's wire shape: `git_url` is the clone source,
  1663. * and the checkout lives at
  1664. * `SADDLE_PROJECTS_ROOT/task-<task_id>/<name>` inside the
  1665. * saddle's container — one workspace per task, so concurrent
  1666. * tasks never share a checkout.
  1667. */
  1668. /**
  1669. * The projects root assignment paths are addressed against: the
  1670. * ASKING saddle's registered `workspace_root` (the daemon's own
  1671. * SADDLE_WORKSPACE_ROOT, reported at /register — goal 8 task 6,
  1672. * so a fleet no longer assumes one shared /workspace mount),
  1673. * else the `SADDLE_PROJECTS_ROOT` environment override, else
  1674. * the historical "/workspace" default. Paths are ADVISORY: the
  1675. * daemon owns its filesystem and rewrites them onto its own
  1676. * workspace root.
  1677. */
  1678. private static string projects_root_for_saddle(Saddle? saddle) {
  1679. var registered = saddle != null ? ((!)saddle).workspace_root : null;
  1680. if (registered != null && ((!)registered).strip().length > 0) {
  1681. return ((!)registered).strip();
  1682. }
  1683. var root_var = (Environment.get_variable("SADDLE_PROJECTS_ROOT") ?? "").strip();
  1684. return root_var.length > 0 ? root_var : "/workspace";
  1685. }
  1686. private static Json.Node project_node(Project project, int64 task_id, bool is_reference = false,
  1687. Saddle? saddle = null) {
  1688. string projects_root = projects_root_for_saddle(saddle);
  1689. var builder = new Json.Builder();
  1690. builder.begin_object();
  1691. builder.set_member_name("name");
  1692. builder.add_string_value(project.name);
  1693. builder.set_member_name("git_url");
  1694. builder.add_string_value(project.git_url);
  1695. builder.set_member_name("path");
  1696. builder.add_string_value(is_reference
  1697. ? @"$(projects_root)/task-$(task_id)/reference/$(project.name)"
  1698. : @"$(projects_root)/task-$(task_id)/$(project.name)");
  1699. builder.set_member_name("branch");
  1700. builder.add_string_value(project.default_branch);
  1701. builder.end_object();
  1702. return builder.get_root();
  1703. }
  1704. /**
  1705. * The answer as the saddle daemon expects it over /answer (or in a
  1706. * resume payload): verbatim. The daemon applies its own
  1707. * BlockRequest formatting, so this only strips the "Human
  1708. * selected: …"/"Human answered (other): …" prefixes still stored
  1709. * by answers recorded before the daemon took over formatting.
  1710. */
  1711. private static string raw_answer_for_saddle(string answer) {
  1712. if (answer.has_prefix("Human selected: ")) {
  1713. return answer.substring("Human selected: ".length);
  1714. }
  1715. if (answer.has_prefix("Human answered (other): ")) {
  1716. return answer.substring("Human answered (other): ".length);
  1717. }
  1718. return answer;
  1719. }
  1720. }
  1721. }