| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031 |
- using Invercargill;
- using Invercargill.DataStructures;
- using Invercargill.Expressions;
- using InvercargillSql;
- using InvercargillSql.Orm;
- using Inversion;
- using Soup;
- using ClankerSaddle.Data;
- namespace ClankerSaddle.Services {
- /**
- * Valid saddle statuses (the `saddles.status` wire values, plan 16
- * R2 / arch item 26 — the raw literals died with these). The row
- * default and the migration seed keep their literals (the Data
- * layer does not reference the services).
- */
- public const string SADDLE_IDLE = "idle";
- public const string SADDLE_BUSY = "busy";
- public const string SADDLE_OFFLINE = "offline";
- /**
- * Session outcomes (the `sessions.outcome` values the park, the
- * ONE release path and the terminal pushes write): `parked` (a
- * round that parked on a block or a suspend — the row closes at
- * park time and the workspace awaits its resuming allocation;
- * plan 16 D4, aligned with the daemon's file-log brackets),
- * `stopped` (a cancelled/requeued/unreachable release),
- * `done`/`failed` (the terminal pushes).
- */
- public const string OUTCOME_STOPPED = "stopped";
- public const string OUTCOME_PARKED = "parked";
- public const string OUTCOME_DONE = "done";
- public const string OUTCOME_FAILED = "failed";
- /** Caps of the `project_notes` array in the /start payload. */
- public const int64 PROJECT_NOTES_MAX_ENTRIES = 30;
- public const int64 PROJECT_NOTES_MAX_CHARS = 20000;
- /**
- * Cap of the `understanding` member in the /start payload: the
- * primary project's current understanding document (goal 24).
- * The whole document rides the payload and renders into the
- * system prompt next to the project notes, so an over-grown
- * revision is head-clamped rather than bloating every prompt.
- */
- public const int64 UNDERSTANDING_MAX_CHARS = 16000;
- /**
- * Caps of the `discoveries` array in the /start payload (goal 24):
- * the fleet's newest discoveries — non-project facts earlier
- * agents recorded — so a fresh agent inherits the knowledge base
- * before it ever searches. Head-kept char cap across the whole
- * array, like PROJECT_NOTES_MAX_CHARS.
- */
- public const int64 DISCOVERIES_MAX_ENTRIES = 10;
- public const int64 DISCOVERIES_MAX_CHARS = 8000;
- /**
- * Upper bound of a saddle's prompt_note served to the daemon (goal 8
- * task 5): the note rides EVERY assignment payload, so a fat note
- * would bloat every system prompt and cache.
- */
- public const int64 PROMPT_NOTE_MAX_CHARS = 2000;
- /** A block answer awaiting collection by one saddle's next poll. */
- public class PendingAnswer : Object {
- public int64 block_id { get; set; }
- public string answer { get; set; default = ""; }
- public bool branch_advanced { get; set; }
- }
- /** A saddle registration plus the API token generated for it. */
- public class SaddleRegistration : Object {
- public Saddle saddle { get; set; }
- public string token { get; set; default = ""; }
- }
- /**
- * A saddle's current work as the cards render it: the running
- * task's title, plus the GOAL it stands for when that task is a
- * goal agent's internal run.
- */
- public class CurrentWork : Object {
- public int64 task_id { get; set; }
- public string task_title { get; set; default = ""; }
- /** Set when the current task is a goal agent's run: the card links the goal. */
- public int64 goal_id { get; set; }
- public string goal_title { get; set; default = ""; }
- /** The running session the derivation keyed on (plan 16 item 23). */
- public int64 session_id { get; set; }
- /** When that session started — the "allocated" display stamp. */
- public DateTime? started_at { get; set; default = null; }
- }
- /**
- * The OpenAI-compatible endpoint every saddle assignment is sent to,
- * read from the environment: `SADDLE_LLM_BASE_URL` (required — dispatch
- * pauses without it), `SADDLE_LLM_API_KEY` and `SADDLE_LLM_MODEL`.
- */
- public class LlmEndpoint : Object {
- public string base_url { get; set; default = ""; }
- public string api_key { get; set; default = ""; }
- public string model { get; set; default = ""; }
- /** The configured endpoint, or null when the base URL is unset. */
- public static LlmEndpoint? from_environment() {
- var base_url = (Environment.get_variable("SADDLE_LLM_BASE_URL") ?? "").strip();
- if (base_url.length == 0) {
- return null;
- }
- return new LlmEndpoint() {
- base_url = base_url,
- api_key = (Environment.get_variable("SADDLE_LLM_API_KEY") ?? "").strip(),
- model = (Environment.get_variable("SADDLE_LLM_MODEL") ?? "").strip()
- };
- }
- }
- /**
- * Saddle registry and control-plane client (plan 10 phase 2): registers
- * containers, derives idle capacity for the dispatcher's wake
- * computation (plan 16 item 23: busy iff an open session on an
- * assigned/working task exists — {@link current_work_view} is the ONE
- * truth accessor; the saddle row keeps only the OFFLINE probe state),
- * wakes sleeping daemons, and ingests the telemetry the daemons PUSH —
- * raw PTY chunks (`POST /api/saddle/chunks`) and round results with
- * state transitions (`POST /api/saddle/results`) — reusing the storage
- * logic the deleted `/output` + `/log` pollers ran. GET /status
- * against a daemon is exceptional only: the staleness probe for a
- * silent runner and the manual refresh button. Block answers are
- * handed out exactly once (delivered ids are remembered in memory; a
- * restart re-delivers and the daemon deduplicates).
- */
- public class SaddleService : Object {
- private OrmSession db = inject<OrmSession>();
- private Connection connection = inject<Connection>();
- /**
- * Every daemon request shares this session and therefore its
- * timeouts: without them a single hung request (a daemon wedged
- * mid-response over the keep-alive pool) froze the dispatcher's
- * tick latch forever. `timeout` bounds each request
- * (SADDLE_HTTP_TIMEOUT_S, default 30s); `idle_timeout` reaps
- * pooled connections so a peer that vanished between requests is
- * not dialed again.
- */
- private Soup.Session http = new Soup.Session() {
- timeout = http_timeout_seconds(),
- idle_timeout = 10
- };
- private static uint http_timeout_seconds() {
- var raw = Environment.get_variable("SADDLE_HTTP_TIMEOUT_S") ?? "";
- int64 parsed = int64.parse(raw);
- return parsed > 0 ? (uint) parsed : 30;
- }
- private SaddlePollState poll_state = inject<SaddlePollState>();
- private StatePulse pulse = inject<StatePulse>();
- private PushService push = inject<PushService>();
- private RuleService rules = inject<RuleService>();
- private CapabilityService capability_service = inject<CapabilityService>();
- private TaskService task_service = inject<TaskService>();
- private GoalService goal_service = inject<GoalService>();
- private EventService event_service = inject<EventService>();
- private UnderstandingService understandings = inject<UnderstandingService>();
- private DiscoveryService discoveries = inject<DiscoveryService>();
- private Astralis.WebConfig config = inject<Astralis.WebConfig>();
- /**
- * Registers a saddle container: inserts the row with a freshly
- * minted per-saddle token (plan 10) and hands the credential to
- * the daemon via `POST /register {saddle_token, web_base_url}`
- * (best effort — an older daemon without the route stays
- * registered; the operator copies the token from the saddles
- * page instead). `host` is host:port of its control server; the
- * returned {@link SaddleRegistration} carries the token the
- * daemon must present as `Authorization: Bearer …` on
- * `/api/saddle/*` calls.
- */
- public async SaddleRegistration register_saddle(string name, string host) throws Error {
- var saddle = new Saddle() {
- name = name,
- host = host,
- status = SADDLE_IDLE,
- token = mint_saddle_token()
- };
- yield db.insert_async<Saddle>(saddle);
- yield register_daemon(saddle);
- yield pulse.bump(0, "saddle registered");
- return new SaddleRegistration() { saddle = saddle, token = saddle.token ?? "" };
- }
- /** A fresh per-saddle bearer token (`saddle-<random>`). */
- private static string mint_saddle_token() {
- return @"saddle-$(Uuid.string_random().replace("-", ""))";
- }
- /**
- * Delivers the per-saddle token to the daemon's
- * `POST /register` endpoint (wire contract): the daemon
- * persists it for its calls home and answers
- * `{ok, ssh_public_key, version}` — the key is stored on the
- * row the moment the daemon reports one. A daemon without the
- * route (HTTP 404/405) or unreachable at registration time is
- * tolerated: the row keeps the token, the operator can rety
- * via a refresh or copy the token manually.
- */
- public async void register_daemon(Saddle saddle) throws Error {
- var payload = new Json.Builder();
- payload.begin_object();
- payload.set_member_name("saddle_token");
- payload.add_string_value(saddle.token ?? "");
- payload.set_member_name("web_base_url");
- payload.add_string_value(callback_base_url());
- payload.end_object();
- try {
- var response = yield request_saddle_json(saddle.host, "POST", "/register",
- (owned) payload);
- string reported_key = EntryJson.string_member(response, "ssh_public_key");
- // goal 8 task 6: the daemon reports the workspace root it
- // mounts so assignment paths address the ASKING saddle's
- // filesystem; absent on older daemons (kept null → the
- // SADDLE_PROJECTS_ROOT fallback chain)
- string reported_root = EntryJson.string_member(response, "workspace_root");
- if ((reported_key.length > 0 && reported_key != (saddle.ssh_public_key ?? ""))
- || reported_root != (saddle.workspace_root ?? "")) {
- if (reported_key.length > 0) {
- saddle.ssh_public_key = reported_key;
- }
- saddle.workspace_root = reported_root.length > 0 ? reported_root : null;
- yield db.update_async<Saddle>(saddle);
- }
- print(@"[ClankerSaddle] Registered with saddle '$(saddle.name)'s daemon (token delivered)\n");
- } catch (Error e) {
- print(@"[ClankerSaddle] Saddle '$(saddle.name)'s daemon did not accept /register ($(e.message)); token stays on the row\n");
- }
- }
- /**
- * The saddle a bearer token belongs to, or null — the
- * credential check for the id-less `/api/saddle/*` endpoints
- * (`next-task`, `note`, `task`): the per-saddle row token,
- * the single credential store since plan 10 (arch 24 dropped
- * the legacy api_tokens fallback — nothing deployed predates
- * the row token). Removing the saddle revokes it.
- */
- public async Saddle? saddle_for_token(string token) throws Error {
- if (token.length == 0) {
- return null;
- }
- return yield db.query<Saddle>()
- .where(expr("token == $0", elem(token)))
- .first_async();
- }
- /**
- * This app's externally reachable base URL — what daemons use to
- * POST notes back — read from the `saddle_callback_base_url` key
- * of web-config.json, falling back to the
- * `SADDLE_CALLBACK_BASE_URL` environment variable and then to
- * `http://localhost:8080`. The daemon typically runs in a
- * container or on another host, so its view of this app differs
- * from the browser's; set the key to whatever reaches this
- * process from there.
- */
- public string callback_base_url() {
- string configured = config.get_string("saddle_callback_base_url", "").strip();
- if (configured.length > 0) {
- return configured;
- }
- var from_env = (Environment.get_variable("SADDLE_CALLBACK_BASE_URL") ?? "").strip();
- if (from_env.length > 0) {
- return from_env;
- }
- return "http://localhost:8080";
- }
- /**
- * Removes a saddle registration; the task it was running (the
- * DERIVED running work — plan 16 item 23: sessions are the
- * truth) releases through the ONE path — back to the queue,
- * its open sessions closed stopped (no orphan open session
- * outlives the row). Revocation: the row dies here, and its
- * per-saddle token with it — nothing authorises
- * `/api/saddle/*` for it afterwards.
- */
- public async void unregister_saddle(int64 id) throws Error {
- var saddle = yield get_saddle(id);
- if (saddle == null) {
- return;
- }
- var work = yield current_work_view((!)saddle);
- if (work != null) {
- yield task_service.release_task(((!)work).task_id, OUTCOME_STOPPED, true);
- }
- yield db.delete_async<Saddle>((!)saddle);
- yield pulse.bump(0, "saddle removed");
- }
- /** Loads the saddle identified by `id`, or null. */
- public async Saddle? get_saddle(int64 id) throws Error {
- return yield db.query<Saddle>()
- .where(expr("id == $0", elem<int64?>(id)))
- .first_async();
- }
- /** Lists every saddle ordered by name. */
- public async ImmutableLot<Saddle> list_saddles() throws Error {
- return yield db.query<Saddle>()
- .order_by(expr("name"))
- .materialise_async();
- }
- /**
- * The saddle's current work resolved from the SESSION records —
- * the ONE truth accessor every surface reads (plan 16 item 23:
- * sessions own "who runs what"; the saddle row's cache columns
- * are gone). The running session is the newest open session on
- * the saddle whose task is still assigned/working — busy iff it
- * exists, idle otherwise; a parked task's session closed
- * 'parked' at its park (D4), so parks read as free exactly
- * like the daemon's file-log brackets say. The task join also
- * makes the derivation drift-tolerant: a stale open row on a
- * task that has since moved on cannot wedge a saddle busy.
- * Null when the saddle holds no running work.
- */
- public async CurrentWork? current_work_view(Saddle saddle) throws Error {
- var session = yield running_session_on_saddle(saddle.id);
- if (session == null) {
- return null;
- }
- var work = new CurrentWork() {
- task_id = ((!)session).task_id,
- session_id = ((!)session).id,
- started_at = ((!)session).started_at
- };
- var task = yield db.query<TaskEntity>()
- .where(expr("id == $0", elem<int64?>(((!)session).task_id)))
- .first_async();
- if (task == null) {
- return work;
- }
- work.task_title = ((!)task).title;
- if (((!)task).kind == TASK_KIND_GOAL_RUN) {
- var goal = yield db.query<Goal>()
- .where(expr("id == $0", elem<int64?>(((!)task).goal_id)))
- .first_async();
- if (goal != null) {
- work.goal_id = ((!)goal).id;
- work.goal_title = ((!)goal).title;
- }
- }
- return work;
- }
- /**
- * The derived busy/idle/offline status label every saddle view
- * shares — 'busy' iff {@link current_work_view} holds running
- * work, else the row's stored OFFLINE probe state ('offline' or
- * 'idle').
- */
- public static string derived_status(Saddle saddle, CurrentWork? work) {
- if (work != null) {
- return SADDLE_BUSY;
- }
- return saddle.status == SADDLE_OFFLINE ? SADDLE_OFFLINE : SADDLE_IDLE;
- }
- /**
- * The saddle's RUNNING session — the newest open session whose
- * task is assigned/working (the null check runs in SQL: the
- * expression layer cannot compare columns against null). At
- * most one exists per saddle by construction: every park closes
- * its row (D4) and the claim's guard refuses a saddle that
- * already holds one.
- */
- private async ClankerSaddle.Data.Session? running_session_on_saddle(int64 saddle_id)
- throws Error {
- var command = connection.create_command(
- "SELECT s.id AS session_id, s.task_id AS task_id, s.started_at AS started"
- + " FROM sessions s JOIN tasks t ON t.id = s.task_id"
- + " WHERE s.saddle_id = :saddle AND s.ended_at IS NULL"
- + " AND t.status IN ('assigned', 'working')"
- + " ORDER BY s.id DESC LIMIT 1");
- command.with_parameter<Invercargill.Element>("saddle",
- elem<int64?>(saddle_id));
- ClankerSaddle.Data.Session? found = null;
- foreach (var row in yield command.execute_query_async()) {
- int64? session_id = 0;
- int64? task_id = 0;
- ((!)row.get("session_id")).try_get_as<int64?>(out session_id);
- ((!)row.get("task_id")).try_get_as<int64?>(out task_id);
- found = new ClankerSaddle.Data.Session() {
- id = session_id ?? 0,
- task_id = task_id ?? 0,
- saddle_id = saddle_id,
- started_at = raw_date_time_of(row.get("started"))
- };
- }
- return found;
- }
- /** A raw query column back as a DateTime (null when null/unreadable). */
- private static DateTime? raw_date_time_of(Invercargill.Element? element) {
- if (element == null) {
- return null;
- }
- DateTime value;
- return ((!)element).try_get_as<DateTime>(out value) ? value : null;
- }
- /**
- * The ids (decimal strings — the OR-chain batch convention) of
- * every saddle holding a running session — the derived busy
- * set the fleet lists and the offline stamp share.
- */
- private async Invercargill.DataStructures.HashSet<string> busy_saddle_ids()
- throws Error {
- var busy = new Invercargill.DataStructures.HashSet<string>();
- var command = connection.create_command(
- "SELECT DISTINCT s.saddle_id AS saddle_id FROM sessions s"
- + " JOIN tasks t ON t.id = s.task_id"
- + " WHERE s.ended_at IS NULL"
- + " AND t.status IN ('assigned', 'working')");
- foreach (var row in yield command.execute_query_async()) {
- int64? saddle_id = 0;
- ((!)row.get("saddle_id")).try_get_as<int64?>(out saddle_id);
- if (saddle_id != null) {
- busy.add(((!)saddle_id).to_string());
- }
- }
- return busy;
- }
- /**
- * Saddles holding a running session (the dispatcher's poll
- * candidates) — derived from the session records, registration
- * order.
- */
- public async ImmutableLot<Saddle> list_busy_saddles() throws Error {
- var busy = yield busy_saddle_ids();
- if (busy.count() == 0) {
- return (new Vector<Saddle>()).to_immutable_buffer();
- }
- var rows = new Vector<Saddle>();
- foreach (var saddle in yield db.query<Saddle>()
- .where(TaskService.id_in("id", busy))
- .order_by(expr("id"))
- .materialise_async()) {
- rows.add(saddle);
- }
- return rows.to_immutable_buffer();
- }
- /**
- * Persists the saddle's git identity — what its daemon commits
- * as (`git_identity` in every /start). Empty/null clears the
- * fields so the daemon's fallback applies.
- */
- public async void update_git_identity(int64 id, string? git_name,
- string? git_email) throws Error {
- var saddle = yield get_saddle(id);
- if (saddle == null) {
- throw new Error(Quark.from_string("clanker-saddle"), 17,
- @"No saddle exists with id $(id)");
- }
- ((!)saddle).git_name = git_name != null && git_name.length > 0 ? git_name : null;
- ((!)saddle).git_email = git_email != null && git_email.length > 0 ? git_email : null;
- yield db.update_async<Saddle>((!)saddle);
- yield pulse.bump(0, "saddle identity saved");
- }
- /**
- * Saves the saddle admin's standing prompt note (goal 8 task 5):
- * the message about THIS machine rendered into every
- * assignment's system prompt. Empty clears it (the daemon then
- * omits the section entirely); the value is clamped to
- * {@link PROMPT_NOTE_MAX_CHARS} so a fat note can never bloat
- * every prompt.
- */
- public async void update_prompt_note(int64 id, string? prompt_note) throws Error {
- var saddle = yield get_saddle(id);
- if (saddle == null) {
- throw new Error(Quark.from_string("clanker-saddle"), 17,
- @"No saddle exists with id $(id)");
- }
- var trimmed = prompt_note != null ? prompt_note.strip() : "";
- if (trimmed.length > PROMPT_NOTE_MAX_CHARS) {
- trimmed = trimmed.substring(0, (int) PROMPT_NOTE_MAX_CHARS);
- }
- ((!)saddle).prompt_note = trimmed.length > 0 ? trimmed : null;
- yield db.update_async<Saddle>((!)saddle);
- yield pulse.bump(0, "saddle prompt note saved");
- }
- /**
- * Persists the daemon's reported `ssh_public_key` onto the saddle
- * row (the plan-9 contract: the key travels on /status, empty
- * until the daemon generates one; the saddles page shows it for
- * adding as a deploy key). A single-column UPDATE — the
- * full-row write could stomp a concurrent ask claim. Only writes
- * when it changed.
- */
- private async void store_reported_ssh_key(Saddle saddle, Json.Object status) throws Error {
- if (!status.has_member("ssh_public_key")) {
- return;
- }
- string reported = status.get_string_member_with_default("ssh_public_key", "");
- string stored = saddle.ssh_public_key ?? "";
- if (reported == stored) {
- return;
- }
- var command = connection.create_command(
- "UPDATE saddles SET ssh_public_key = :key WHERE id = :id");
- command.with_parameter<Invercargill.Element>("key", elem(reported.length > 0 ? reported : null));
- command.with_parameter<Invercargill.Element>("id", elem<int64?>(saddle.id));
- yield command.execute_non_query_async();
- }
- /**
- * Records a /status observation: the health snapshot, the
- * daemon-reported ssh key, and the goal-branch push flags
- * (plan 15 C4). The `resumed` flag needs no projection — the
- * restore's record is the daemon's own system entry in the
- * raw terminal log (plan 16 S5/U2).
- */
- private async void record_status_probe(Saddle saddle, Json.Object status) throws Error {
- string state = status.has_member("state")
- ? status.get_string_member_with_default("state", "") : "";
- int64 uptime = status.has_member("uptime_seconds")
- && status.get_member("uptime_seconds").get_value_type() == typeof(int64)
- ? status.get_int_member("uptime_seconds") : 0;
- int64 activity = status.has_member("last_activity_epoch")
- && status.get_member("last_activity_epoch").get_value_type() == typeof(int64)
- ? status.get_int_member("last_activity_epoch") : 0;
- poll_state.record_probe(saddle.id, state, uptime, activity);
- yield store_reported_ssh_key(saddle, status);
- // Plan 15 C4: a daemon running (or parking — the
- // workspace stays while the saddle frees) a goal agent
- // exposes the per-repo push flags of the goal branches on
- // /status; they are recorded as LIVE probe state on the
- // saddle (never a stored snapshot on the goal row) and
- // the goal page reads them back at render time for its
- // advisory push chips. The push rule itself is enforced
- // daemon-side at the agent's completion attempt.
- if (status.has_member("goal_branches")) {
- var goal_branches = status.get_member("goal_branches");
- if (goal_branches.get_node_type() == Json.NodeType.ARRAY
- || goal_branches.get_node_type() == Json.NodeType.OBJECT) {
- var generator = new Json.Generator();
- generator.set_root(goal_branches);
- poll_state.record_goal_branches(saddle.id, generator.to_data(null));
- }
- }
- // Plan 16 S5 (U2): the daemon's own system entry (typed
- // 'system' in the results stream) is the raw-terminal
- // record of a workspace restore — the web no longer authors
- // a duplicate timeline row for it. The `resumed` flag
- // carries no further web-side projection.
- }
- /**
- * Wakes a sleeping daemon (plan 10 wire contract): `POST /wake`
- * `{}` — a capable task is ready; sleeping daemons ask for work,
- * busy ones ignore it, idle-awake ones re-ask. Idempotent. The
- * caller records the pending wake for the lost-wake guard.
- */
- public async void wake_saddle(Saddle saddle) throws Error {
- var payload = new Json.Builder();
- payload.begin_object();
- payload.end_object();
- Json.Node? root = null;
- int status_code = yield request_saddle(saddle.host, "POST", "/wake",
- (owned) payload, out root);
- if (status_code >= 400) {
- throw new Error(Quark.from_string("clanker-saddle"), 49,
- @"wake returned HTTP $(status_code)");
- }
- }
- /**
- * Cancelling / requeueing / stopping a task with a live session
- * reaches its daemon (wire contract): `POST /task-update
- * {kind: "cancel"}` — the authoritative stop through the daemon's
- * abort machinery, the workspace kept on disk per the park
- * contract. Success is narrated as "<verb>; daemon aborted"; a
- * transport failure marks the saddle offline (the daemon's next
- * ask proves it alive again and flips the row back to idle). The
- * daemon must never keep running a task the web has moved on
- * from as a zombie.
- */
- public async void abort_live_session(TaskService tasks, int64 task_id,
- string verb) throws Error {
- var session = yield tasks.open_session_for_task(task_id);
- if (session == null) {
- return;
- }
- yield abort_session_on_its_saddle(tasks, (!)session, verb);
- }
- /**
- * SADDLE_STALE_S (default 90s): the silence window after which a
- * running daemon owes an exceptional GET /status probe.
- */
- public static uint stale_after_seconds() {
- var raw = Environment.get_variable("SADDLE_STALE_S") ?? "";
- int64 parsed = int64.parse(raw);
- return parsed > 0 ? (uint) parsed : 90;
- }
- /**
- * {@link abort_live_session} keyed by saddle instead of task —
- * the unregister path. The cancel is delivered to the saddle's
- * RUNNING session (the derived record; plan 16 item 23) — not
- * re-resolved through a task's lowest-id open session, which a
- * stale fixture row on another saddle would redirect to the
- * wrong daemon.
- */
- public async void abort_saddle_session(TaskService tasks, int64 saddle_id,
- string verb) throws Error {
- var session = yield running_session_on_saddle(saddle_id);
- if (session == null) {
- return;
- }
- yield abort_session_on_its_saddle(tasks, (!)session, verb);
- }
- /**
- * The cancel delivery core shared by the task-keyed and
- * saddle-keyed abort paths: `POST /task-update {kind: cancel}`
- * to the daemon holding THIS session.
- */
- private async void abort_session_on_its_saddle(TaskService tasks,
- ClankerSaddle.Data.Session session, string verb) throws Error {
- var saddle = yield get_saddle(session.saddle_id);
- if (saddle == null) {
- return;
- }
- var payload = new Json.Builder();
- payload.begin_object();
- payload.set_member_name("kind");
- payload.add_string_value("cancel");
- payload.end_object();
- Json.Node? root = null;
- try {
- int status_code = yield request_saddle(((!)saddle).host, "POST",
- "/task-update", (owned) payload, out root);
- if (status_code >= 400) {
- throw new Error(Quark.from_string("clanker-saddle"), 45,
- @"task-update cancel returned HTTP $(status_code)");
- }
- yield tasks.add_system_comment(session.task_id, @"$(verb); daemon aborted");
- } catch (Error e) {
- warning(@"[ClankerSaddle] Abort for task $(session.task_id) failed: $(e.message); marking saddle '$(((!)saddle).name)' offline");
- yield mark_saddle_unreachable(((!)saddle).id, tasks,
- @"abort delivery failed ($(e.message))");
- yield tasks.add_system_comment(session.task_id,
- @"$(verb); daemon unreachable — saddle marked offline");
- }
- }
- /**
- * Requeues tasks caught running on saddles that have since gone
- * offline (a manual status refresh, a restart) — without this
- * pass they would wait forever for a daemon that will not answer.
- * Only `assigned`/`working` tasks move: parked tasks
- * (blocked/review/suspended, their sessions closed 'parked')
- * keep their records, which the resume machinery needs. Each
- * move is {@link TaskService.requeue_task} — the release owner
- * closes the session (the close IS the free, item 23) — plus a
- * system comment so the timeline shows why the task restarted.
- */
- public async void requeue_tasks_on_offline_saddles(TaskService tasks) throws Error {
- foreach (var status in new string[] { STATUS_ASSIGNED, STATUS_WORKING }) {
- foreach (var task in yield db.query<TaskEntity>()
- .where(expr("status == $0", elem(status)))
- .materialise_async()) {
- var session = yield tasks.open_session_for_task(task.id);
- if (session == null) {
- continue;
- }
- if (yield tasks.newest_block_answered(task.id)) {
- // Parked-with-answered: a verdict awaits the
- // resume machinery — never requeue it.
- continue;
- }
- var saddle = yield get_saddle(((!)session).saddle_id);
- if (saddle == null || ((!)saddle).status != SADDLE_OFFLINE) {
- continue;
- }
- print(@"[ClankerSaddle] Task $(task.id) requeued: its saddle went offline\n");
- yield tasks.requeue_task(task.id);
- yield tasks.add_system_comment(task.id,
- "saddle went offline; task requeued");
- }
- }
- }
- /**
- * Every saddle free for work — derived (plan 16 item 23): not
- * offline (the probe state) and holding no running session —
- * registration order, the dispatcher's capability-matching
- * candidate list. A parked task's closed session reads free
- * exactly like the freed daemon it brackets.
- */
- public async ImmutableLot<Saddle> list_idle_saddles() throws Error {
- var busy = yield busy_saddle_ids();
- var rows = new Vector<Saddle>();
- foreach (var saddle in yield db.query<Saddle>()
- .order_by(expr("id"))
- .materialise_async()) {
- if (saddle.status == SADDLE_OFFLINE
- || busy.contains(((int64) saddle.id).to_string())) {
- continue;
- }
- rows.add(saddle);
- }
- return rows.to_immutable_buffer();
- }
- /**
- * Marks a saddle offline when its control server cannot be
- * reached. The task it was RUNNING (the derived running
- * session — plan 16 item 23) releases back to 'queued' through
- * {@link TaskService.release_task} (the ONE owner; it closes
- * the session 'stopped'), with a system comment so the timeline
- * shows the interruption; combined with the daemon's
- * crash-resume, the next dispatch restores the workspace and
- * reports `resumed` on /status. A task that parked mid-flight
- * (its session already closed 'parked', the verdict awaiting
- * the resume) keeps its park. The offline stamp itself is the
- * ONE surviving guarded write — see {@link stamp_offline}.
- */
- public async void mark_saddle_unreachable(int64 id, TaskService tasks,
- string? reason = null) throws Error {
- var saddle = yield get_saddle(id);
- if (saddle == null) {
- return;
- }
- var session = yield running_session_on_saddle(id);
- int64 seen_session = session != null ? ((!)session).id : 0;
- int64 held_task = 0;
- if (session != null) {
- held_task = ((!)session).task_id;
- // The requeue flip is guarded: a park (or terminal)
- // interleaving this path keeps its state — the parked
- // verdict must survive for the resume.
- var flip = connection.create_command(
- "UPDATE tasks SET status = 'queued' WHERE id = :id"
- + " AND status IN ('assigned', 'working')");
- flip.with_parameter<Invercargill.Element>("id", elem<int64?>(held_task));
- if ((yield flip.execute_non_query_async()) > 0) {
- string unreachable_text = reason != null && reason.length > 0
- ? @"Saddle became unreachable ($(reason)); task requeued"
- : "Saddle became unreachable; task requeued";
- yield tasks.add_system_comment(held_task, unreachable_text);
- // Closes the held session FIRST — the offline
- // stamp's guard refuses to land over a running
- // session, and by then there is none.
- yield tasks.release_task(held_task, OUTCOME_STOPPED, false);
- } else {
- // Parked (or finished) mid-flight — the park closed
- // its session and the verdict must survive for the
- // resume; nothing to release.
- held_task = 0;
- }
- }
- yield stamp_offline(id, seen_session);
- // An offline saddle is neither sleeping nor wake-awaiting;
- // a restarted daemon re-establishes both by asking.
- poll_state.clear_ask(id);
- yield pulse.bump(0, "saddle unreachable");
- }
- /**
- * The ONE surviving guarded saddle write (plan 16 item 23):
- * the OFFLINE probe stamp. The guard is the sessions-truth
- * analogue of the old seen-assignment guard — the stamp lands
- * unless a running session the probe did NOT see at entry
- * appeared meanwhile (an ask claim is proof of life and must
- * not be stomped offline). The session observed at entry
- * (`seen_session`) never blocks its own stamp.
- */
- private async void stamp_offline(int64 saddle_id, int64 seen_session) throws Error {
- var command = connection.create_command(
- "UPDATE saddles SET status = '" + SADDLE_OFFLINE + "' WHERE id = :id"
- + " AND NOT EXISTS (SELECT 1 FROM sessions s"
- + " JOIN tasks t ON t.id = s.task_id"
- + " WHERE s.saddle_id = :id AND s.ended_at IS NULL"
- + " AND t.status IN ('assigned', 'working')"
- + " AND s.id != :seen)");
- command.with_parameter<Invercargill.Element>("id", elem<int64?>(saddle_id));
- command.with_parameter<Invercargill.Element>("seen", elem<int64?>(seen_session));
- yield command.execute_non_query_async();
- }
- /**
- * Assembles the assignment payload shared by the /start push and
- * the `POST /api/saddle/next-task` response — `{task, kind,
- * mode, creator, goal?, rules, project_notes, callback,
- * llm_config, session_id}` — with the project, links (the
- * goal's own links for goal work, else the project's), branch
- * fields and recent comments inside the task object, plus any
- * block answers not yet handed over (a resumed workspace picks
- * its pending answers up from the assignment itself), the
- * effective rule set ({@link RuleService.effective_rules}) as
- * the top-level `rules` array, the capped `project_notes` of
- * the primary and co-developed projects, the note-creation
- * `callback` (base URL + the saddle's own bearer token), the
- * saddle's `git_identity`, a `capabilities` context (plan 10
- * phase 3: `saddle_tags` — the allocated saddle's own tags —
- * and `fleet_tags` — every tag name known across the fleet),
- * the plan-12 goal context (`goal` for goal work; the
- * worker's `ephemeral_branch`) and — for resumed workspaces —
- * the `resume_notice`/`branch_advanced` context. `session_id`
- * (the web-side run row the daemon reports against) rides
- * along so pushed telemetry can key on it. Project checkouts
- * are addressed by the `SADDLE_PROJECTS_ROOT` convention
- * (`root/task-<task id>/name`) and cloned from each project's
- * `git_url`; the LLM endpoint by the {@link LlmEndpoint}
- * environment.
- */
- public async Json.Object build_assignment_payload(Saddle saddle, TaskEntity task,
- int64 session_id, ProjectService projects, string? resume_notice = null,
- bool? branch_advanced = null,
- string? context_update = null) throws Error {
- var endpoint = LlmEndpoint.from_environment();
- if (endpoint == null) {
- throw new Error(Quark.from_string("clanker-saddle"), 24,
- "No LLM endpoint configured (set SADDLE_LLM_BASE_URL)");
- }
- var project = task.project_id > 0
- ? yield db.query<Project>()
- .where(expr("id == $0", elem<int64?>(task.project_id)))
- .first_async()
- : null;
- // Goal 8 task 4: project_id 0 is a STANDALONE task — no
- // project row exists (and none is looked up), the
- // co-developed/reference arrays ship EMPTY and the daemon
- // renders the standalone prompt section. A task with a
- // non-zero project_id whose row is missing is still the
- // data-integrity error it always was.
- if (project == null && task.project_id > 0) {
- throw new Error(Quark.from_string("clanker-saddle"), 12,
- @"Task $(task.id) has no project $(task.project_id)");
- }
- // Plan 12: goal work (the agent's own run and its worker
- // tasks) scopes its co-developed/reference set through the
- // GOAL's links; standalone tasks through the project's.
- // Standalone tasks (project 0) own no links at all — the
- // simplest consistent shape: both arrays stay empty.
- Goal? goal = null;
- if (task.goal_id > 0) {
- goal = yield db.query<Goal>()
- .where(expr("id == $0", elem<int64?>(task.goal_id)))
- .first_async();
- }
- var co_developed = new Vector<Project>();
- var references = new Vector<Project>();
- if (goal != null) {
- foreach (var link in yield goal_service.list_links(task.goal_id)) {
- var linked = yield db.query<Project>()
- .where(expr("id == $0", elem<int64?>(link.project_id)))
- .first_async();
- if (linked == null) {
- continue;
- }
- if (link.relationship == LINK_CO_DEVELOPED) {
- co_developed.add((!)linked);
- } else if (link.relationship == LINK_REFERENCE) {
- references.add((!)linked);
- }
- }
- } else if (project != null) {
- foreach (var link in yield projects.list_links(task.project_id)) {
- var linked = yield db.query<Project>()
- .where(expr("id == $0", elem<int64?>(link.linked_project_id)))
- .first_async();
- if (linked == null) {
- continue;
- }
- if (link.relationship == LINK_CO_DEVELOPED) {
- co_developed.add((!)linked);
- } else if (link.relationship == LINK_REFERENCE) {
- references.add((!)linked);
- }
- }
- }
- var payload = new Json.Builder();
- payload.begin_object();
- payload.set_member_name("task");
- payload.begin_object();
- payload.set_member_name("id");
- payload.add_int_value(task.id);
- payload.set_member_name("title");
- payload.add_string_value(task.title);
- payload.set_member_name("description");
- payload.add_string_value(task.description ?? "");
- payload.set_member_name("project_name");
- payload.add_string_value(project != null ? ((!)project).name : "standalone");
- payload.set_member_name("project_branch");
- payload.add_string_value(project != null ? ((!)project).default_branch : "");
- payload.set_member_name("task_branch");
- payload.add_string_value(task.task_branch ?? "");
- payload.set_member_name("branch_from");
- payload.add_string_value(task.branch_from
- ?? (project != null ? ((!)project).default_branch : ""));
- payload.set_member_name("co_developed");
- payload.begin_array();
- if (project != null) {
- payload.add_value(project_node(((!)project), task.id, false, saddle));
- }
- foreach (var linked in co_developed) {
- payload.add_value(project_node(linked, task.id, false, saddle));
- }
- payload.end_array();
- payload.set_member_name("reference");
- payload.begin_array();
- foreach (var linked in references) {
- payload.add_value(project_node(linked, task.id, true, saddle));
- }
- payload.end_array();
- payload.set_member_name("comments");
- payload.begin_array();
- var comments = yield db.query<TaskComment>()
- .where(expr("task_id == $0", elem<int64?>(task.id)))
- .order_by(expr("id"))
- .materialise_async();
- foreach (var comment in comments) {
- payload.begin_object();
- payload.set_member_name("author");
- payload.add_string_value(comment.author);
- payload.set_member_name("body");
- payload.add_string_value(comment.body);
- if (comment.quote != null) {
- payload.set_member_name("quote");
- payload.add_string_value(((!)comment.quote));
- }
- payload.end_object();
- }
- payload.end_array();
- payload.set_member_name("answers");
- payload.begin_array();
- foreach (var pending in yield take_pending_answers_for_task(task.id)) {
- payload.begin_object();
- payload.set_member_name("block_id");
- payload.add_int_value(pending.block_id);
- payload.set_member_name("answer");
- payload.add_string_value(raw_answer_for_saddle(pending.answer));
- if (branch_advanced != null) {
- payload.set_member_name("branch_advanced");
- payload.add_boolean_value((!)branch_advanced);
- }
- payload.end_object();
- }
- payload.end_array();
- payload.end_object();
- // The plan-12 wire additions: kind ("task"|"goal_run"),
- // mode and creator ride the top level beside the task.
- payload.set_member_name("kind");
- payload.add_string_value(task.kind);
- payload.set_member_name("mode");
- payload.add_string_value(task.mode);
- payload.set_member_name("creator");
- payload.add_string_value(task.creator);
- // The goal context (goal work only): the goal agent gets
- // the full brief + approved plan; a goal worker gets the
- // same object as read-only reference (the daemon adds the
- // stick-to-mandate shaping) plus its ephemeral branch.
- if (goal != null) {
- payload.set_member_name("goal");
- payload.begin_object();
- payload.set_member_name("id");
- payload.add_int_value(((!)goal).id);
- payload.set_member_name("branch");
- payload.add_string_value(((!)goal).branch_name);
- payload.set_member_name("base_branch");
- payload.add_string_value(((!)goal).base_branch
- ?? (project != null ? ((!)project).default_branch : ""));
- payload.set_member_name("brief");
- payload.add_string_value(((!)goal).brief);
- payload.set_member_name("plan");
- payload.add_string_value(((!)goal).plan ?? "");
- payload.end_object();
- if (task.kind == TASK_KIND_TASK) {
- payload.set_member_name("ephemeral_branch");
- payload.add_string_value(@"task/$(task.goal_id)-$(task.id)");
- }
- }
- // The wire contract's top-level `rules` addition: the
- // effective rule set (task ∪ primary ∪ co-developed, deduped,
- // stable order — see RuleService.effective_rules). Goal work
- // expands co-developed through the goal's links.
- payload.set_member_name("rules");
- payload.begin_array();
- foreach (var rule in yield rules.effective_rules_for(task, projects,
- goal != null ? task.goal_id : 0)) {
- payload.begin_object();
- payload.set_member_name("name");
- payload.add_string_value(rule.name);
- payload.set_member_name("body");
- payload.add_string_value(rule.body);
- payload.end_object();
- }
- payload.end_array();
- // The wire contract's `git_identity` addition (plan 9): the
- // name/email the daemon commits as, from the saddle row.
- // Omitted entirely when unset so the daemon's fallback
- // identity applies.
- string identity_name = saddle.git_name ?? "";
- string identity_email = saddle.git_email ?? "";
- if (identity_name.length > 0 || identity_email.length > 0) {
- payload.set_member_name("git_identity");
- payload.begin_object();
- payload.set_member_name("name");
- payload.add_string_value(identity_name);
- payload.set_member_name("email");
- payload.add_string_value(identity_email);
- payload.end_object();
- }
- // The wire contract's `prompt_note` addition (goal 8 task
- // 5): the saddle admin's standing message about THIS
- // machine, served on EVERY assignment (the daemon renders
- // it verbatim under an admin heading). Empty string when
- // unset, clamped so a fat note can never bloat prompts.
- var prompt_note_text = saddle.prompt_note ?? "";
- if (prompt_note_text.length > PROMPT_NOTE_MAX_CHARS) {
- prompt_note_text = prompt_note_text.substring(0, (int) PROMPT_NOTE_MAX_CHARS);
- }
- payload.set_member_name("prompt_note");
- payload.add_string_value(prompt_note_text);
- // The wire contract's `capabilities` context (plan 10
- // phase 3): the allocated saddle's own tags plus every tag
- // name known across the fleet (the operator-managed
- // catalogue) — the daemon renders both into its system
- // prompt and validates `task create --capability`.
- payload.set_member_name("capabilities");
- payload.begin_object();
- payload.set_member_name("saddle_tags");
- payload.begin_array();
- foreach (var carried in yield capability_service.list_for_saddle(saddle.id)) {
- payload.add_string_value(carried.name);
- }
- payload.end_array();
- payload.set_member_name("fleet_tags");
- payload.begin_array();
- foreach (var known in yield capability_service.list_capabilities()) {
- payload.add_string_value(known.name);
- }
- payload.end_array();
- payload.end_object();
- // The wire contract's `resume_notice` addition: on a
- // workspace-adopting start (a suspended parent's resume), the
- // daemon injects it as the next user turn instead of the
- // opening prompt.
- if (resume_notice != null && resume_notice.length > 0) {
- payload.set_member_name("resume_notice");
- payload.add_string_value(resume_notice);
- }
- // Goal 28 task F: `pending_question` — the task's newest
- // block is UNANSWERED (a crashed parked task resumed
- // through the allocation). The daemon appends the
- // still-open notice to the restored conversation's opening
- // turn so the agent does not barrel past its own question.
- var newest_open = yield task_service.newest_block(task.id);
- if (newest_open != null && ((!)newest_open).answer == null) {
- payload.set_member_name("pending_question");
- payload.add_boolean_value(true);
- }
- // The wire's `branch_advanced` (resume pushes): whether the
- // shared branch moved while this task was parked — false or
- // absent means the daemon omits the pull instruction entirely.
- if (branch_advanced != null) {
- payload.set_member_name("branch_advanced");
- payload.add_boolean_value((!)branch_advanced);
- }
- // The wire's `context_update` (resume pushes): a short
- // markdown delta of the instruction context that changed
- // while the workspace was parked — the goal's plan
- // approved since the run's last round, rules created,
- // edited or attached, or new project notes. The daemon
- // appends it as a TURN (prompt-cache friendly) instead of
- // refreshing the system prompt; absent when nothing
- // changed.
- if (context_update != null && context_update.length > 0) {
- payload.set_member_name("context_update");
- payload.add_string_value(context_update);
- }
- // The wire contract's `project_notes` addition: notes of the
- // primary and co-developed projects, oldest first, capped at
- // 30 entries / 20 000 chars (head kept).
- var note_project_names = new Dictionary<string, string>();
- var project_notes_rows = new Vector<ClankerSaddle.Data.ProjectNote>();
- if (project != null) {
- note_project_names.set(((!)project).id.to_string(), ((!)project).name);
- foreach (var note in yield projects.list_notes(((!)project).id)) {
- project_notes_rows.add(note);
- }
- }
- foreach (var linked in co_developed) {
- note_project_names.set(linked.id.to_string(), linked.name);
- foreach (var note in yield projects.list_notes(linked.id)) {
- project_notes_rows.add(note);
- }
- }
- var project_notes = project_notes_rows.sort((a, b) => (int)(a.id - b.id));
- payload.set_member_name("project_notes");
- payload.begin_array();
- int64 note_entries = 0;
- int64 note_chars = 0;
- foreach (var note in project_notes) {
- if (note_entries >= PROJECT_NOTES_MAX_ENTRIES
- || note_chars >= PROJECT_NOTES_MAX_CHARS) {
- break;
- }
- string note_body = note.body;
- if (note_chars + note_body.length > PROJECT_NOTES_MAX_CHARS) {
- note_body = note_body.substring(0,
- (int)(PROJECT_NOTES_MAX_CHARS - note_chars));
- }
- payload.begin_object();
- payload.set_member_name("project");
- payload.add_string_value(note_project_names.get_or_default(note.project_id.to_string()) ?? "");
- payload.set_member_name("body");
- payload.add_string_value(note_body);
- payload.end_object();
- note_entries++;
- note_chars += note_body.length;
- }
- payload.end_array();
- // The wire contract's `understanding` addition (goal 24):
- // the PRIMARY project's current understanding document —
- // a markdown summary of the project's structure and its
- // unusual corners, stamped with the commit it was written
- // against. Present but empty when the project has none;
- // head-clamped at UNDERSTANDING_MAX_CHARS.
- string understanding_body = "";
- if (project != null) {
- var current_understanding = yield understandings
- .current_understanding(((!)project).id);
- if (current_understanding != null) {
- understanding_body = ((!)current_understanding).body;
- if (understanding_body.length > UNDERSTANDING_MAX_CHARS) {
- understanding_body = understanding_body.substring(0,
- (int) UNDERSTANDING_MAX_CHARS);
- }
- }
- }
- payload.set_member_name("understanding");
- payload.add_string_value(understanding_body);
- // The wire contract's `discoveries` addition (goal 24):
- // the fleet's NEWEST discoveries — non-project facts
- // (compiler quirks, library behaviours, tool flags,
- // protocol gotchas) earlier agents recorded — newest
- // first, capped at 10 entries / 8 000 chars (head kept),
- // so a fresh agent inherits the knowledge base before it
- // ever searches. An empty array when there are none.
- payload.set_member_name("discoveries");
- payload.begin_array();
- int64 discovery_entries = 0;
- int64 discovery_chars = 0;
- foreach (var discovery in yield discoveries.list_newest_first(
- (int) DISCOVERIES_MAX_ENTRIES)) {
- string discovery_body = discovery.body;
- if (discovery_chars + discovery_body.length
- > DISCOVERIES_MAX_CHARS) {
- discovery_body = discovery_body.substring(0,
- (int)(DISCOVERIES_MAX_CHARS - discovery_chars));
- }
- payload.begin_object();
- payload.set_member_name("body");
- payload.add_string_value(discovery_body);
- payload.end_object();
- discovery_entries++;
- discovery_chars += discovery_body.length;
- if (discovery_entries >= DISCOVERIES_MAX_ENTRIES
- || discovery_chars >= DISCOVERIES_MAX_CHARS) {
- break;
- }
- }
- payload.end_array();
- // The wire contract's `ethos` addition (goal 27, canon
- // centralisation): the ONE web-side ethos text rides every
- // assignment so the daemon's system prompt carries it
- // verbatim without a second fetch (its compiled-in copy is
- // the compatibility fallback for older webs).
- payload.set_member_name("ethos");
- payload.add_string_value(ClankerSaddle.Services.Ethos.TEXT);
- // The wire contract's `callback` addition: this app's
- // externally reachable base URL plus the bearer token the
- // daemon presents when posting back — its own per-saddle
- // credential (plan 10), which every `/api/saddle/*` call
- // carries.
- payload.set_member_name("callback");
- payload.begin_object();
- payload.set_member_name("base_url");
- payload.add_string_value(callback_base_url());
- payload.set_member_name("token");
- payload.add_string_value(saddle.token ?? "");
- payload.end_object();
- payload.set_member_name("llm_config");
- payload.begin_object();
- payload.set_member_name("base_url");
- payload.add_string_value(((!)endpoint).base_url);
- payload.set_member_name("api_key");
- payload.add_string_value(((!)endpoint).api_key);
- payload.set_member_name("model");
- payload.add_string_value(((!)endpoint).model);
- payload.end_object();
- // The web-side run row the daemon's pushed telemetry keys on
- // (plan-10 phase-1 wire addition: it rides every assignment
- // so /start consumers can ignore it harmlessly).
- payload.set_member_name("session_id");
- payload.add_int_value(session_id);
- payload.end_object();
- var root = payload.get_root().copy();
- return root.get_object();
- }
- /**
- * Probes a saddle's `GET /status` and records what a probe can
- * tell: the health snapshot, the daemon-reported ssh key and
- * the goal-branch push flags. Busy/idle need NO write anymore —
- * they are derived from the session records (plan 16 item 23).
- * The row's only remaining write is the OFFLINE probe state: a
- * successful probe is proof of life and clears it; a failed
- * probe stamps it through the guarded write (an ask claim that
- * landed while the probe was in flight is a NEWER proof of
- * life and must not be stomped).
- */
- public async void refresh_saddle_status(int64 id) throws Error {
- var saddle = yield get_saddle(id);
- if (saddle == null) {
- throw new Error(Quark.from_string("clanker-saddle"), 17,
- @"No saddle exists with id $(id)");
- }
- var session = yield running_session_on_saddle(id);
- int64 seen_session = session != null ? ((!)session).id : 0;
- try {
- var status = yield request_saddle_json(((!)saddle).host, "GET", "/status", null);
- yield record_status_probe(((!)saddle), status);
- var clear = connection.create_command(
- "UPDATE saddles SET status = '" + SADDLE_IDLE
- + "' WHERE id = :id AND status = '" + SADDLE_OFFLINE + "'");
- clear.with_parameter<Invercargill.Element>("id", elem<int64?>(id));
- yield clear.execute_non_query_async();
- } catch (Error e) {
- yield stamp_offline(id, seen_session);
- }
- yield pulse.bump(0, "saddle status refreshed");
- }
- /**
- * Answered blocks of `task_id` not yet handed over anywhere;
- * marks them delivered — a resumed assignment carries its
- * pending answers in the payload ({@link push_task_to_saddle}).
- */
- public async ImmutableLot<PendingAnswer> take_pending_answers_for_task(int64 task_id) throws Error {
- var blocks = yield db.query<TaskBlock>()
- .where(expr("task_id == $0", elem<int64?>(task_id)))
- .materialise_async();
- var rows = new Vector<PendingAnswer>();
- foreach (var block in blocks) {
- if (block.answer == null) {
- continue;
- }
- if (!poll_state.mark_delivered(block.id)) {
- continue;
- }
- rows.add(new PendingAnswer() {
- block_id = block.id,
- answer = raw_answer_for_saddle(((!)block).answer ?? "")
- });
- }
- return rows.to_immutable_buffer();
- }
- /**
- * Pushes the task's undelivered human comments to the RUNNING
- * daemon's `POST /task-update {kind: "comment"}` endpoint so
- * they land in the live conversation. Best effort: a daemon
- * without the route (HTTP 404/405) keeps the comments pending,
- * and other failures are logged and retried on the next
- * dispatcher tick.
- */
- public async void deliver_pending_comments(int64 saddle_id, int64 task_id) throws Error {
- var saddle = yield get_saddle(saddle_id);
- if (saddle == null) {
- return;
- }
- var comments = yield db.query<TaskComment>()
- .where(expr("task_id == $0", elem<int64?>(task_id)))
- .order_by(expr("id"))
- .materialise_async();
- foreach (var comment in comments) {
- if (!comment.author.has_prefix("user:")
- || poll_state.is_comment_delivered(comment.id)) {
- continue;
- }
- var payload = new Json.Builder();
- payload.begin_object();
- payload.set_member_name("kind");
- payload.add_string_value("comment");
- payload.set_member_name("text");
- payload.add_string_value(comment.body);
- payload.set_member_name("quote");
- payload.add_string_value(comment.quote ?? "");
- payload.end_object();
- Json.Node? root = null;
- int status_code = yield request_saddle(((!)saddle).host, "POST",
- "/task-update", (owned) payload, out root);
- if (status_code == 200 || status_code == 204) {
- poll_state.mark_comment_delivered(comment.id);
- } else if (status_code != 404 && status_code != 405) {
- warning(@"[ClankerSaddle] Comment task-update to saddle $(saddle_id) returned HTTP $(status_code)");
- }
- }
- }
- /**
- * Marks the task's human comments as delivered — they were handed
- * over inside an assignment payload, so neither the poll endpoint
- * nor `/notify` should repeat them.
- */
- public async void mark_task_comments_delivered(int64 task_id) throws Error {
- var comments = yield db.query<TaskComment>()
- .where(expr("task_id == $0", elem<int64?>(task_id)))
- .materialise_async();
- foreach (var comment in comments) {
- if (comment.author.has_prefix("user:")) {
- poll_state.mark_comment_delivered(comment.id);
- }
- }
- }
- /**
- * Goal 28 task F (busy-answer delivery): the hub drainer. An
- * answer whose task holds a RUNNING session reaches that
- * session THROUGH the daemon's task-update channel (the same
- * injection path human comments ride) — `POST /task-update
- * {kind: "answer", text, question}` — because no affinity
- * resume can deliver it: the busy saddle cannot ask, and the
- * running round's conversation predates the block.
- *
- * Delivery-once: the poll-state answers ledger (the poll
- * endpoint's memory) is the guard, so the resume notice never
- * repeats a verdict the running round already saw. The
- * session's `answers_injected` count increments per verdict —
- * a later park's resume notice summarises them instead of
- * repeating the texts. Failures (transport, HTTP) re-offer
- * the delivery for the next tick: an answer must never be
- * dropped, and the daemon tolerates a duplicate as one extra
- * user turn.
- */
- public async void deliver_pending_answers(TaskService tasks,
- AnswerDeliveryHub hub) throws Error {
- foreach (var delivery in hub.drain()) {
- var session = yield tasks.open_session_for_task(delivery.task_id);
- if (session == null) {
- // No running round: the resume path owns delivery
- // (the verdict rides the next resuming
- // assignment's notice). Drop the queue entry —
- // the answer is safe in its block row.
- continue;
- }
- var saddle = yield get_saddle(((!)session).saddle_id);
- if (saddle == null) {
- hub.offer(delivery);
- continue;
- }
- var payload = new Json.Builder();
- payload.begin_object();
- payload.set_member_name("kind");
- payload.add_string_value("answer");
- payload.set_member_name("text");
- payload.add_string_value(delivery.notice);
- payload.set_member_name("task_id");
- payload.add_int_value(delivery.task_id);
- payload.end_object();
- Json.Node? root = null;
- bool handed = false;
- try {
- int status_code = yield request_saddle(((!)saddle).host, "POST",
- "/task-update", (owned) payload, out root);
- // 404/405: a daemon without the answer kind keeps
- // the delivery pending — re-offer for the next
- // tick (an upgrade may land under us). Everything
- // else that fails re-offers too: never drop an
- // answer.
- handed = status_code == 200 || status_code == 204;
- if (!handed) {
- warning("[ClankerSaddle] Answer task-update to saddle $(((!)saddle).id) returned HTTP %d",
- status_code);
- hub.offer(delivery);
- continue;
- }
- } catch (Error e) {
- warning("[ClankerSaddle] Answer task-update to saddle $(((!)saddle).id) failed: %s",
- e.message);
- hub.offer(delivery);
- continue;
- }
- poll_state.mark_delivered(delivery.block_id);
- ((!)session).answers_injected += 1;
- yield db.update_async<ClankerSaddle.Data.Session>((!)session);
- yield task_service.add_system_comment(delivery.task_id,
- @"verdict delivered into the running session (block #$(delivery.block_id))");
- yield pulse.bump(delivery.task_id, "answer delivered mid-session");
- }
- }
- /**
- * The goal id when `task_id` is a goal agent's run task (kind
- * 'goal_run'), else 0 — the discriminator for the goal-context
- * verb handling above.
- */
- /**
- * True when the goal owns at least one WORKER task that could
- * still fire a terminal transition (the goal-run park's wake
- * source): every status but done/failed counts — a queued,
- * assigned, working, blocked, review-requested or suspended
- * row will end its life in a transition that wakes the run.
- * DRAFT rows do NOT count: a draft nobody ever enqueued
- * produces no transition, ever (the goal-2 stall class).
- */
- private async bool goal_has_wakeable_task(int64 goal_id) throws Error {
- foreach (var task in yield db.query<TaskEntity>()
- .where(expr("goal_id == $0 && kind == $1",
- elem<int64?>(goal_id), elem(TASK_KIND_TASK)))
- .materialise_async()) {
- if (task.status != STATUS_DONE && task.status != STATUS_FAILED
- && task.status != STATUS_DRAFT) {
- return true;
- }
- }
- return false;
- }
- /**
- * The parking guard's refusal: the park mechanics run (status
- * 'suspended', the round's session closed 'parked',
- * suspended_at stamped) and the ONE-SHOT
- * {@link TaskEntity.pending_resume_notice} is set — which
- * makes the run resume-offerable IMMEDIATELY (no terminal
- * event or coalescing window). The daemon asks again the
- * moment its round ends; that ask claims the run back and the
- * refusal text rides the assignment as its resume_notice, so
- * the agent is told to finish (submit its report) or cancel
- * instead of sleeping forever. Keeping the session open
- * instead would dead-end: an open session on an assigned task
- * marks the saddle busy, so the next ask would 204 and the
- * daemon would sleep with the workspace wedged.
- */
- private async void refuse_park(int64 task_id, string notice) throws Error {
- yield task_service.set_parked(task_id);
- var task = yield db.query<TaskEntity>()
- .where(expr("id == $0", elem<int64?>(task_id)))
- .first_async();
- if (task == null || ((!)task).pending_resume_notice != null) {
- // The first refusal stands (an already-pending notice
- // keeps its original wording).
- return;
- }
- ((!)task).pending_resume_notice = notice;
- yield db.update_async<TaskEntity>((!)task);
- var refused = new Json.Object();
- refused.set_string_member("text", notice);
- yield event_service.append(EVENT_STATE_CHANGE, EVENT_ACTOR_SYSTEM,
- EVENT_SCOPE_TASK, task_id, refused);
- yield task_service.add_system_comment(task_id, notice);
- yield pulse.bump(task_id, "park refused (nothing to wait on)");
- }
- private async int64 goal_id_of_run(int64 task_id) throws Error {
- var task = yield db.query<TaskEntity>()
- .where(expr("id == $0", elem<int64?>(task_id)))
- .first_async();
- return task != null && ((!)task).kind == TASK_KIND_GOAL_RUN
- ? ((!)task).goal_id : 0;
- }
- /**
- * Raises the daemon's pushed blocking prompt as a `task_blocks`
- * row exactly once: a not-yet-answered row, or an answered row
- * whose answer has not been delivered to the daemon yet, means the
- * prompt is already represented. The results push's `block`
- * object carries the question plus the block type, options and
- * markdown body (the `/status` field names — `block_question`
- * &c — are accepted as a fallback). `saddle block rfc` carries
- * no question text, so rfc prompts surface under a stable
- * fallback title.
- */
- private async void ensure_block(TaskService tasks, int64 task_id, Json.Object block) throws Error {
- string question = block_question_from_payload(block);
- if (question.length == 0) {
- return;
- }
- string incoming_type = block_string_or(block, "type", "block_type");
- foreach (var existing in yield tasks.list_blocks(task_id)) {
- if (existing.answer == null || !poll_state.is_delivered(existing.id)) {
- // A pending review-family block of EITHER kind
- // (review/completion) swallows the other's push: the
- // daemon's done park and the web's completion raise
- // are the SAME review raised from two paths — one
- // row, whichever landed first.
- if ((incoming_type == BLOCK_REVIEW || incoming_type == BLOCK_COMPLETION)
- && (existing.block_type == BLOCK_REVIEW
- || existing.block_type == BLOCK_COMPLETION)) {
- return;
- }
- }
- if (existing.question != question) {
- continue;
- }
- if (existing.answer == null || !poll_state.is_delivered(existing.id)) {
- return;
- }
- }
- yield tasks.add_block(task_id, question,
- block_string_or(block, "type", "block_type"),
- EntryJson.string_array_member(block, block.has_member("options") ? "options" : "block_options"),
- EntryJson.optional_string_member(block, block.has_member("body") ? "body" : "block_body"));
- }
- /** The first non-empty of two string members (empty when both are). */
- private static string block_string_or(Json.Object obj, string primary, string fallback) {
- string value = EntryJson.string_member(obj, primary);
- return value.length > 0 ? value : EntryJson.string_member(obj, fallback);
- }
- /**
- * The question a daemon block is recognised by: `question`
- * (or the `/status`-style `block_question`) verbatim, or the
- * rfc fallback when the daemon raised an RFC (the
- * `saddle block rfc` shape carries its document as the body and
- * no question operand). Both the block-raising and answer-
- * delivery paths key blocks by this string.
- */
- private static string block_question_from_payload(Json.Object block) {
- string question = block_string_or(block, "question", "block_question");
- if (question.length > 0) {
- return question;
- }
- return block_string_or(block, "type", "block_type") == BLOCK_RFC
- ? "Request for comments" : "";
- }
- /**
- * Applies a pushed state transition (`POST /api/saddle/results`,
- * plan 10 phase 2) — the same task/saddle bookkeeping the
- * deleted /status poll switch ran, driven by the daemon's push:
- * `done`/`failed` complete or fail the task with the pushed
- * summary (closing the session through the release owner) —
- * except on a goal-run session, where they submit the
- * completion report or fail the goal (plan 12); `blocked`
- * raises the block payload as an answerable block on the
- * goal run's OWN task row (plan 15 C1: the goal agent's
- * blocks ride the run task — no mirrored second row exists)
- * and parks the round (the raise closes its session 'parked',
- * D4); `suspended`
- * parks the same way — for a goal agent the resume is driven
- * by the goal's task-terminal events; and `working` flips an
- * ASSIGNED task to working (a parked task keeps its park —
- * the verdict arrives as a resuming allocation, never as a
- * state push; plan 16 D4). No saddle-row write remains
- * (plan 16 item 23): busy/idle derive from the sessions.
- *
- * Plan 13 B2: a push against a task that is already terminal —
- * or a goal-context push against a finished goal — is an
- * outbox redelivery and is consumed silently, whatever state
- * it carries; terminal transitions and their narrative fire
- * once, ever.
- */
- public async void apply_result_state(ClankerSaddle.Data.Session session,
- string state, Json.Object? block, string? summary, TaskService tasks) throws Error {
- int64 task_id = session.task_id;
- int64 goal_id = yield goal_id_of_run(task_id);
- bool terminal = yield task_is_terminal(task_id);
- if (!terminal && goal_id > 0) {
- terminal = yield goal_is_terminal(goal_id);
- }
- if (terminal) {
- return;
- }
- switch (state) {
- case "done":
- if (goal_id > 0) {
- // Goal context: `saddle done` submits the
- // completion report and enters completion
- // review — the run parks on the mirrored
- // block (submit_completion's raise already
- // closed the session 'parked'), it does not
- // complete.
- yield goal_service.submit_completion(goal_id,
- summary != null && summary.length > 0 ? summary
- : "The goal agent submitted its completion report");
- break;
- }
- yield tasks.complete_task(task_id, summary, OUTCOME_DONE);
- break;
- case "failed":
- string explanation = summary ?? "";
- if (goal_id > 0) {
- yield goal_service.fail_goal(goal_id,
- explanation.length > 0 ? explanation : null);
- break;
- }
- yield tasks.fail_task(task_id, explanation.length > 0 ? explanation : null);
- break;
- case "blocked":
- if (goal_id > 0) {
- // Goal context: the block lands on the run
- // task itself (plan 15 C1) — a plan payload
- // enters plan review, anything else raises a
- // goal question/RFC; the run parks on that
- // very row for the verdict resume. A BLOCKED
- // push WITHOUT a
- // block payload (the park-unwinding round-end
- // push, its trailing entries) raises nothing —
- // the payload-carrying transition already did,
- // and a synthesised fallback row would dupe it.
- if (block == null) {
- yield tasks.set_status(task_id, STATUS_BLOCKED);
- yield tasks.park_open_sessions(task_id);
- break;
- }
- string question = block_question_from_payload((!)block);
- if (question.length == 0) {
- question = "The goal agent raised a question";
- }
- if (block_string_or((!)block, "type", "block_type") == BLOCK_PLAN) {
- yield goal_service.submit_plan(goal_id,
- EntryJson.optional_string_member((!)block, "body") ?? question);
- } else if (block_string_or((!)block, "type", "block_type") == BLOCK_REVIEW) {
- // The daemon's submit_report done-park: a
- // review-kind block on the goal run IS the
- // completion submission (submit_completion
- // adopts the raised row — one row, whichever
- // path landed first — and drives the
- // completion_review transition).
- yield goal_service.submit_completion(goal_id,
- EntryJson.optional_string_member((!)block, "body") ?? question);
- } else {
- yield goal_service.raise_agent_block(goal_id, question,
- block_string_or((!)block, "type", "block_type"),
- EntryJson.string_array_member((!)block,
- (!)block.has_member("options") ? "options" : "block_options"),
- EntryJson.optional_string_member((!)block, "body"));
- }
- yield tasks.set_status(task_id, STATUS_BLOCKED);
- break;
- }
- yield ensure_block(tasks, task_id, block != null ? (!)block : new Json.Object());
- {
- // A review-kind block parks as review_requested
- // (the banner state); every other kind as
- // blocked.
- bool review = block != null
- && block_string_or((!)block, "type", "block_type") == BLOCK_REVIEW;
- yield tasks.set_status(task_id,
- review ? STATUS_REVIEW_REQUESTED : STATUS_BLOCKED);
- }
- // The daemon parks the task's workspace while
- // blocked, freeing the saddle for other work
- // (plan 16 item 23: the raise already closed the
- // round's session 'parked' and the task's blocked
- // status IS the pause — no saddle-row write
- // remains; busy/idle derive from the sessions).
- break;
- case "suspended":
- // `saddle suspend`: the same parking as a block, but
- // the web drives resumption — a wake of the affine
- // saddle once a goal event lands, the resume
- // assignment carrying the coalesced report.
- // (set_parked closed the round's session 'parked';
- // the freed saddle is the derived view's reading.)
- //
- // Goal 27 refinements — the PARKING GUARD: a goal
- // run parks so its OWNED tasks' terminal
- // transitions can wake it. When the goal owns no
- // task whose terminal transition could ever fire
- // (no non-terminal worker — only done/failed rows
- // and drafts nobody enqueued), the park would be a
- // dead sleep: nobody but a human force-wake could
- // end it. Refuse it instead — the park mechanics
- // run but the one-shot refusal notice makes the
- // run resume-offerable IMMEDIATELY, so the
- // daemon's next ask hands it "Parking refused — …"
- // instead of a sleep nothing will ever wake. A
- // live task anywhere under the goal keeps the
- // normal park.
- if (goal_id > 0 && !(yield goal_has_wakeable_task(goal_id))) {
- yield refuse_park(task_id,
- "Parking refused: the goal owns no live tasks to wait on —"
- + " finish the run (submit your report) or cancel.");
- break;
- }
- // A PLAIN task parking is ALLOWED (no guard): it
- // has no wake source either way, but its park is a
- // human-facing pause by design — the task page's
- // force-wake is the resume path a goal run lacks.
- yield tasks.set_parked(task_id);
- break;
- case "working":
- // A working push only advances a task that is
- // actually running — assigned (the claim's flip)
- // or a legacy never-allocated queued row. A
- // PARKED task (blocked / review / suspended, its
- // session closed at the raise — plan 16 D4) keeps
- // its park: the verdict arrives as a resuming
- // allocation, never as a state push, so a
- // park-unwinding or late round push can never
- // resurrect the workspace as "working" out from
- // under its review banner.
- var advance = connection.create_command(
- "UPDATE tasks SET status = 'working' WHERE id = :id"
- + " AND status IN ('assigned', 'queued')"
- + " AND status != 'working'");
- advance.with_parameter<Invercargill.Element>("id",
- elem<int64?>(task_id));
- bool flipped = (yield advance.execute_non_query_async()) > 0;
- var current = yield db.query<TaskEntity>()
- .where(expr("id == $0", elem<int64?>(task_id)))
- .first_async();
- if (flipped) {
- var moved = new Json.Object();
- moved.set_string_member("from", STATUS_ASSIGNED);
- moved.set_string_member("to", STATUS_WORKING);
- moved.set_string_member("text", "Status → working");
- yield event_service.append(EVENT_STATE_CHANGE,
- EVENT_ACTOR_SYSTEM, EVENT_SCOPE_TASK, task_id, moved);
- }
- // started_at (and the first-start push) only stamp
- // for work that never allocated; the guarded
- // advance above means the re-read task is working
- // exactly when the flip landed or it already ran.
- if (current != null && ((!)current).status == STATUS_WORKING
- && ((!)current).started_at == null) {
- ((!)current).started_at = new DateTime.now_local();
- yield db.update_async<TaskEntity>((!)current);
- // The first-start push lives at the allocation
- // (where started_at is stamped for dispatched
- // tasks); this path only covers legacy status
- // pushes for never-allocated work.
- push.notify_opted_in(task_id, PushService.lifecycle_notification(
- ((!)current).title, task_id, "started", null));
- }
- break;
- default:
- break;
- }
- }
- /** True when the task's status is done or failed. */
- private async bool task_is_terminal(int64 task_id) throws Error {
- var task = yield db.query<TaskEntity>()
- .where(expr("id == $0", elem<int64?>(task_id)))
- .first_async();
- return task != null && (((!)task).status == STATUS_DONE
- || ((!)task).status == STATUS_FAILED);
- }
- /** True when the goal's status is complete, failed or cancelled. */
- private async bool goal_is_terminal(int64 goal_id) throws Error {
- var goal = yield db.query<Goal>()
- .where(expr("id == $0", elem<int64?>(goal_id)))
- .first_async();
- return goal != null && (((!)goal).status == GOAL_COMPLETE
- || ((!)goal).status == GOAL_FAILED || ((!)goal).status == GOAL_CANCELLED);
- }
- /**
- * The running-liveness probe (plan 10 phase 2): a busy saddle's
- * session has seen neither chunks nor results for the staleness
- * window, so ONE exceptional `GET /status` decides its fate —
- * an answer means the daemon is alive (the health snapshot is
- * refreshed and the task keeps running; the silence episode
- * gets no second probe), a failure requeues the task with an agent
- * comment and marks the saddle offline (the normal unreachable
- * bookkeeping). The probe flag was taken by the caller; the
- * next push resets it.
- */
- public async void probe_stale_saddle(Saddle saddle, TaskService tasks,
- int64 running_task_id) throws Error {
- try {
- var status = yield request_saddle_json(saddle.host, "GET", "/status", null);
- yield record_status_probe(saddle, status);
- // An answer proves the DAEMON lives - not that it is
- // running THIS task. A daemon that crashed and
- // restarted answers from its idle loop (task_id null,
- // or another task it won since): the stale session is
- // a crash orphan, and its task requeues through the
- // same guarded bookkeeping as the unreachable path
- // (the comment names the restart). The saddle itself
- // stays as-is - it is demonstrably alive and asking
- // for work; stamping it offline would only block the
- // requeued task's next claim until another ask.
- var answered_task = status.has_member("task_id")
- && status.get_member("task_id").get_node_type() == Json.NodeType.VALUE
- && status.get_member("task_id").get_value_type() == typeof(int64)
- ? status.get_int_member("task_id") : 0;
- if (answered_task != running_task_id) {
- var flip = connection.create_command(
- "UPDATE tasks SET status = 'queued' WHERE id = :id"
- + " AND status IN ('assigned', 'working')");
- flip.with_parameter<Invercargill.Element>("id",
- elem<int64?>(running_task_id));
- if ((yield flip.execute_non_query_async()) > 0) {
- yield tasks.add_system_comment(running_task_id,
- @"Saddle '$(saddle.name)' restarted mid-run (its status probe answered without the task); task requeued");
- yield tasks.release_task(running_task_id, OUTCOME_STOPPED, false);
- }
- yield pulse.bump(0, "crash orphan requeued at probe");
- print(@"[ClankerSaddle] Stale saddle '$(saddle.name)' answered its probe without task $(running_task_id); requeued\n");
- return;
- }
- print(@"[ClankerSaddle] Stale saddle '$(saddle.name)' answered its exceptional status probe; task $(running_task_id) keeps running\n");
- } catch (Error e) {
- warning(@"[ClankerSaddle] Saddle '$(saddle.name)' sent neither chunks nor results for $(stale_after_seconds())s and its status probe failed ($(e.message)); marking offline");
- yield mark_saddle_unreachable(saddle.id, tasks,
- @"no chunks or results for $(stale_after_seconds())s; status probe failed ($(e.message))");
- }
- }
- /**
- * Performs an HTTP request against a saddle daemon and parses the
- * response body as JSON; transport failures propagate to the
- * caller (the dispatcher marks the saddle unreachable). Transport
- * blips — typically a pooled keep-alive connection closed by the
- * daemon at the same instant the request reused it — are retried
- * once before giving up. Unlike {@link request_saddle_json} the
- * HTTP status is returned rather than thrown, and array bodies
- * parse too.
- */
- private async int request_saddle(string host, string method, string path,
- owned Json.Builder? payload, out Json.Node? root) throws Error {
- root = null;
- GLib.Error? failure = null;
- for (var attempt = 0; attempt < 2; attempt++) {
- try {
- root = null;
- int status = yield request_saddle_once(host, method, path, payload, out root);
- return status;
- } catch (Error e) {
- failure = e;
- }
- if (((!)failure).code != 13 && ((!)failure).code != 14) {
- break;
- }
- GLib.Timeout.add(250, request_saddle.callback);
- yield;
- }
- throw (!)failure;
- }
- private async int request_saddle_once(string host, string method, string path,
- Json.Builder? payload, out Json.Node? root) throws Error {
- root = null;
- string base_url = host.has_prefix("http") ? host : @"http://$(host)";
- var message = new Soup.Message(method, @"$(base_url)$(path)");
- if (payload != null) {
- var generator = new Json.Generator();
- generator.set_root(((!)payload).get_root());
- message.set_request_body_from_bytes("application/json",
- new Bytes(generator.to_data(null).data));
- }
- InputStream input;
- try {
- input = yield http.send_async(message, Priority.DEFAULT, null);
- } catch (Error e) {
- throw new Error(Quark.from_string("clanker-saddle"), 13,
- @"Saddle request to $(base_url)$(path) failed: $(e.message)");
- }
- var output = new MemoryOutputStream.resizable();
- try {
- yield output.splice_async(input, OutputStreamSpliceFlags.CLOSE_SOURCE,
- Priority.DEFAULT, null);
- yield output.close_async(Priority.DEFAULT, null);
- } catch (Error e) {
- throw new Error(Quark.from_string("clanker-saddle"), 14,
- @"Failed reading saddle response: $(e.message)");
- }
- var written = output.get_data_size();
- uint8[] data = output.steal_data();
- // steal_data's array length is -1 and the buffer is sized to
- // the data — there is no NUL terminator to rely on — so slice
- // by the written byte count before reading it as a string.
- string response_text = written > 0 && data != null && data[0] != '\0'
- ? ((string) data).substring(0, (long) written) : "{}";
- var parser = new Json.Parser();
- try {
- parser.load_from_data(response_text);
- } catch (Error e) {
- throw new Error(Quark.from_string("clanker-saddle"), 16,
- @"Saddle returned invalid JSON: $(e.message)");
- }
- root = parser.get_root();
- return (int) message.status_code;
- }
- /**
- * Performs an HTTP request against a saddle daemon and returns the
- * parsed JSON object body (an empty object for non-object bodies);
- * HTTP statuses of 400 and above throw.
- */
- private async Json.Object request_saddle_json(string host, string method, string path,
- owned Json.Builder? payload) throws Error {
- Json.Node? root = null;
- int status_code = yield request_saddle(host, method, path, (owned) payload, out root);
- if (status_code == 409) {
- // Machine-readable per the wire contract: the body (state,
- // running_task_id) rides in the message so the dispatch
- // path can distinguish zombie conflicts from transport
- // failures and recover.
- string body_text = "{}";
- if (root != null && root.get_node_type() == Json.NodeType.OBJECT) {
- var generator = new Json.Generator();
- generator.set_root(root);
- body_text = generator.to_data(null);
- }
- throw new Error(Quark.from_string("clanker-saddle"), 409,
- @"busy: $(body_text)");
- }
- if (status_code >= 400) {
- throw new Error(Quark.from_string("clanker-saddle"), 15,
- @"Saddle request returned HTTP $(status_code)");
- }
- if (root == null || root.get_node_type() != Json.NodeType.OBJECT) {
- return new Json.Object();
- }
- return root.get_object();
- }
- /**
- * A `{"name", "git_url", "path", "branch"}` project reference in
- * the saddle daemon's wire shape: `git_url` is the clone source,
- * and the checkout lives at
- * `SADDLE_PROJECTS_ROOT/task-<task_id>/<name>` inside the
- * saddle's container — one workspace per task, so concurrent
- * tasks never share a checkout.
- */
- /**
- * The projects root assignment paths are addressed against: the
- * ASKING saddle's registered `workspace_root` (the daemon's own
- * SADDLE_WORKSPACE_ROOT, reported at /register — goal 8 task 6,
- * so a fleet no longer assumes one shared /workspace mount),
- * else the `SADDLE_PROJECTS_ROOT` environment override, else
- * the historical "/workspace" default. Paths are ADVISORY: the
- * daemon owns its filesystem and rewrites them onto its own
- * workspace root.
- */
- private static string projects_root_for_saddle(Saddle? saddle) {
- var registered = saddle != null ? ((!)saddle).workspace_root : null;
- if (registered != null && ((!)registered).strip().length > 0) {
- return ((!)registered).strip();
- }
- var root_var = (Environment.get_variable("SADDLE_PROJECTS_ROOT") ?? "").strip();
- return root_var.length > 0 ? root_var : "/workspace";
- }
- private static Json.Node project_node(Project project, int64 task_id, bool is_reference = false,
- Saddle? saddle = null) {
- string projects_root = projects_root_for_saddle(saddle);
- var builder = new Json.Builder();
- builder.begin_object();
- builder.set_member_name("name");
- builder.add_string_value(project.name);
- builder.set_member_name("git_url");
- builder.add_string_value(project.git_url);
- builder.set_member_name("path");
- builder.add_string_value(is_reference
- ? @"$(projects_root)/task-$(task_id)/reference/$(project.name)"
- : @"$(projects_root)/task-$(task_id)/$(project.name)");
- builder.set_member_name("branch");
- builder.add_string_value(project.default_branch);
- builder.end_object();
- return builder.get_root();
- }
- /**
- * The answer as the saddle daemon expects it over /answer (or in a
- * resume payload): verbatim. The daemon applies its own
- * BlockRequest formatting, so this only strips the "Human
- * selected: …"/"Human answered (other): …" prefixes still stored
- * by answers recorded before the daemon took over formatting.
- */
- private static string raw_answer_for_saddle(string answer) {
- if (answer.has_prefix("Human selected: ")) {
- return answer.substring("Human selected: ".length);
- }
- if (answer.has_prefix("Human answered (other): ")) {
- return answer.substring("Human answered (other): ".length);
- }
- return answer;
- }
- }
- }
|