|
|
@@ -6,20 +6,20 @@ using ClankerSaddle.Data;
|
|
|
namespace ClankerSaddle.Services {
|
|
|
|
|
|
/**
|
|
|
- * The task dispatcher (plan 10, phase 1): a five-second timer that
|
|
|
+ * The task dispatcher (plan 10, phase 2): a five-second timer that
|
|
|
* computes wakes — when a task becomes offerable and a capable
|
|
|
* SLEEPING saddle exists (its last ask returned 204), the daemon is
|
|
|
* `POST /wake`d and fetches the work itself via
|
|
|
* `POST /api/saddle/next-task`; parked tasks target their affine
|
|
|
* saddle first. The lost-wake guard declares a saddle offline when
|
|
|
* no ask arrives within the guard window after a wake, freeing the
|
|
|
- * work for the next capable sleeping saddle.
|
|
|
- *
|
|
|
- * The pre-flip machinery stays fully functional during phase 1
|
|
|
- * (rollback path): parked-task resumes on non-sleeping (pre-flip)
|
|
|
- * daemons still go out as /answer deliveries and /start pushes,
|
|
|
- * and busy saddles keep being polled for /output, /log and /status
|
|
|
- * with the 3-strike, heal and fence layers.
|
|
|
+ * work for the next capable sleeping saddle. The staleness pass
|
|
|
+ * probes a RUNNING daemon that has pushed neither chunks nor
|
|
|
+ * results for the SADDLE_STALE_S window — one exceptional
|
|
|
+ * `GET /status`; a failed probe requeues the task with a bot
|
|
|
+ * comment and marks the saddle offline. Round telemetry arrives as
|
|
|
+ * pushes (`/api/saddle/chunks`, `/api/saddle/results`); the
|
|
|
+ * dispatcher never polls for output.
|
|
|
*
|
|
|
* Each tick resolves fresh request-scoped services (so no database
|
|
|
* scope is held between ticks) and never overlaps with itself: a slow
|
|
|
@@ -41,14 +41,6 @@ namespace ClankerSaddle.Services {
|
|
|
private Invercargill.DataStructures.HashSet<string> capability_wait_logged =
|
|
|
new Invercargill.DataStructures.HashSet<string>();
|
|
|
|
|
|
- /**
|
|
|
- * Consecutive communication failures per saddle id. Transient
|
|
|
- * errors (a pooled connection dropped mid-reuse, a momentary
|
|
|
- * SQLite lock) must not tear down a running session, so a saddle
|
|
|
- * is only declared unreachable after three strikes in a row.
|
|
|
- */
|
|
|
- private Dictionary<string, string> strikes = new Dictionary<string, string>();
|
|
|
-
|
|
|
/** Starts the repeating timer. */
|
|
|
public void start() {
|
|
|
Timeout.add_seconds(5, () => {
|
|
|
@@ -84,15 +76,6 @@ namespace ClankerSaddle.Services {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- private int strike_count(int64 saddle_id) {
|
|
|
- string stored = "";
|
|
|
- return strikes.try_get(saddle_id.to_string(), out stored) ? int.parse(stored) : 0;
|
|
|
- }
|
|
|
-
|
|
|
- private void record_strike(int64 saddle_id, int count) {
|
|
|
- strikes.set(saddle_id.to_string(), count.to_string());
|
|
|
- }
|
|
|
-
|
|
|
/** SADDLE_LOST_WAKE_S (default 15s), in milliseconds. */
|
|
|
public static uint lost_wake_guard_ms() {
|
|
|
var raw = Environment.get_variable("SADDLE_LOST_WAKE_S") ?? "";
|
|
|
@@ -101,16 +84,16 @@ namespace ClankerSaddle.Services {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * One dispatch cycle: recovery passes first (stale-busy healing
|
|
|
- * frees saddles within the same tick they strand), then the
|
|
|
- * lost-wake guard offlines saddles whose wake went unanswered,
|
|
|
- * then the wake computation offers every offerable task to a
|
|
|
- * capable sleeping daemon. Parked-task resumes against
|
|
|
- * non-sleeping (pre-flip) daemons still run the /answer + /start
|
|
|
- * resume paths, subtask events are pushed, and busy saddles are
|
|
|
- * polled for output and status changes. Wake computation pauses
|
|
|
- * while {@link LlmEndpoint} is unconfigured (polling and
|
|
|
- * resumes continue).
|
|
|
+ * One dispatch cycle: the lost-wake guard offlines saddles whose
|
|
|
+ * wake went unanswered, then the wake computation offers every
|
|
|
+ * offerable task to a capable sleeping daemon (a parked task's
|
|
|
+ * answered block or terminal subtask event resumes it through
|
|
|
+ * the affinity allocation — no push paths remain). Pending
|
|
|
+ * subtask events and human comments are delivered to RUNNING
|
|
|
+ * daemons via `POST /task-update`, and silent runners get their
|
|
|
+ * one exceptional staleness probe. Wake computation pauses while
|
|
|
+ * {@link LlmEndpoint} is unconfigured (deliveries and probes
|
|
|
+ * continue).
|
|
|
*/
|
|
|
public async void tick() throws Error {
|
|
|
var scope = container.create_scope();
|
|
|
@@ -126,119 +109,38 @@ namespace ClankerSaddle.Services {
|
|
|
}
|
|
|
} else {
|
|
|
warned_no_llm = false;
|
|
|
- // Stale-busy healing comes first so a freed saddle is
|
|
|
- // dispatchable within the same tick; tasks stranded on
|
|
|
- // offline saddles requeue right after.
|
|
|
- yield saddles.heal_stale_busy_saddles(tasks);
|
|
|
- yield saddles.requeue_tasks_on_offline_saddles(tasks);
|
|
|
- yield saddles.recover_stranded_assignments(tasks);
|
|
|
- yield saddles.recover_fenced_saddles();
|
|
|
yield enforce_lost_wake_guard(saddles, tasks);
|
|
|
yield wake_sleeping_saddles(saddles, capabilities, allocation);
|
|
|
}
|
|
|
|
|
|
- yield resume_answered_tasks(tasks, saddles, scope.resolve<ProjectService>());
|
|
|
- yield resume_suspended_parents(tasks, saddles, scope.resolve<ProjectService>());
|
|
|
+ yield saddles.requeue_tasks_on_offline_saddles(tasks);
|
|
|
yield saddles.deliver_pending_subtask_events(tasks);
|
|
|
|
|
|
foreach (var saddle in yield saddles.list_busy_saddles()) {
|
|
|
+ // Human comments ride POST /task-update to the RUNNING
|
|
|
+ // daemon (best effort — a transport failure stays
|
|
|
+ // undelivered for the next tick; the staleness probe
|
|
|
+ // owns the daemon's fate, not a strike counter).
|
|
|
try {
|
|
|
- yield saddles.poll_saddle_updates(saddle.id, tasks);
|
|
|
yield saddles.deliver_pending_comments(saddle.id, saddle.current_task_id);
|
|
|
- record_strike(saddle.id, 0);
|
|
|
- } catch (Error e) {
|
|
|
- int count = strike_count(saddle.id) + 1;
|
|
|
- record_strike(saddle.id, count);
|
|
|
- warning(@"[ClankerSaddle] Polling saddle $(saddle.id) failed (attempt $(count)/3): $(e.message)");
|
|
|
- if (count >= 3) {
|
|
|
- yield abandon_saddle(saddle.id, saddles, tasks,
|
|
|
- @"daemon unreachable after $(count) failed polls: $(e.message)");
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Resumes blocked/review-requested tasks whose newest block has
|
|
|
- * been answered, on the saddle holding their open (parked)
|
|
|
- * session — the workspace-pause counterpart of the dispatch loop.
|
|
|
- * Capabilities do not apply here: the parked workspace pins the
|
|
|
- * task to the saddle that holds it.
|
|
|
- */
|
|
|
- private async void resume_answered_tasks(TaskService tasks, SaddleService saddles,
|
|
|
- ProjectService projects) throws Error {
|
|
|
- var candidates = new Vector<TaskEntity>();
|
|
|
- foreach (var status in new string[] { STATUS_BLOCKED, STATUS_REVIEW_REQUESTED }) {
|
|
|
- foreach (var task in yield tasks.list_tasks(status, null)) {
|
|
|
- candidates.add(task);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- foreach (var task in candidates) {
|
|
|
- TaskBlock? newest = null;
|
|
|
- foreach (var block in yield tasks.list_blocks(task.id)) {
|
|
|
- newest = block;
|
|
|
- }
|
|
|
- if (newest == null || ((!)newest).answer == null) {
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- var session = yield tasks.open_session_for_task(task.id);
|
|
|
- if (session == null) {
|
|
|
- continue;
|
|
|
- }
|
|
|
- try {
|
|
|
- yield saddles.resume_answered_task(task, (!)session, tasks, projects);
|
|
|
} catch (Error e) {
|
|
|
- warning(@"[ClankerSaddle] Resuming task $(task.id) failed: $(e.message)");
|
|
|
+ warning(@"[ClankerSaddle] Comment delivery to saddle $(saddle.id) failed: $(e.message)");
|
|
|
}
|
|
|
- }
|
|
|
- }
|
|
|
|
|
|
- /**
|
|
|
- * Resumes suspended parents whose subtasks produced events since
|
|
|
- * they parked (a subtask completed or failed): a fresh /start on
|
|
|
- * the saddle holding the parked workspace, carrying the coalesced
|
|
|
- * markdown status report of every subtask as the resume notice.
|
|
|
- * Resumes on EACH such event — not only when all subtasks finish.
|
|
|
- */
|
|
|
- private async void resume_suspended_parents(TaskService tasks, SaddleService saddles,
|
|
|
- ProjectService projects) throws Error {
|
|
|
- foreach (var parent in yield tasks.list_tasks(STATUS_SUSPENDED, null)) {
|
|
|
- if (!(yield tasks.has_terminal_subtask_since(parent))) {
|
|
|
+ if (saddle.current_task_id == 0) {
|
|
|
continue;
|
|
|
}
|
|
|
- var session = yield tasks.open_session_for_task(parent.id);
|
|
|
+ var session = yield tasks.open_session_for_task(saddle.current_task_id);
|
|
|
if (session == null) {
|
|
|
continue;
|
|
|
}
|
|
|
- try {
|
|
|
- string notice = yield tasks.subtask_status_report(parent.id);
|
|
|
- bool resumed = yield saddles.resume_answered_task(parent, (!)session,
|
|
|
- tasks, projects, notice);
|
|
|
- if (resumed) {
|
|
|
- // The parking window is over: a later suspension
|
|
|
- // starts a fresh event window.
|
|
|
- yield tasks.clear_suspension(parent.id);
|
|
|
- print(@"[ClankerSaddle] Resumed suspended task $(parent.id) with a subtask report\n");
|
|
|
- }
|
|
|
- } catch (Error e) {
|
|
|
- warning(@"[ClankerSaddle] Resuming suspended task $(parent.id) failed: $(e.message)");
|
|
|
+ if (poll_state.take_stale_probe(((!)session).id,
|
|
|
+ SaddleService.stale_after_seconds() * 1000)) {
|
|
|
+ yield saddles.probe_stale_saddle(saddle, tasks);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * Declares a saddle unreachable once its failures accumulated:
|
|
|
- * the task it held returns to the queue (with a bot comment) and
|
|
|
- * its session closes.
|
|
|
- */
|
|
|
- private async void abandon_saddle(int64 saddle_id, SaddleService saddles,
|
|
|
- TaskService tasks, string? reason = null) throws Error {
|
|
|
- record_strike(saddle_id, 0);
|
|
|
- yield saddles.mark_saddle_unreachable(saddle_id, tasks, reason);
|
|
|
- }
|
|
|
-
|
|
|
/**
|
|
|
* The lost-wake guard: a wake was sent, the guard window has
|
|
|
* passed, and still no ask arrived ({@link
|
|
|
@@ -268,10 +170,8 @@ namespace ClankerSaddle.Services {
|
|
|
* SLEEPING saddle and POST /wake — the daemon asks and the
|
|
|
* allocation transaction hands the task over. Parked tasks
|
|
|
* target their affine saddle first (only it can be allocated
|
|
|
- * them); a non-sleeping affine saddle (a pre-flip daemon, or
|
|
|
- * one blocked in its answer loop) is left to the /answer +
|
|
|
- * /start resume paths. Saddles with a pending wake inside the
|
|
|
- * guard window are skipped — the guard decides their fate.
|
|
|
+ * them). Saddles with a pending wake inside the guard window
|
|
|
+ * are skipped — the guard decides their fate.
|
|
|
*/
|
|
|
private async void wake_sleeping_saddles(SaddleService saddles,
|
|
|
CapabilityService capabilities, AllocationService allocation) throws Error {
|
|
|
@@ -285,7 +185,8 @@ namespace ClankerSaddle.Services {
|
|
|
&& !poll_state.wake_pending(((!)affine).id, lost_wake_guard_ms())) {
|
|
|
target = (!)affine;
|
|
|
}
|
|
|
- // Otherwise: the resume paths own the task this tick.
|
|
|
+ // Otherwise: the task stays offerable for the affine
|
|
|
+ // saddle's own idle-awake ask.
|
|
|
} else {
|
|
|
foreach (var candidate in yield saddles.list_idle_saddles()) {
|
|
|
if (!(yield capabilities.saddle_covers_task(candidate.id,
|
|
|
@@ -293,9 +194,6 @@ namespace ClankerSaddle.Services {
|
|
|
continue;
|
|
|
}
|
|
|
if (!poll_state.is_sleeping(candidate.id)) {
|
|
|
- // A pre-flip daemon never asks; waking it is
|
|
|
- // pointless. It stays eligible for the
|
|
|
- // rollback /start paths only.
|
|
|
continue;
|
|
|
}
|
|
|
if (poll_state.wake_pending(candidate.id, lost_wake_guard_ms())) {
|