|
@@ -1564,6 +1564,35 @@ namespace ClankerSaddle.Services {
|
|
|
try {
|
|
try {
|
|
|
var status = yield request_saddle_json(saddle.host, "GET", "/status", null);
|
|
var status = yield request_saddle_json(saddle.host, "GET", "/status", null);
|
|
|
yield record_status_probe(saddle, status);
|
|
yield record_status_probe(saddle, status);
|
|
|
|
|
+ // An answer proves the DAEMON lives - not that it is
|
|
|
|
|
+ // running THIS task. A daemon that crashed and
|
|
|
|
|
+ // restarted answers from its idle loop (task_id null,
|
|
|
|
|
+ // or another task it won since): the stale session is
|
|
|
|
|
+ // a crash orphan, and its task requeues through the
|
|
|
|
|
+ // same guarded bookkeeping as the unreachable path
|
|
|
|
|
+ // (the comment names the restart). The saddle itself
|
|
|
|
|
+ // stays as-is - it is demonstrably alive and asking
|
|
|
|
|
+ // for work; stamping it offline would only block the
|
|
|
|
|
+ // requeued task's next claim until another ask.
|
|
|
|
|
+ var answered_task = status.has_member("task_id")
|
|
|
|
|
+ && status.get_member("task_id").get_node_type() == Json.NodeType.VALUE
|
|
|
|
|
+ && status.get_member("task_id").get_value_type() == typeof(int64)
|
|
|
|
|
+ ? status.get_int_member("task_id") : 0;
|
|
|
|
|
+ if (answered_task != running_task_id) {
|
|
|
|
|
+ var flip = connection.create_command(
|
|
|
|
|
+ "UPDATE tasks SET status = 'queued' WHERE id = :id"
|
|
|
|
|
+ + " AND status IN ('assigned', 'working')");
|
|
|
|
|
+ flip.with_parameter<Invercargill.Element>("id",
|
|
|
|
|
+ elem<int64?>(running_task_id));
|
|
|
|
|
+ if ((yield flip.execute_non_query_async()) > 0) {
|
|
|
|
|
+ yield tasks.add_system_comment(running_task_id,
|
|
|
|
|
+ @"Saddle '$(saddle.name)' restarted mid-run (its status probe answered without the task); task requeued");
|
|
|
|
|
+ yield tasks.release_task(running_task_id, OUTCOME_STOPPED, false);
|
|
|
|
|
+ }
|
|
|
|
|
+ yield pulse.bump(0, "crash orphan requeued at probe");
|
|
|
|
|
+ print(@"[ClankerSaddle] Stale saddle '$(saddle.name)' answered its probe without task $(running_task_id); requeued\n");
|
|
|
|
|
+ return;
|
|
|
|
|
+ }
|
|
|
print(@"[ClankerSaddle] Stale saddle '$(saddle.name)' answered its exceptional status probe; task $(running_task_id) keeps running\n");
|
|
print(@"[ClankerSaddle] Stale saddle '$(saddle.name)' answered its exceptional status probe; task $(running_task_id) keeps running\n");
|
|
|
} catch (Error e) {
|
|
} catch (Error e) {
|
|
|
warning(@"[ClankerSaddle] Saddle '$(saddle.name)' sent neither chunks nor results for $(stale_after_seconds())s and its status probe failed ($(e.message)); marking offline");
|
|
warning(@"[ClankerSaddle] Saddle '$(saddle.name)' sent neither chunks nor results for $(stale_after_seconds())s and its status probe failed ($(e.message)); marking offline");
|