|
@@ -389,6 +389,7 @@ def start_app():
|
|
|
"SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
|
|
"SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
|
|
|
"SADDLE_HTTP_TIMEOUT_S": "5",
|
|
"SADDLE_HTTP_TIMEOUT_S": "5",
|
|
|
"SADDLE_RAW_PROBE_INTERVAL_S": "5",
|
|
"SADDLE_RAW_PROBE_INTERVAL_S": "5",
|
|
|
|
|
+ "SADDLE_STRAND_GRACE_S": "10",
|
|
|
# Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
|
|
# Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
|
|
|
# resolves the ancient /lib64 copies): empty makes the loader use
|
|
# resolves the ancient /lib64 copies): empty makes the loader use
|
|
|
# the binary's RUNPATH, which is the dev stack it was built
|
|
# the binary's RUNPATH, which is the dev stack it was built
|
|
@@ -1873,6 +1874,30 @@ def main():
|
|
|
check(wait_task_done(auth, raw_retry, timeout=60) is not None,
|
|
check(wait_task_done(auth, raw_retry, timeout=60) is not None,
|
|
|
"raw retry task completes")
|
|
"raw retry task completes")
|
|
|
|
|
|
|
|
|
|
+
|
|
|
|
|
+ print("== R6 F4: stranded assigned recovery ==")
|
|
|
|
|
+ # The live incident's shape: task assigned, session open, zero
|
|
|
|
|
+ # output ever, daemon idle. Grace period 10s in the harness env.
|
|
|
|
|
+ fake.state = "idle"
|
|
|
|
|
+ conn = db()
|
|
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
|
|
|
|
|
+ " values (1, 'Stranded assigned', 'assigned', 2, strftime('%s','now'))")
|
|
|
|
|
+ stranded = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
|
|
+ conn.execute("insert into sessions (task_id, saddle_id, started_at)"
|
|
|
|
|
+ " values (?, ?, strftime('%s','now','-180 seconds'))", (stranded, fake_id))
|
|
|
|
|
+ # Saddle row idle (the busy@task shape is the stale-busy heal's
|
|
|
|
|
+ # case); this exercises the strand pass specifically.
|
|
|
|
|
+ conn.execute("update saddles set status = 'idle', current_task_id = 0,"
|
|
|
|
|
+ " allocated_at = null where name = 'fake-saddle'")
|
|
|
|
|
+ conn.commit()
|
|
|
|
|
+ conn.close()
|
|
|
|
|
+ check(wait_for(lambda: any(c["body"] == "task stranded with no daemon output; task requeued"
|
|
|
|
|
+ for c in task_state(auth, stranded).get("comments", []))
|
|
|
|
|
+ or False, timeout=40, what="stranded requeue comment") is not None,
|
|
|
|
|
+ "stranded assignment requeues with a bot comment")
|
|
|
|
|
+ check(wait_task_done(auth, stranded, timeout=90) is not None,
|
|
|
|
|
+ "recovered task dispatches and completes")
|
|
|
|
|
+
|
|
|
finally:
|
|
finally:
|
|
|
# Exit-code diagnostic: distinguishes a crashed app (negative
|
|
# Exit-code diagnostic: distinguishes a crashed app (negative
|
|
|
# returncode, e.g. -11 SIGSEGV) from a clean shutdown. When
|
|
# returncode, e.g. -11 SIGSEGV) from a clean shutdown. When
|