|
|
@@ -215,6 +215,7 @@ class FakeSaddle:
|
|
|
self.subtask_status_events = [] # recorded POST /subtask-status payloads
|
|
|
self.fail_subtask_status = False # exercise the web's retry path
|
|
|
self.hang_next = {} # path suffix -> hang count (never responds)
|
|
|
+ self.fail_log_count = 0 # /log requests failing (500) while > 0
|
|
|
|
|
|
def set_state(self, state, summary=None):
|
|
|
"""Advances the scripted lifecycle.
|
|
|
@@ -351,6 +352,10 @@ class FakeSaddle:
|
|
|
self.daemon.pending_output = []
|
|
|
self._send(200, entries)
|
|
|
elif self.path.startswith("/log"):
|
|
|
+ if self.daemon.fail_log_count > 0:
|
|
|
+ self.daemon.fail_log_count -= 1
|
|
|
+ self._send(500, {"error": "log unavailable (simulated)"})
|
|
|
+ return
|
|
|
query = urllib.parse.urlparse(self.path).query
|
|
|
since = int(urllib.parse.parse_qs(query).get("since", ["0"])[0])
|
|
|
# Simulate the live PTY: each poll produces one more chunk
|
|
|
@@ -383,6 +388,7 @@ def start_app():
|
|
|
"SADDLE_LLM_BASE_URL": "http://127.0.0.1:%d" % DAEMON_PORT,
|
|
|
"SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
|
|
|
"SADDLE_HTTP_TIMEOUT_S": "5",
|
|
|
+ "SADDLE_RAW_PROBE_INTERVAL_S": "5",
|
|
|
# Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
|
|
|
# resolves the ancient /lib64 copies): empty makes the loader use
|
|
|
# the binary's RUNPATH, which is the dev stack it was built
|
|
|
@@ -1840,6 +1846,33 @@ def main():
|
|
|
check(wait_task_done(auth, parked, timeout=90) is not None,
|
|
|
"parked parent still resumes after the refresh")
|
|
|
|
|
|
+
|
|
|
+ print("== R6 F3: raw-log probe retries instead of deciding once ==")
|
|
|
+ fake.suspend_after_start = False
|
|
|
+ fake.fail_log_count = 1 # the seed-time probe fails exactly once
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Raw retry task",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ raw_retry = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Raw retry task"][0])
|
|
|
+
|
|
|
+ def raw_rows_for(task_id):
|
|
|
+ conn = db()
|
|
|
+ try:
|
|
|
+ return conn.execute(
|
|
|
+ "select count(*) from session_raw_log where session_id in"
|
|
|
+ " (select id from sessions where task_id = ?)", (task_id,)).fetchone()[0]
|
|
|
+ finally:
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+ check(wait_for(lambda: raw_rows_for(raw_retry) > 0 or False,
|
|
|
+ timeout=60, what="raw log rows appear after retry") is not None,
|
|
|
+ "a failed seed probe retries and the raw log fills")
|
|
|
+ check(wait_task_done(auth, raw_retry, timeout=60) is not None,
|
|
|
+ "raw retry task completes")
|
|
|
+
|
|
|
finally:
|
|
|
# Exit-code diagnostic: distinguishes a crashed app (negative
|
|
|
# returncode, e.g. -11 SIGSEGV) from a clean shutdown. When
|