|
@@ -2488,15 +2488,20 @@ def bundle_dispatch(ctx):
|
|
|
act(auth, detail["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
|
|
act(auth, detail["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
|
|
|
check(wait_for(lambda: [c["name"] for c in
|
|
check(wait_for(lambda: [c["name"] for c in
|
|
|
task_state(auth, gpu_task)["capabilities"]] == ["gpu"]
|
|
task_state(auth, gpu_task)["capabilities"]] == ["gpu"]
|
|
|
- or False, timeout=60, what="capability attached") is not None,
|
|
|
|
|
|
|
+ or False, timeout=90, what="capability attached") is not None,
|
|
|
"task page lists the required capability")
|
|
"task page lists the required capability")
|
|
|
detail = task_state(auth, gpu_task)
|
|
detail = task_state(auth, gpu_task)
|
|
|
# The only idle saddle lacks the tag, so the queue flags the wait
|
|
# The only idle saddle lacks the tag, so the queue flags the wait
|
|
|
- # (read before any other task claims the idle saddle).
|
|
|
|
|
- queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
|
|
- if int(t["id"]) == gpu_task][0]
|
|
|
|
|
- check(queue["status"] == "queued" and queue["capabilities"] == ["gpu"]
|
|
|
|
|
- and queue.get("waiting_for_saddle") is True,
|
|
|
|
|
|
|
+ # (read before any other task claims the idle saddle). The state is
|
|
|
|
|
+ # stable until the any-saddle task below dispatches — poll for it
|
|
|
|
|
+ # rather than sampling once (a single read under load once sampled
|
|
|
|
|
+ # the row before the attach's pulse settled).
|
|
|
|
|
+ check(wait_for(lambda: (lambda row: row["status"] == "queued"
|
|
|
|
|
+ and row["capabilities"] == ["gpu"]
|
|
|
|
|
+ and row.get("waiting_for_saddle") is True)(
|
|
|
|
|
+ [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
|
|
+ if int(t["id"]) == gpu_task][0]) or False,
|
|
|
|
|
+ timeout=20, what="waiting flag") is not None,
|
|
|
"queue row shows the waiting task with its tag and wait flag")
|
|
"queue row shows the waiting task with its tag and wait flag")
|
|
|
|
|
|
|
|
task_id_tmp = create_queued(auth, {
|
|
task_id_tmp = create_queued(auth, {
|
|
@@ -2513,7 +2518,7 @@ def bundle_dispatch(ctx):
|
|
|
with open(os.path.join(WORK, "app.log")) as handle:
|
|
with open(os.path.join(WORK, "app.log")) as handle:
|
|
|
app_log = handle.read()
|
|
app_log = handle.read()
|
|
|
return "waits for a capable saddle" in app_log and "requires: gpu" in app_log
|
|
return "waits for a capable saddle" in app_log and "requires: gpu" in app_log
|
|
|
- check(wait_for(capability_wait_logged, timeout=45,
|
|
|
|
|
|
|
+ check(wait_for(capability_wait_logged, timeout=90,
|
|
|
what="capability wait log") is not None,
|
|
what="capability wait log") is not None,
|
|
|
"dispatcher logs the capability wait")
|
|
"dispatcher logs the capability wait")
|
|
|
|
|
|
|
@@ -3452,7 +3457,7 @@ def bundle_allocation(ctx):
|
|
|
"the affinity worker completes (on a non-affine racer)")
|
|
"the affinity worker completes (on a non-affine racer)")
|
|
|
|
|
|
|
|
check(wait_for(lambda: len(fake2.wake_log) > wakes_before_2 or False,
|
|
check(wait_for(lambda: len(fake2.wake_log) > wakes_before_2 or False,
|
|
|
- timeout=30, what="affine wake") is not None,
|
|
|
|
|
|
|
+ timeout=90, what="affine wake") is not None,
|
|
|
"the dispatcher's wake targets the affine saddle")
|
|
"the dispatcher's wake targets the affine saddle")
|
|
|
status, resume_payload = fake2.ask_now()
|
|
status, resume_payload = fake2.ask_now()
|
|
|
fake2.ask_on_wake = True # restored whatever the claim returned
|
|
fake2.ask_on_wake = True # restored whatever the claim returned
|
|
@@ -4811,9 +4816,10 @@ def bundle_answers(ctx):
|
|
|
" values (?, ?, strftime('%s','now'))", (unreachable_row, fake_id))
|
|
" values (?, ?, strftime('%s','now'))", (unreachable_row, fake_id))
|
|
|
conn.commit()
|
|
conn.commit()
|
|
|
conn.close()
|
|
conn.close()
|
|
|
- row = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
|
|
- if int(t["id"]) == unreachable_row][0]
|
|
|
|
|
- check(row.get("unreachable") is True,
|
|
|
|
|
|
|
+ check(wait_for(lambda: (lambda row: row.get("unreachable") is True)(
|
|
|
|
|
+ [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
|
|
+ if int(t["id"]) == unreachable_row][0]) or False,
|
|
|
|
|
+ timeout=20, what="unreachable flag") is not None,
|
|
|
"queue row shows the unreachable state in the window")
|
|
"queue row shows the unreachable state in the window")
|
|
|
|
|
|
|
|
print("== phase 1: startup migrates the per-saddle token ==")
|
|
print("== phase 1: startup migrates the per-saddle token ==")
|
|
@@ -5367,7 +5373,7 @@ def bundle_goals(ctx):
|
|
|
# (The plan script was armed before the run's first dispatch — see
|
|
# (The plan script was armed before the run's first dispatch — see
|
|
|
# the listing-hiding section above — so the first round submits it.)
|
|
# the listing-hiding section above — so the first round submits it.)
|
|
|
check(wait_for(lambda: goal_state(goal1)["status"] == "plan_review" or False,
|
|
check(wait_for(lambda: goal_state(goal1)["status"] == "plan_review" or False,
|
|
|
- timeout=60, what="goal agent submits its plan") is not None,
|
|
|
|
|
|
|
+ timeout=120, what="goal agent submits its plan") is not None,
|
|
|
"the goal agent's plan push enters plan review")
|
|
"the goal agent's plan push enters plan review")
|
|
|
state = goal_state(goal1)
|
|
state = goal_state(goal1)
|
|
|
check(task_state(auth, run_task)["status"] == "blocked",
|
|
check(task_state(auth, run_task)["status"] == "blocked",
|
|
@@ -6673,10 +6679,17 @@ def bundle_goalui(ctx):
|
|
|
conn.execute("update saddles set status = 'offline' where id = ?", (fake_id,))
|
|
conn.execute("update saddles set status = 'offline' where id = ?", (fake_id,))
|
|
|
conn.commit()
|
|
conn.commit()
|
|
|
conn.close()
|
|
conn.close()
|
|
|
- check(wait_for(lambda: goal_state(goal).get("requeue_run") is not None
|
|
|
|
|
- or False, timeout=30, what="requeue offered") is not None,
|
|
|
|
|
|
|
+
|
|
|
|
|
+ def requeue_offered():
|
|
|
|
|
+ # Act on the SAME state that satisfied the wait: a second read
|
|
|
|
|
+ # once raced the scripted daemon's ask reviving the saddle row,
|
|
|
|
|
+ # and the conditionally-authored action vanished between the
|
|
|
|
|
+ # two reads (KeyError on act).
|
|
|
|
|
+ offered = goal_state(goal)
|
|
|
|
|
+ return offered if offered.get("requeue_run") is not None else False
|
|
|
|
|
+ state = wait_for(requeue_offered, timeout=30, what="requeue offered")
|
|
|
|
|
+ check(state is not None,
|
|
|
"a parked run on a dead saddle offers Requeue-to-resume")
|
|
"a parked run on a dead saddle offers Requeue-to-resume")
|
|
|
- state = goal_state(goal)
|
|
|
|
|
check(state.get("retry_run") is None,
|
|
check(state.get("retry_run") is None,
|
|
|
"an active goal offers no Retry supervision")
|
|
"an active goal offers no Retry supervision")
|
|
|
open_before = open_session(run_task)
|
|
open_before = open_session(run_task)
|