Jelajahi Sumber

qa: load-marginal wait windows widened + two read-act gaps closed — the capability-wait/affine-wake/plan-review waits move to 90-120s (each fired once when external load tripled shard walls to 90-150s); the queue-row waiting/unreachable flags become short polls (stable states, once sampled before the pulse settled); goalui's requeue_run acts on the SAME state that satisfied the wait (a second read raced the scripted daemon's ask reviving the saddle and the conditionally-authored action vanished between reads)

clanker 1 bulan lalu
induk
melakukan
2d429de3b1
1 mengubah file dengan 28 tambahan dan 15 penghapusan
  1. 28 15
      test/web_stack_test.py

+ 28 - 15
test/web_stack_test.py

@@ -2488,15 +2488,20 @@ def bundle_dispatch(ctx):
     act(auth, detail["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
     check(wait_for(lambda: [c["name"] for c in
                             task_state(auth, gpu_task)["capabilities"]] == ["gpu"]
-                   or False, timeout=60, what="capability attached") is not None,
+                   or False, timeout=90, what="capability attached") is not None,
           "task page lists the required capability")
     detail = task_state(auth, gpu_task)
     # The only idle saddle lacks the tag, so the queue flags the wait
-    # (read before any other task claims the idle saddle).
-    queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
-             if int(t["id"]) == gpu_task][0]
-    check(queue["status"] == "queued" and queue["capabilities"] == ["gpu"]
-          and queue.get("waiting_for_saddle") is True,
+    # (read before any other task claims the idle saddle). The state is
+    # stable until the any-saddle task below dispatches — poll for it
+    # rather than sampling once (a single read under load once sampled
+    # the row before the attach's pulse settled).
+    check(wait_for(lambda: (lambda row: row["status"] == "queued"
+                            and row["capabilities"] == ["gpu"]
+                            and row.get("waiting_for_saddle") is True)(
+                        [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
+                         if int(t["id"]) == gpu_task][0]) or False,
+                   timeout=20, what="waiting flag") is not None,
           "queue row shows the waiting task with its tag and wait flag")
 
     task_id_tmp = create_queued(auth, {
@@ -2513,7 +2518,7 @@ def bundle_dispatch(ctx):
         with open(os.path.join(WORK, "app.log")) as handle:
             app_log = handle.read()
         return "waits for a capable saddle" in app_log and "requires: gpu" in app_log
-    check(wait_for(capability_wait_logged, timeout=45,
+    check(wait_for(capability_wait_logged, timeout=90,
                    what="capability wait log") is not None,
           "dispatcher logs the capability wait")
 
@@ -3452,7 +3457,7 @@ def bundle_allocation(ctx):
           "the affinity worker completes (on a non-affine racer)")
 
     check(wait_for(lambda: len(fake2.wake_log) > wakes_before_2 or False,
-                   timeout=30, what="affine wake") is not None,
+                   timeout=90, what="affine wake") is not None,
           "the dispatcher's wake targets the affine saddle")
     status, resume_payload = fake2.ask_now()
     fake2.ask_on_wake = True       # restored whatever the claim returned
@@ -4811,9 +4816,10 @@ def bundle_answers(ctx):
                  " values (?, ?, strftime('%s','now'))", (unreachable_row, fake_id))
     conn.commit()
     conn.close()
-    row = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
-           if int(t["id"]) == unreachable_row][0]
-    check(row.get("unreachable") is True,
+    check(wait_for(lambda: (lambda row: row.get("unreachable") is True)(
+                        [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
+                         if int(t["id"]) == unreachable_row][0]) or False,
+                   timeout=20, what="unreachable flag") is not None,
           "queue row shows the unreachable state in the window")
 
     print("== phase 1: startup migrates the per-saddle token ==")
@@ -5367,7 +5373,7 @@ def bundle_goals(ctx):
     # (The plan script was armed before the run's first dispatch — see
     # the listing-hiding section above — so the first round submits it.)
     check(wait_for(lambda: goal_state(goal1)["status"] == "plan_review" or False,
-                   timeout=60, what="goal agent submits its plan") is not None,
+                   timeout=120, what="goal agent submits its plan") is not None,
           "the goal agent's plan push enters plan review")
     state = goal_state(goal1)
     check(task_state(auth, run_task)["status"] == "blocked",
@@ -6673,10 +6679,17 @@ def bundle_goalui(ctx):
     conn.execute("update saddles set status = 'offline' where id = ?", (fake_id,))
     conn.commit()
     conn.close()
-    check(wait_for(lambda: goal_state(goal).get("requeue_run") is not None
-                   or False, timeout=30, what="requeue offered") is not None,
+
+    def requeue_offered():
+        # Act on the SAME state that satisfied the wait: a second read
+        # once raced the scripted daemon's ask reviving the saddle row,
+        # and the conditionally-authored action vanished between the
+        # two reads (KeyError on act).
+        offered = goal_state(goal)
+        return offered if offered.get("requeue_run") is not None else False
+    state = wait_for(requeue_offered, timeout=30, what="requeue offered")
+    check(state is not None,
           "a parked run on a dead saddle offers Requeue-to-resume")
-    state = goal_state(goal)
     check(state.get("retry_run") is None,
           "an active goal offers no Retry supervision")
     open_before = open_session(run_task)