|
|
@@ -5943,6 +5943,10 @@ def bundle_replay(ctx):
|
|
|
auth = login("admin", "adminpass123")
|
|
|
ctx.app = app # teardown tracks the restarted process
|
|
|
ctx.auth = auth
|
|
|
+ # The restart wiped the in-memory ask bookkeeping; the daemon
|
|
|
+ # re-establishes sleeping with a fresh ask (the real daemon's
|
|
|
+ # idle-awake ask / 15-minute safety self-ask).
|
|
|
+ fake.ask_now()
|
|
|
status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
|
|
|
{"session_id": churn_session, "state": "done",
|
|
|
"entries": burst + fresh})
|
|
|
@@ -5950,6 +5954,134 @@ def bundle_replay(ctx):
|
|
|
and rows_for(src_session) == stored_once,
|
|
|
"a post-restart redelivery still stores nothing (dedup is persistent)")
|
|
|
|
|
|
+ print("== plan 13 B2: duplicate failed entries project once (task) ==")
|
|
|
+ # The post-mortem's honest burst: the same failure entry redelivered
|
|
|
+ # under a churned session AND a second, DISTINCT failure entry id
|
|
|
+ # in the same session — one transition, one comment, ever.
|
|
|
+ def agent_comments(task_id, body=None):
|
|
|
+ conn = db()
|
|
|
+ try:
|
|
|
+ if body is None:
|
|
|
+ return conn.execute(
|
|
|
+ "select count(*) from task_comments"
|
|
|
+ " where task_id = ? and author = 'agent'",
|
|
|
+ (task_id,)).fetchone()[0]
|
|
|
+ return conn.execute(
|
|
|
+ "select count(*) from task_comments"
|
|
|
+ " where task_id = ? and author = 'agent' and body = ?",
|
|
|
+ (task_id, body)).fetchone()[0]
|
|
|
+ finally:
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+ conn = db()
|
|
|
+ conn.execute(
|
|
|
+ "insert into tasks (project_id, title, status, priority, created_at)"
|
|
|
+ " values (1, 'Burst failure task', 'working', 5, strftime('%s','now'))")
|
|
|
+ conn.commit()
|
|
|
+ burst_task = conn.execute(
|
|
|
+ "select max(id) from tasks where title = 'Burst failure task'").fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ burst_session = open_session_on_fake(burst_task)
|
|
|
+ first_fail = [{"id": 6001, "command": "saddle failure",
|
|
|
+ "output": "bash -n rejected the script", "exit_code": 1,
|
|
|
+ "type": "failed", "timestamp": "2026-09-11T00:01:00Z"}]
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
|
|
|
+ {"session_id": burst_session, "state": "failed",
|
|
|
+ "entries": first_fail,
|
|
|
+ "summary": "bash -n rejected the script"})
|
|
|
+ check(status == 200
|
|
|
+ and task_state(auth, burst_task)["status"] == "failed",
|
|
|
+ "the failure entry + state pair fails the task")
|
|
|
+ check(agent_comments(burst_task, "bash -n rejected the script") == 1,
|
|
|
+ "the entry path and the state path together leave one failure comment")
|
|
|
+
|
|
|
+ # The burst: a DISTINCT failure entry id in the same session, plus
|
|
|
+ # both ids redelivered under a churned session — nothing re-runs.
|
|
|
+ second_fail = [{"id": 6002, "command": "saddle failure",
|
|
|
+ "output": "bash -n rejected the script", "exit_code": 1,
|
|
|
+ "type": "failed", "timestamp": "2026-09-11T00:01:01Z"}]
|
|
|
+ churned_burst = open_session_on_fake(burst_task)
|
|
|
+ for target in (burst_session, churned_burst):
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
|
|
|
+ {"session_id": target, "state": "failed",
|
|
|
+ "entries": first_fail + second_fail,
|
|
|
+ "summary": "bash -n rejected the script"})
|
|
|
+ check(status == 200,
|
|
|
+ "the burst push under %s is consumed, not rejected" % (
|
|
|
+ "its session" if target == burst_session else "a churned session"))
|
|
|
+ check(task_state(auth, burst_task)["status"] == "failed"
|
|
|
+ and agent_comments(burst_task) == 1,
|
|
|
+ "duplicate failed entries leave one transition and one comment")
|
|
|
+
|
|
|
+ print("== plan 13 B2: goal failure projects once ==")
|
|
|
+ # Goal context: `saddle failure` fails the goal ONCE — the 115
|
|
|
+ # goal-failed events of the live failure are structurally gone.
|
|
|
+ def goal_state(goal_id):
|
|
|
+ return slot_public(auth, "/goals/%d" % goal_id, "goal")
|
|
|
+
|
|
|
+ def goal_failed_events(goal_id):
|
|
|
+ conn = db()
|
|
|
+ try:
|
|
|
+ return conn.execute(
|
|
|
+ "select count(*) from goal_events"
|
|
|
+ " where goal_id = ? and kind = 'goal-failed'",
|
|
|
+ (goal_id,)).fetchone()[0]
|
|
|
+ finally:
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+ act(auth, slot_public(auth, "/goals", "goals")["create"], {
|
|
|
+ "project_id": "1", "title": "Burst goal", "brief": "Fails honestly.",
|
|
|
+ "branch_name": "goal/burst"})
|
|
|
+ conn = db()
|
|
|
+ burst_goal = conn.execute(
|
|
|
+ "select id from goals where title = 'Burst goal'").fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ fake.goal_script = [("suspend",)]
|
|
|
+ act(auth, goal_state(burst_goal)["start"], {})
|
|
|
+ conn = db()
|
|
|
+ run_task = conn.execute(
|
|
|
+ "select id from tasks where goal_id = ? and kind = 'goal_run'",
|
|
|
+ (burst_goal,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(wait_for(lambda: task_state(auth, run_task)["status"] == "suspended"
|
|
|
+ or False, timeout=60, what="burst goal agent parks") is not None,
|
|
|
+ "the burst goal's agent parks first")
|
|
|
+ conn = db()
|
|
|
+ run_session = conn.execute(
|
|
|
+ "select id from sessions where task_id = ? and ended_at is null",
|
|
|
+ (run_task,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+ goal_fail = [{"id": 7001, "command": "saddle failure",
|
|
|
+ "output": "the harness collapsed", "exit_code": 1,
|
|
|
+ "type": "failed", "timestamp": "2026-09-11T00:02:00Z"},
|
|
|
+ {"id": 7002, "command": "saddle failure",
|
|
|
+ "output": "the harness collapsed", "exit_code": 1,
|
|
|
+ "type": "failed", "timestamp": "2026-09-11T00:02:01Z"}]
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
|
|
|
+ {"session_id": run_session, "state": "failed",
|
|
|
+ "entries": goal_fail, "summary": "the harness collapsed"})
|
|
|
+ check(status == 200 and goal_state(burst_goal)["status"] == "failed",
|
|
|
+ "the goal agent's failure fails the goal")
|
|
|
+ check(goal_failed_events(burst_goal) == 1,
|
|
|
+ "a two-entry failure burst records exactly one goal-failed event")
|
|
|
+
|
|
|
+ # Redeliveries — same ids under the run's session AND under a
|
|
|
+ # churned session — plus the bare failed state again: still one.
|
|
|
+ churned_run = open_session_on_fake(run_task)
|
|
|
+ for target in (run_session, churned_run):
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
|
|
|
+ {"session_id": target, "state": "failed",
|
|
|
+ "entries": goal_fail,
|
|
|
+ "summary": "the harness collapsed"})
|
|
|
+ check(status == 200, "the goal-failure redelivery under %s is consumed"
|
|
|
+ % ("its session" if target == run_session else "a churned session"))
|
|
|
+ check(goal_state(burst_goal)["status"] == "failed"
|
|
|
+ and goal_failed_events(burst_goal) == 1,
|
|
|
+ "redelivered failures never re-run the goal failure (one event, ever)")
|
|
|
+ check(task_state(auth, run_task)["status"] == "failed",
|
|
|
+ "the goal-run task stays failed after the replays")
|
|
|
+
|
|
|
# Drain: nothing queued remains for the shard's shutdown.
|
|
|
check(queued_count() == 0, "the replay shard drains its queue")
|
|
|
|