Переглянути джерело

fix: raw-log capability probe retries on later polls (throttled, SADDLE_RAW_PROBE_INTERVAL_S default 60s) instead of one seed-time failure stripping the session's raw terminal view

clanker 1 місяць тому
батько
коміт
e654f61840

+ 24 - 0
src/Services/SaddlePollState.vala

@@ -19,6 +19,7 @@ namespace ClankerSaddle.Services {
 
 
         private Dictionary<string, string> last_stored_output_id = new Dictionary<string, string>();
         private Dictionary<string, string> last_stored_output_id = new Dictionary<string, string>();
         private Dictionary<string, string> last_stored_raw_seq = new Dictionary<string, string>();
         private Dictionary<string, string> last_stored_raw_seq = new Dictionary<string, string>();
+        private Dictionary<string, string> last_raw_probe_ms = new Dictionary<string, string>();
         private HashSet<string> raw_polling_started = new HashSet<string>();
         private HashSet<string> raw_polling_started = new HashSet<string>();
         private HashSet<string> delivered_answers = new HashSet<string>();
         private HashSet<string> delivered_answers = new HashSet<string>();
         private HashSet<string> delivered_comments = new HashSet<string>();
         private HashSet<string> delivered_comments = new HashSet<string>();
@@ -81,6 +82,29 @@ namespace ClankerSaddle.Services {
             state_lock.unlock();
             state_lock.unlock();
         }
         }
 
 
+        /**
+         * True when a fresh raw-log capability probe for `session_id` is
+         * due: never probed, or the last probe is older than
+         * `interval_ms` (a single transport blip at seed time must not
+         * permanently strip the session's raw terminal view).
+         */
+        public bool raw_probe_due(int64 session_id, uint interval_ms) {
+            state_lock.lock();
+            string stored = "";
+            bool due = !last_raw_probe_ms.try_get(session_id.to_string(), out stored)
+                || (get_monotonic_time() / 1000 - int64.parse(stored)) >= interval_ms;
+            state_lock.unlock();
+            return due;
+        }
+
+        /** Records a raw-log capability probe attempt for `session_id`. */
+        public void mark_raw_probe(int64 session_id) {
+            state_lock.lock();
+            last_raw_probe_ms.set(session_id.to_string(),
+                (get_monotonic_time() / 1000).to_string());
+            state_lock.unlock();
+        }
+
         /**
         /**
          * Marks `session_id`'s daemon-side resume as notified; returns
          * Marks `session_id`'s daemon-side resume as notified; returns
          * false when the progress entry already went out.
          * false when the progress entry already went out.

+ 44 - 0
src/Services/SaddleService.vala

@@ -1227,6 +1227,14 @@ namespace ClankerSaddle.Services {
         private async void store_new_raw_log(int64 saddle_id, int64 session_id) throws Error {
         private async void store_new_raw_log(int64 saddle_id, int64 session_id) throws Error {
             int64 since = poll_state.raw_since_for(session_id) ?? -1;
             int64 since = poll_state.raw_since_for(session_id) ?? -1;
             if (since < 0) {
             if (since < 0) {
+                // The capability probe never succeeded (a transport blip
+                // at seed time must not decide forever): retry it, capped
+                // so old daemons are not hammered.
+                if (!poll_state.raw_probe_due(session_id, raw_probe_interval_ms())) {
+                    return;
+                }
+                poll_state.mark_raw_probe(session_id);
+                yield probe_raw_log_capability(saddle_id, session_id);
                 return;
                 return;
             }
             }
             var saddle = yield get_saddle(saddle_id);
             var saddle = yield get_saddle(saddle_id);
@@ -1277,6 +1285,42 @@ namespace ClankerSaddle.Services {
             }
             }
         }
         }
 
 
+        /** SADDLE_RAW_PROBE_INTERVAL_S (default 60s) in milliseconds. */
+        private static uint raw_probe_interval_ms() {
+            var raw = Environment.get_variable("SADDLE_RAW_PROBE_INTERVAL_S") ?? "";
+            int64 parsed = int64.parse(raw);
+            return parsed > 0 ? (uint) (parsed * 1000) : 60000;
+        }
+
+        /**
+         * One raw-log capability probe attempt: on success the session's
+         * raw cursor starts at the daemon's current `next` and later
+         * polls store the chunks produced after it.
+         */
+        private async void probe_raw_log_capability(int64 saddle_id, int64 session_id) throws Error {
+            var saddle = yield get_saddle(saddle_id);
+            if (saddle == null) {
+                return;
+            }
+            try {
+                Json.Node? root = null;
+                int code = yield request_saddle(((!)saddle).host, "GET", "/log?since=0",
+                    null, out root);
+                if (code >= 400 || root == null || root.get_node_type() != Json.NodeType.OBJECT) {
+                    return;
+                }
+                var log = root.get_object();
+                int64 next = log.has_member("next")
+                    && log.get_member("next").get_value_type() == typeof(int64)
+                    ? log.get_int_member("next") : 0;
+                if (next > 0) {
+                    poll_state.start_raw_polling(session_id, next);
+                }
+            } catch (Error e) {
+                debug(@"[ClankerSaddle] Raw log probe retry for session $(session_id) failed: $(e.message)");
+            }
+        }
+
         /**
         /**
          * Projects the agent's conversation-level events onto the task:
          * Projects the agent's conversation-level events onto the task:
          * `saddle comment` lines become bot comments, `saddle subtask`
          * `saddle comment` lines become bot comments, `saddle subtask`

+ 33 - 0
test/web_stack_test.py

@@ -215,6 +215,7 @@ class FakeSaddle:
         self.subtask_status_events = []  # recorded POST /subtask-status payloads
         self.subtask_status_events = []  # recorded POST /subtask-status payloads
         self.fail_subtask_status = False  # exercise the web's retry path
         self.fail_subtask_status = False  # exercise the web's retry path
         self.hang_next = {}              # path suffix -> hang count (never responds)
         self.hang_next = {}              # path suffix -> hang count (never responds)
+        self.fail_log_count = 0          # /log requests failing (500) while > 0
 
 
     def set_state(self, state, summary=None):
     def set_state(self, state, summary=None):
         """Advances the scripted lifecycle.
         """Advances the scripted lifecycle.
@@ -351,6 +352,10 @@ class FakeSaddle:
                     self.daemon.pending_output = []
                     self.daemon.pending_output = []
                 self._send(200, entries)
                 self._send(200, entries)
             elif self.path.startswith("/log"):
             elif self.path.startswith("/log"):
+                if self.daemon.fail_log_count > 0:
+                    self.daemon.fail_log_count -= 1
+                    self._send(500, {"error": "log unavailable (simulated)"})
+                    return
                 query = urllib.parse.urlparse(self.path).query
                 query = urllib.parse.urlparse(self.path).query
                 since = int(urllib.parse.parse_qs(query).get("since", ["0"])[0])
                 since = int(urllib.parse.parse_qs(query).get("since", ["0"])[0])
                 # Simulate the live PTY: each poll produces one more chunk
                 # Simulate the live PTY: each poll produces one more chunk
@@ -383,6 +388,7 @@ def start_app():
         "SADDLE_LLM_BASE_URL": "http://127.0.0.1:%d" % DAEMON_PORT,
         "SADDLE_LLM_BASE_URL": "http://127.0.0.1:%d" % DAEMON_PORT,
         "SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
         "SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
         "SADDLE_HTTP_TIMEOUT_S": "5",
         "SADDLE_HTTP_TIMEOUT_S": "5",
+        "SADDLE_RAW_PROBE_INTERVAL_S": "5",
         # Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
         # Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
         # resolves the ancient /lib64 copies): empty makes the loader use
         # resolves the ancient /lib64 copies): empty makes the loader use
         # the binary's RUNPATH, which is the dev stack it was built
         # the binary's RUNPATH, which is the dev stack it was built
@@ -1840,6 +1846,33 @@ def main():
         check(wait_task_done(auth, parked, timeout=90) is not None,
         check(wait_task_done(auth, parked, timeout=90) is not None,
               "parked parent still resumes after the refresh")
               "parked parent still resumes after the refresh")
 
 
+
+        print("== R6 F3: raw-log probe retries instead of deciding once ==")
+        fake.suspend_after_start = False
+        fake.fail_log_count = 1  # the seed-time probe fails exactly once
+        act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
+            "project_id": "1",
+            "title": "Raw retry task",
+            "priority": "2",
+        })
+        raw_retry = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
+                         if t["title"] == "Raw retry task"][0])
+
+        def raw_rows_for(task_id):
+            conn = db()
+            try:
+                return conn.execute(
+                    "select count(*) from session_raw_log where session_id in"
+                    " (select id from sessions where task_id = ?)", (task_id,)).fetchone()[0]
+            finally:
+                conn.close()
+
+        check(wait_for(lambda: raw_rows_for(raw_retry) > 0 or False,
+                       timeout=60, what="raw log rows appear after retry") is not None,
+              "a failed seed probe retries and the raw log fills")
+        check(wait_task_done(auth, raw_retry, timeout=60) is not None,
+              "raw retry task completes")
+
     finally:
     finally:
         # Exit-code diagnostic: distinguishes a crashed app (negative
         # Exit-code diagnostic: distinguishes a crashed app (negative
         # returncode, e.g. -11 SIGSEGV) from a clean shutdown. When
         # returncode, e.g. -11 SIGSEGV) from a clean shutdown. When