SaddlePollState.vala 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320
  1. using Invercargill.DataStructures;
  2. using Inversion;
  3. namespace ClankerSaddle.Services {
  4. /**
  5. * Process-wide saddle poll bookkeeping: the block answers and human
  6. * comments already handed to a saddle (each delivered exactly once;
  7. * a restart re-delivers and the daemon deduplicates), plus the
  8. * plan-10 ask/wake liveness bookkeeping: the outcome of a saddle's
  9. * last `POST /api/saddle/next-task` ask (a saddle counts as SLEEPING
  10. * when its last ask returned 204) and the pending wake each sleeping
  11. * saddle was sent (the lost-wake guard's timer), and the phase-2
  12. * push-liveness clock: the last chunks/results arrival per session
  13. * (the staleness probe's timer — one exceptional probe per silent
  14. * episode, reset by the next push).
  15. *
  16. * The plan-13 entry-ingestion dedup deliberately does NOT live
  17. * here: it is daemon-global and persistent (the
  18. * `saddle_entry_dedup` table), so it survives restarts and session
  19. * churn — see SaddleService.store_pushed_entries.
  20. *
  21. * A singleton by necessity: {@link SaddleService} is request-scoped,
  22. * so this object is where the cross-request cursors live. Ids are
  23. * stringified: the Invercargill collections cannot key on value types.
  24. */
  25. public class SaddlePollState : Object {
  26. private Dictionary<string, string> probe_state = new Dictionary<string, string>();
  27. private Dictionary<string, string> probe_uptime = new Dictionary<string, string>();
  28. private Dictionary<string, string> probe_activity = new Dictionary<string, string>();
  29. private Dictionary<string, string> probe_contact = new Dictionary<string, string>();
  30. private HashSet<string> delivered_answers = new HashSet<string>();
  31. private HashSet<string> delivered_comments = new HashSet<string>();
  32. private HashSet<string> resumed_notified = new HashSet<string>();
  33. private Dictionary<string, string> ask_outcome = new Dictionary<string, string>();
  34. private HashSet<string> ask_seen = new HashSet<string>();
  35. private Dictionary<string, string> wake_at_ms = new Dictionary<string, string>();
  36. private Dictionary<string, string> wake_task = new Dictionary<string, string>();
  37. private Dictionary<string, string> push_at_ms = new Dictionary<string, string>();
  38. private HashSet<string> push_seen = new HashSet<string>();
  39. private HashSet<string> stale_probed = new HashSet<string>();
  40. // Plan 15 C4: the daemon-reported per-repo push flags of the
  41. // goal branches, as the LAST /status probe saw them — live
  42. // probe state only, NEVER a stored snapshot on the goal row.
  43. private Dictionary<string, string> probe_goal_branches = new Dictionary<string, string>();
  44. private Mutex state_lock = Mutex();
  45. /**
  46. * Records a daemon /status observation for the saddles-page
  47. * health panel: reported state, uptime, last-activity epoch and
  48. * the web-side contact time (monotonic ms).
  49. */
  50. public void record_probe(int64 saddle_id, string state, int64 uptime_seconds,
  51. int64 last_activity_epoch) {
  52. state_lock.lock();
  53. string key = saddle_id.to_string();
  54. probe_state.set(key, state);
  55. probe_uptime.set(key, uptime_seconds.to_string());
  56. probe_activity.set(key, last_activity_epoch.to_string());
  57. probe_contact.set(key, (get_monotonic_time() / 1000).to_string());
  58. state_lock.unlock();
  59. }
  60. /**
  61. * Records the daemon-reported goal-branch push flags (plan 15
  62. * C4): raw JSON as the probed daemon sent it — the goal page
  63. * and the completion-accept gate read it back live. A missing
  64. * member clears the record (nothing to render or enforce).
  65. */
  66. public void record_goal_branches(int64 saddle_id, string raw_json) {
  67. state_lock.lock();
  68. probe_goal_branches.set(saddle_id.to_string(), raw_json);
  69. state_lock.unlock();
  70. }
  71. /** The last probed goal-branch flags ("" when never reported). */
  72. public string goal_branches_for(int64 saddle_id) {
  73. state_lock.lock();
  74. string stored = "";
  75. bool found = probe_goal_branches.try_get(saddle_id.to_string(), out stored);
  76. state_lock.unlock();
  77. return found ? stored : "";
  78. }
  79. /** The last probe snapshot for `saddle_id` ("" when never probed). */
  80. public void health_for(int64 saddle_id, out string state, out int64 uptime,
  81. out int64 activity, out int64 contact_ms) {
  82. state_lock.lock();
  83. string key = saddle_id.to_string();
  84. string stored = "";
  85. state = probe_state.try_get(key, out stored) ? stored : "";
  86. uptime = int64.parse(probe_uptime.get_or_default(key) ?? "0");
  87. activity = int64.parse(probe_activity.get_or_default(key) ?? "0");
  88. contact_ms = int64.parse(probe_contact.get_or_default(key) ?? "0");
  89. state_lock.unlock();
  90. }
  91. /**
  92. * Marks `session_id`'s daemon-side resume as notified; returns
  93. * false when the progress entry already went out.
  94. */
  95. public bool mark_resumed(int64 session_id) {
  96. state_lock.lock();
  97. bool already = resumed_notified.has(session_id.to_string());
  98. if (!already) {
  99. resumed_notified.add(session_id.to_string());
  100. }
  101. state_lock.unlock();
  102. return !already;
  103. }
  104. /**
  105. * Marks `block_id`'s answer as delivered; returns false when it
  106. * had already been handed over.
  107. */
  108. public bool mark_delivered(int64 block_id) {
  109. state_lock.lock();
  110. bool already = delivered_answers.has(block_id.to_string());
  111. if (!already) {
  112. delivered_answers.add(block_id.to_string());
  113. }
  114. state_lock.unlock();
  115. return !already;
  116. }
  117. /** True when `block_id`'s answer was already handed over. */
  118. public bool is_delivered(int64 block_id) {
  119. state_lock.lock();
  120. bool already = delivered_answers.has(block_id.to_string());
  121. state_lock.unlock();
  122. return already;
  123. }
  124. /**
  125. * Marks `comment_id` as delivered to its saddle; returns false
  126. * when it had already been handed over.
  127. */
  128. public bool mark_comment_delivered(int64 comment_id) {
  129. state_lock.lock();
  130. bool already = delivered_comments.has(comment_id.to_string());
  131. if (!already) {
  132. delivered_comments.add(comment_id.to_string());
  133. }
  134. state_lock.unlock();
  135. return !already;
  136. }
  137. /** True when `comment_id` was already handed over. */
  138. public bool is_comment_delivered(int64 comment_id) {
  139. state_lock.lock();
  140. bool already = delivered_comments.has(comment_id.to_string());
  141. state_lock.unlock();
  142. return already;
  143. }
  144. /**
  145. * Records an ask arriving from `saddle_id` (plan 10): the
  146. * pending wake is answered — the lost-wake guard must not fire
  147. * for it — and the saddle is awake until the ask resolves. The
  148. * first ask also marks the daemon ask-capable (post-flip): its
  149. * parked-task resumes flow through wake + next-task from then
  150. * on, never through /start pushes.
  151. */
  152. public void mark_ask(int64 saddle_id) {
  153. state_lock.lock();
  154. string key = saddle_id.to_string();
  155. ask_outcome.set(key, "awake");
  156. ask_seen.add(key);
  157. wake_at_ms.remove(key);
  158. wake_task.remove(key);
  159. state_lock.unlock();
  160. }
  161. /**
  162. * Records the outcome of `saddle_id`'s ask: `204` puts it to
  163. * sleep (a later wake targets it), anything else leaves it
  164. * awake.
  165. */
  166. public void mark_ask_result(int64 saddle_id, bool assigned) {
  167. state_lock.lock();
  168. ask_outcome.set(saddle_id.to_string(), assigned ? "assigned" : "204");
  169. state_lock.unlock();
  170. }
  171. /**
  172. * True when `saddle_id`'s last ask returned 204 — the sleeping
  173. * state a wake targets.
  174. */
  175. public bool is_sleeping(int64 saddle_id) {
  176. state_lock.lock();
  177. string stored = "";
  178. bool sleeping = ask_outcome.try_get(saddle_id.to_string(), out stored)
  179. && stored == "204";
  180. state_lock.unlock();
  181. return sleeping;
  182. }
  183. /**
  184. * True once `saddle_id` has ever asked (post-flip daemon): it
  185. * asks whenever idle-awake, so its work — fresh or resumed —
  186. * always flows through the allocation.
  187. */
  188. public bool has_ever_asked(int64 saddle_id) {
  189. state_lock.lock();
  190. bool seen = ask_seen.has(saddle_id.to_string());
  191. state_lock.unlock();
  192. return seen;
  193. }
  194. /** Forgets the ask state (the saddle went offline / was removed). */
  195. public void clear_ask(int64 saddle_id) {
  196. state_lock.lock();
  197. string key = saddle_id.to_string();
  198. ask_outcome.remove(key);
  199. wake_at_ms.remove(key);
  200. wake_task.remove(key);
  201. state_lock.unlock();
  202. }
  203. /**
  204. * Records a wake sent to `saddle_id` for `task_id` — starts the
  205. * lost-wake guard's timer.
  206. */
  207. public void note_wake(int64 saddle_id, int64 task_id) {
  208. state_lock.lock();
  209. string key = saddle_id.to_string();
  210. wake_at_ms.set(key, (get_monotonic_time() / 1000).to_string());
  211. wake_task.set(key, task_id.to_string());
  212. state_lock.unlock();
  213. }
  214. /**
  215. * True when `saddle_id` has a pending wake at least
  216. * `guard_ms` old — an ask should have arrived by now; the
  217. * guard declares the saddle offline and frees the work for
  218. * the next capable one.
  219. */
  220. public bool wake_overdue(int64 saddle_id, uint guard_ms) {
  221. state_lock.lock();
  222. string stored = "";
  223. bool overdue = wake_at_ms.try_get(saddle_id.to_string(), out stored)
  224. && (get_monotonic_time() / 1000 - int64.parse(stored)) >= guard_ms;
  225. state_lock.unlock();
  226. return overdue;
  227. }
  228. /**
  229. * True when `saddle_id` carries a pending wake younger than
  230. * `guard_ms` — the wake pass skips it instead of re-waking on
  231. * every tick while the guard window runs.
  232. */
  233. public bool wake_pending(int64 saddle_id, uint guard_ms) {
  234. state_lock.lock();
  235. string stored = "";
  236. bool pending = wake_at_ms.try_get(saddle_id.to_string(), out stored)
  237. && (get_monotonic_time() / 1000 - int64.parse(stored)) < guard_ms;
  238. state_lock.unlock();
  239. return pending;
  240. }
  241. /** The task id `saddle_id` was last woken for (0 when none). */
  242. public int64 wake_task_of(int64 saddle_id) {
  243. state_lock.lock();
  244. string stored = "";
  245. bool found = wake_task.try_get(saddle_id.to_string(), out stored);
  246. state_lock.unlock();
  247. return found ? int64.parse(stored) : 0;
  248. }
  249. /**
  250. * Records a chunks/results push arriving for `session_id`
  251. * (plan 10 phase 2): the running daemon is alive, and any
  252. * running staleness episode ends — a later silent episode
  253. * earns its own exceptional probe.
  254. */
  255. public void note_push(int64 session_id) {
  256. state_lock.lock();
  257. string key = session_id.to_string();
  258. push_at_ms.set(key, (get_monotonic_time() / 1000).to_string());
  259. push_seen.add(key);
  260. stale_probed.remove(key);
  261. state_lock.unlock();
  262. }
  263. /**
  264. * Atomically decides the one exceptional staleness probe for
  265. * `session_id`: true exactly when pushes have been seen, the
  266. * silence has reached `stale_ms` and no probe fired for this
  267. * episode yet (the flag flips here, so a slow probe is never
  268. * double-fired by the next tick). A session with no recorded
  269. * push (a web restart wiped the clock) is seeded now and
  270. * starts its window instead of firing immediately — a fresh
  271. * baseline must not masquerade as 90 s of silence.
  272. */
  273. public bool take_stale_probe(int64 session_id, uint stale_ms) {
  274. state_lock.lock();
  275. string key = session_id.to_string();
  276. int64 now_ms = get_monotonic_time() / 1000;
  277. if (!push_seen.has(key)) {
  278. push_seen.add(key);
  279. push_at_ms.set(key, now_ms.to_string());
  280. state_lock.unlock();
  281. return false;
  282. }
  283. if (stale_probed.has(key)) {
  284. state_lock.unlock();
  285. return false;
  286. }
  287. string stored = "";
  288. bool known = push_at_ms.try_get(key, out stored);
  289. bool due = known && (now_ms - int64.parse(stored)) >= stale_ms;
  290. if (due) {
  291. stale_probed.add(key);
  292. }
  293. state_lock.unlock();
  294. return due;
  295. }
  296. }
  297. }