SaddlePollState.vala 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309
  1. using Invercargill.DataStructures;
  2. using Inversion;
  3. namespace ClankerSaddle.Services {
  4. /**
  5. * Process-wide saddle poll bookkeeping: the last daemon-side output
  6. * entry id stored per session (so redelivered results pushes never
  7. * duplicate `session_commands` rows), the block answers and human
  8. * comments already handed to a saddle (each delivered exactly once;
  9. * a restart re-delivers and the daemon deduplicates), plus the
  10. * plan-10 ask/wake liveness bookkeeping: the outcome of a saddle's
  11. * last `POST /api/saddle/next-task` ask (a saddle counts as SLEEPING
  12. * when its last ask returned 204) and the pending wake each sleeping
  13. * saddle was sent (the lost-wake guard's timer), and the phase-2
  14. * push-liveness clock: the last chunks/results arrival per session
  15. * (the staleness probe's timer — one exceptional probe per silent
  16. * episode, reset by the next push).
  17. *
  18. * A singleton by necessity: {@link SaddleService} is request-scoped,
  19. * so this object is where the cross-request cursors live. Ids are
  20. * stringified: the Invercargill collections cannot key on value types.
  21. */
  22. public class SaddlePollState : Object {
  23. private Dictionary<string, string> probe_state = new Dictionary<string, string>();
  24. private Dictionary<string, string> probe_uptime = new Dictionary<string, string>();
  25. private Dictionary<string, string> probe_activity = new Dictionary<string, string>();
  26. private Dictionary<string, string> probe_contact = new Dictionary<string, string>();
  27. private Dictionary<string, string> last_stored_output_id = new Dictionary<string, string>();
  28. private HashSet<string> delivered_answers = new HashSet<string>();
  29. private HashSet<string> delivered_comments = new HashSet<string>();
  30. private HashSet<string> resumed_notified = new HashSet<string>();
  31. private Dictionary<string, string> ask_outcome = new Dictionary<string, string>();
  32. private HashSet<string> ask_seen = new HashSet<string>();
  33. private Dictionary<string, string> wake_at_ms = new Dictionary<string, string>();
  34. private Dictionary<string, string> wake_task = new Dictionary<string, string>();
  35. private Dictionary<string, string> push_at_ms = new Dictionary<string, string>();
  36. private HashSet<string> push_seen = new HashSet<string>();
  37. private HashSet<string> stale_probed = new HashSet<string>();
  38. private Mutex state_lock = Mutex();
  39. /** The last output entry id stored for `session_id` (0 when none). */
  40. public int64 since_for(int64 session_id) {
  41. state_lock.lock();
  42. string stored = "";
  43. bool found = last_stored_output_id.try_get(session_id.to_string(), out stored);
  44. state_lock.unlock();
  45. return found ? int64.parse(stored) : 0;
  46. }
  47. /** Records the new high-water output entry id for `session_id`. */
  48. public void store_since(int64 session_id, int64 entry_id) {
  49. state_lock.lock();
  50. last_stored_output_id.set(session_id.to_string(), entry_id.to_string());
  51. state_lock.unlock();
  52. }
  53. /**
  54. * Records a daemon /status observation for the saddles-page
  55. * health panel: reported state, uptime, last-activity epoch and
  56. * the web-side contact time (monotonic ms).
  57. */
  58. public void record_probe(int64 saddle_id, string state, int64 uptime_seconds,
  59. int64 last_activity_epoch) {
  60. state_lock.lock();
  61. string key = saddle_id.to_string();
  62. probe_state.set(key, state);
  63. probe_uptime.set(key, uptime_seconds.to_string());
  64. probe_activity.set(key, last_activity_epoch.to_string());
  65. probe_contact.set(key, (get_monotonic_time() / 1000).to_string());
  66. state_lock.unlock();
  67. }
  68. /** The last probe snapshot for `saddle_id` ("" when never probed). */
  69. public void health_for(int64 saddle_id, out string state, out int64 uptime,
  70. out int64 activity, out int64 contact_ms) {
  71. state_lock.lock();
  72. string key = saddle_id.to_string();
  73. string stored = "";
  74. state = probe_state.try_get(key, out stored) ? stored : "";
  75. uptime = int64.parse(probe_uptime.get_or_default(key) ?? "0");
  76. activity = int64.parse(probe_activity.get_or_default(key) ?? "0");
  77. contact_ms = int64.parse(probe_contact.get_or_default(key) ?? "0");
  78. state_lock.unlock();
  79. }
  80. /**
  81. * Marks `session_id`'s daemon-side resume as notified; returns
  82. * false when the progress entry already went out.
  83. */
  84. public bool mark_resumed(int64 session_id) {
  85. state_lock.lock();
  86. bool already = resumed_notified.has(session_id.to_string());
  87. if (!already) {
  88. resumed_notified.add(session_id.to_string());
  89. }
  90. state_lock.unlock();
  91. return !already;
  92. }
  93. /**
  94. * Marks `block_id`'s answer as delivered; returns false when it
  95. * had already been handed over.
  96. */
  97. public bool mark_delivered(int64 block_id) {
  98. state_lock.lock();
  99. bool already = delivered_answers.has(block_id.to_string());
  100. if (!already) {
  101. delivered_answers.add(block_id.to_string());
  102. }
  103. state_lock.unlock();
  104. return !already;
  105. }
  106. /** True when `block_id`'s answer was already handed over. */
  107. public bool is_delivered(int64 block_id) {
  108. state_lock.lock();
  109. bool already = delivered_answers.has(block_id.to_string());
  110. state_lock.unlock();
  111. return already;
  112. }
  113. /**
  114. * Marks `comment_id` as delivered to its saddle; returns false
  115. * when it had already been handed over.
  116. */
  117. public bool mark_comment_delivered(int64 comment_id) {
  118. state_lock.lock();
  119. bool already = delivered_comments.has(comment_id.to_string());
  120. if (!already) {
  121. delivered_comments.add(comment_id.to_string());
  122. }
  123. state_lock.unlock();
  124. return !already;
  125. }
  126. /** True when `comment_id` was already handed over. */
  127. public bool is_comment_delivered(int64 comment_id) {
  128. state_lock.lock();
  129. bool already = delivered_comments.has(comment_id.to_string());
  130. state_lock.unlock();
  131. return already;
  132. }
  133. /**
  134. * Records an ask arriving from `saddle_id` (plan 10): the
  135. * pending wake is answered — the lost-wake guard must not fire
  136. * for it — and the saddle is awake until the ask resolves. The
  137. * first ask also marks the daemon ask-capable (post-flip): its
  138. * parked-task resumes flow through wake + next-task from then
  139. * on, never through /start pushes.
  140. */
  141. public void mark_ask(int64 saddle_id) {
  142. state_lock.lock();
  143. string key = saddle_id.to_string();
  144. ask_outcome.set(key, "awake");
  145. ask_seen.add(key);
  146. wake_at_ms.remove(key);
  147. wake_task.remove(key);
  148. state_lock.unlock();
  149. }
  150. /**
  151. * Records the outcome of `saddle_id`'s ask: `204` puts it to
  152. * sleep (a later wake targets it), anything else leaves it
  153. * awake.
  154. */
  155. public void mark_ask_result(int64 saddle_id, bool assigned) {
  156. state_lock.lock();
  157. ask_outcome.set(saddle_id.to_string(), assigned ? "assigned" : "204");
  158. state_lock.unlock();
  159. }
  160. /**
  161. * True when `saddle_id`'s last ask returned 204 — the sleeping
  162. * state a wake targets.
  163. */
  164. public bool is_sleeping(int64 saddle_id) {
  165. state_lock.lock();
  166. string stored = "";
  167. bool sleeping = ask_outcome.try_get(saddle_id.to_string(), out stored)
  168. && stored == "204";
  169. state_lock.unlock();
  170. return sleeping;
  171. }
  172. /**
  173. * True once `saddle_id` has ever asked (post-flip daemon): it
  174. * asks whenever idle-awake, so its work — fresh or resumed —
  175. * always flows through the allocation.
  176. */
  177. public bool has_ever_asked(int64 saddle_id) {
  178. state_lock.lock();
  179. bool seen = ask_seen.has(saddle_id.to_string());
  180. state_lock.unlock();
  181. return seen;
  182. }
  183. /** Forgets the ask state (the saddle went offline / was removed). */
  184. public void clear_ask(int64 saddle_id) {
  185. state_lock.lock();
  186. string key = saddle_id.to_string();
  187. ask_outcome.remove(key);
  188. wake_at_ms.remove(key);
  189. wake_task.remove(key);
  190. state_lock.unlock();
  191. }
  192. /**
  193. * Records a wake sent to `saddle_id` for `task_id` — starts the
  194. * lost-wake guard's timer.
  195. */
  196. public void note_wake(int64 saddle_id, int64 task_id) {
  197. state_lock.lock();
  198. string key = saddle_id.to_string();
  199. wake_at_ms.set(key, (get_monotonic_time() / 1000).to_string());
  200. wake_task.set(key, task_id.to_string());
  201. state_lock.unlock();
  202. }
  203. /**
  204. * True when `saddle_id` has a pending wake at least
  205. * `guard_ms` old — an ask should have arrived by now; the
  206. * guard declares the saddle offline and frees the work for
  207. * the next capable one.
  208. */
  209. public bool wake_overdue(int64 saddle_id, uint guard_ms) {
  210. state_lock.lock();
  211. string stored = "";
  212. bool overdue = wake_at_ms.try_get(saddle_id.to_string(), out stored)
  213. && (get_monotonic_time() / 1000 - int64.parse(stored)) >= guard_ms;
  214. state_lock.unlock();
  215. return overdue;
  216. }
  217. /**
  218. * True when `saddle_id` carries a pending wake younger than
  219. * `guard_ms` — the wake pass skips it instead of re-waking on
  220. * every tick while the guard window runs.
  221. */
  222. public bool wake_pending(int64 saddle_id, uint guard_ms) {
  223. state_lock.lock();
  224. string stored = "";
  225. bool pending = wake_at_ms.try_get(saddle_id.to_string(), out stored)
  226. && (get_monotonic_time() / 1000 - int64.parse(stored)) < guard_ms;
  227. state_lock.unlock();
  228. return pending;
  229. }
  230. /** The task id `saddle_id` was last woken for (0 when none). */
  231. public int64 wake_task_of(int64 saddle_id) {
  232. state_lock.lock();
  233. string stored = "";
  234. bool found = wake_task.try_get(saddle_id.to_string(), out stored);
  235. state_lock.unlock();
  236. return found ? int64.parse(stored) : 0;
  237. }
  238. /**
  239. * Records a chunks/results push arriving for `session_id`
  240. * (plan 10 phase 2): the running daemon is alive, and any
  241. * running staleness episode ends — a later silent episode
  242. * earns its own exceptional probe.
  243. */
  244. public void note_push(int64 session_id) {
  245. state_lock.lock();
  246. string key = session_id.to_string();
  247. push_at_ms.set(key, (get_monotonic_time() / 1000).to_string());
  248. push_seen.add(key);
  249. stale_probed.remove(key);
  250. state_lock.unlock();
  251. }
  252. /**
  253. * Atomically decides the one exceptional staleness probe for
  254. * `session_id`: true exactly when pushes have been seen, the
  255. * silence has reached `stale_ms` and no probe fired for this
  256. * episode yet (the flag flips here, so a slow probe is never
  257. * double-fired by the next tick). A session with no recorded
  258. * push (a web restart wiped the clock) is seeded now and
  259. * starts its window instead of firing immediately — a fresh
  260. * baseline must not masquerade as 90 s of silence.
  261. */
  262. public bool take_stale_probe(int64 session_id, uint stale_ms) {
  263. state_lock.lock();
  264. string key = session_id.to_string();
  265. int64 now_ms = get_monotonic_time() / 1000;
  266. if (!push_seen.has(key)) {
  267. push_seen.add(key);
  268. push_at_ms.set(key, now_ms.to_string());
  269. state_lock.unlock();
  270. return false;
  271. }
  272. if (stale_probed.has(key)) {
  273. state_lock.unlock();
  274. return false;
  275. }
  276. string stored = "";
  277. bool known = push_at_ms.try_get(key, out stored);
  278. bool due = known && (now_ms - int64.parse(stored)) >= stale_ms;
  279. if (due) {
  280. stale_probed.add(key);
  281. }
  282. state_lock.unlock();
  283. return due;
  284. }
  285. }
  286. }