Merge branch 'task/2-13' into goal/selfhosted-qol

README.md

@@ -289,14 +289,26 @@ file is the single home for the keys; delete the section to rotate
 To enable notifications in a browser: open the dashboard and press
 **Enable notifications** — the page requests permission, registers the
 service worker (`GET /sw.js`, scope `/`) and subscribes with the app's
-VAPID key. "Turn off notifications" reverses it. Per task, **Notify
-me** / **Don't notify me** (on the task page's status card) opts you
-into that task's first start, failure and completion; task questions,
-RFCs and review requests notify every subscribed user, and goal blocks
-plus goal success/failure ALWAYS notify every subscribed user. Clicking
-a notification opens the task's or goal's page.
-Delivery is best-effort: failed sends retry on the dispatcher tick and
-endpoints answering 410/404 are pruned automatically.
+VAPID key. "Turn off notifications" reverses it. The bell is
+**per-browser**: it compares THIS browser's subscription (by a
+SHA-256 fingerprint of its endpoint — the endpoint URL itself never
+leaves the server) against your subscribed browsers, so each browser
+shows its own state; when you are subscribed elsewhere it says so and
+still offers Enable here. **Send test notification** (visible when
+this browser is subscribed) pushes a real notification through the
+full protocol and reports the true per-endpoint result, so a broken
+setup surfaces its actual reason instead of silence. Per task,
+**Notify me** / **Don't notify me** (on the task page's status card)
+opts you into that task's first start, failure and completion; task
+questions, RFCs and review requests notify every subscribed user, and
+goal blocks plus goal success/failure ALWAYS notify every subscribed
+user. Clicking a notification opens the task's or goal's page.
+Delivery is best-effort and truthful: failed sends retry on the
+dispatcher tick (up to five attempts — after which the drop is logged
+loudly, never silently claimed as delivered), endpoints answering
+410/404 are pruned automatically, and other 4xx refusals (e.g. a VAPID
+rejection) keep the subscription row and retry, since the endpoint
+may recover.
 
 ## Rules and project notes
 

src/Services/PushService.vala

@@ -46,6 +46,21 @@ namespace ClankerSaddle.Services {
         public string url { get; set; default = ""; }
     }
 
+    /**
+     * The roll-up of one {@link PushService.send_test} probe: counts
+     * per outcome plus the first failure's detail for the toast.
+     * `first_failure` names the HTTP status ("HTTP 401") or the
+     * transport error text — the dashboard's test button toasts the
+     * REAL end-to-end result, never a blind success.
+     */
+    public class PushTestReport : Object {
+        public int delivered { get; set; }
+        public int failed { get; set; }
+        /** Endpoints that answered 404/410 — pruned during the probe. */
+        public int pruned { get; set; }
+        public string first_failure { get; set; default = ""; }
+    }
+
     /**
      * Web Push delivery (plan 11 task E): real Web Push protocol over
      * VAPID. The keypair is generated once and persisted in the config
@@ -92,6 +107,8 @@ namespace ClankerSaddle.Services {
             public string auth = "";
             public string payload = "";
             public int attempts = 0;
+            /** The last HTTP status the endpoint answered (0 = transport error). */
+            public int last_status = 0;
         }
 
         /** A queued notification before target resolution. */
@@ -301,6 +318,21 @@ namespace ClankerSaddle.Services {
             };
         }
 
+        /**
+         * The "Send test notification" payload (per-browser bell): a
+         * REAL Web Push through the full RFC 8291 + VAPID path whose
+         * arrival the human can see — the end-to-end probe that was
+         * missing entirely.
+         */
+        public static PushNotification test_notification() {
+            return new PushNotification() {
+                title = "Test notification",
+                body = "A live Web Push from Clanker Saddle — if you can"
+                    + " read this, delivery works.",
+                url = "/"
+            };
+        }
+
         // ------------------------------------------------------------------
         // Enqueue + delivery
         // ------------------------------------------------------------------
@@ -358,7 +390,15 @@ namespace ClankerSaddle.Services {
                     if (job == null) {
                         break;
                     }
-                    yield deliver((!)job);
+                    bool requeued = yield deliver((!)job);
+                    if (requeued) {
+                        // A failed send goes back for a LATER pass:
+                        // draining in place would burn every attempt in
+                        // one tight loop. The dispatcher tick (or the
+                        // next enqueue's flush) picks the job back up —
+                        // that spacing IS the retry backoff.
+                        break;
+                    }
                 }
             } finally {
                 flushing = false;
@@ -432,7 +472,15 @@ namespace ClankerSaddle.Services {
             GONE
         }
 
-        private async void deliver(SendJob job) {
+        /**
+         * Applies one job's outcome: DELIVERED drops it, GONE prunes
+         * the subscription row, anything else requeues it for a later
+         * tick. Returns true when the job was requeued — the flush
+         * drain then STOPS (the retry waits for the next tick instead
+         * of burning every attempt back-to-back), which is what makes
+         * MAX_ATTEMPTS a real backoff rather than one hot loop.
+         */
+        private async bool deliver(SendJob job) {
             SendOutcome outcome = SendOutcome.RETRY;
             try {
                 outcome = yield deliver_once(job);
@@ -440,29 +488,87 @@ namespace ClankerSaddle.Services {
                 outcome = SendOutcome.RETRY;
             }
             if (outcome == SendOutcome.DELIVERED) {
-                return;
+                return false;
             }
             if (outcome == SendOutcome.GONE) {
                 yield prune_subscription(job.endpoint);
-                return;
+                return false;
             }
             job.attempts++;
             if (job.attempts >= MAX_ATTEMPTS) {
-                warning("[ClankerSaddle] Push to %s gave up after %d attempts",
+                // Giving up is loud on purpose: a silently dropped
+                // notification looks exactly like a delivered one.
+                warning("[ClankerSaddle] Push to %s gave up after %d attempts"
+                    + " — notification dropped undelivered",
                     job.endpoint, job.attempts);
-                return;
+                return false;
             }
             queue_lock.lock();
             queued.push_tail(job);
             queue_lock.unlock();
+            return true;
+        }
+
+        /**
+         * The dashboard's "Send test notification" probe (per-browser
+         * bell, shown only when THIS browser is subscribed): one REAL
+         * delivery attempt to each of the user's endpoints through the
+         * same {@link deliver_once} crypto + VAPID path the queue
+         * uses, awaited inline so the action can report per-endpoint
+         * outcomes. The shared queue's flush is single-flight and
+         * fire-and-forget — its outcomes are unobservable from a
+         * request — so the probe resolves and sends synchronously
+         * instead of enqueueing. A refused endpoint is reported rather
+         * than retried here (the queue's own backoff still governs
+         * real events); 404/410 endpoints are pruned exactly as the
+         * queue would, so the probe also heals dead rows on demand.
+         */
+        public async PushTestReport send_test(OrmSession db, int64 user_id) throws Error {
+            var report = new PushTestReport();
+            var jobs = new GLib.List<SendJob>();
+            foreach (var row in yield db.query<PushSubscription>()
+                    .where(expr("user_id == $0", elem<int64?>(user_id)))
+                    .materialise_async()) {
+                jobs.append(new SendJob() {
+                    endpoint = row.endpoint,
+                    p256dh = row.p256dh,
+                    auth = row.auth,
+                    payload = payload_json(test_notification())
+                });
+            }
+            foreach (var job in jobs) {
+                SendOutcome outcome = SendOutcome.RETRY;
+                string problem = "";
+                try {
+                    outcome = yield deliver_once(job);
+                } catch (Error e) {
+                    problem = e.message;
+                }
+                if (outcome == SendOutcome.DELIVERED) {
+                    report.delivered++;
+                    continue;
+                }
+                if (outcome == SendOutcome.GONE) {
+                    report.pruned++;
+                    yield prune_subscription(job.endpoint);
+                    continue;
+                }
+                report.failed++;
+                if (report.first_failure.length == 0) {
+                    report.first_failure = job.last_status > 0
+                        ? @"HTTP $(job.last_status)" : problem;
+                }
+            }
+            return report;
         }
 
         /**
          * One Web Push protocol POST: RFC 8291 aes128gcm body, ES256
          * VAPID JWT in the Authorization header. Transport errors and
          * 5xx throw (retry); 404/410 report {@link SendOutcome.GONE}
-         * (prune); any other status counts as delivered (the push
-         * service owns delivery from here).
+         * (prune); any OTHER 4xx is a refusal — logged and retried,
+         * never silently delivered; remaining 2xx/3xx counts as
+         * delivered (the push service owns delivery from here).
          */
         private async SendOutcome deliver_once(SendJob job) throws Error {
             uint8[] client_public = base64url_decode(job.p256dh);
@@ -558,9 +664,25 @@ namespace ClankerSaddle.Services {
                 return SendOutcome.GONE;
             }
             if (message.status_code >= 500) {
+                job.last_status = (int) message.status_code;
                 throw new Error(Quark.from_string("clanker-saddle"), 65,
                     @"push service answered HTTP $(message.status_code)");
             }
+            if (message.status_code >= 400) {
+                // 401/403 (VAPID rejected) and every other 4xx that is
+                // not the terminal 404/410 handled above: the push
+                // service did NOT accept the message. Truthful
+                // delivery — log it with the endpoint and status,
+                // retry on later ticks, give up loudly after
+                // MAX_ATTEMPTS, and never count it as delivered. The
+                // subscription row stays: the endpoint may recover
+                // (e.g. a VAPID key rotation heals on a later send).
+                job.last_status = (int) message.status_code;
+                warning("[ClankerSaddle] Push endpoint %s refused delivery:"
+                    + " HTTP %d — not delivered, retrying",
+                    job.endpoint, (int) message.status_code);
+                return SendOutcome.RETRY;
+            }
             return SendOutcome.DELIVERED;
         }
 
@@ -684,7 +806,12 @@ namespace ClankerSaddle.Services {
             }
         }
 
-        /** True when the user holds any subscription (the bell's state). */
+        /**
+         * True when the user holds any subscription. Convenience only
+         * since the per-browser bell: the authoritative per-browser
+         * state is {@link user_endpoint_fingerprints} matched
+         * client-side against the browser's own subscription.
+         */
         public async bool user_subscribed(OrmSession db, int64 user_id) throws Error {
             foreach (var row in yield db.query<PushSubscription>()
                     .where(expr("user_id == $0", elem<int64?>(user_id)))
@@ -694,6 +821,47 @@ namespace ClankerSaddle.Services {
             return false;
         }
 
+        /**
+         * This user's subscription endpoint fingerprints, newest
+         * first (the per-browser bell state): each entry is
+         * {@link endpoint_fingerprint} of one stored endpoint — never
+         * the endpoint itself, which stays server-private. The browser
+         * fingerprints its own pushManager subscription and looks it
+         * up in the list: present => THIS browser is subscribed (show
+         * "Turn off"), absent => "Enable" (+ a count of the others).
+         */
+        public async string[] user_endpoint_fingerprints(OrmSession db, int64 user_id) throws Error {
+            var fingerprints = new string[0];
+            foreach (var row in yield db.query<PushSubscription>()
+                    .where(expr("user_id == $0", elem<int64?>(user_id)))
+                    .order_by_desc(expr("created_at"))
+                    .order_by_desc(expr("id"))
+                    .materialise_async()) {
+                fingerprints += endpoint_fingerprint(row.endpoint);
+            }
+            return fingerprints;
+        }
+
+        /**
+         * A privacy-safe handle for one endpoint: the first 16 hex
+         * chars of its SHA-256 — enough for a browser to recognise its
+         * own subscription, useless for reconstructing the URL. The
+         * client mirrors this with crypto.subtle (both need a secure
+         * context, so any browser that can subscribe can compare).
+         */
+        public static string endpoint_fingerprint(string endpoint) {
+            var checksum = new Checksum(ChecksumType.SHA256);
+            size_t digest_len = 32;
+            var digest = new uint8[32];
+            checksum.update(endpoint.data, endpoint.length);
+            checksum.get_digest(digest, ref digest_len);
+            var builder = new StringBuilder();
+            for (var index = 0; index < 8; index++) {
+                builder.append_printf("%02x", digest[index]);
+            }
+            return builder.str;
+        }
+
         /** True when the user holds a "Notify me" pref for the task. */
         public async bool has_notify_pref(OrmSession db, int64 user_id,
                 int64 task_id) throws Error {

src/actions/PushActions.vala

@@ -83,6 +83,71 @@ namespace ClankerSaddle {
      * and completion on top of the global block events every
      * push-subscribed user gets.
      */
+    /**
+     * The bell's "Send test notification" probe (per-browser bell):
+     * queues a REAL Web Push — full RFC 8291 + VAPID path — targeted at
+     * the signed-in user's own subscriptions ({@link
+     * PushService.send_test}), then reports the per-endpoint outcome
+     * through the notify so the UI toasts the truth. This is the
+     * missing end-to-end check: a green light here proves keys,
+     * crypto, the push service and the browser's subscription all
+     * line up, and a failure surfaces its actual reason instead of
+     * silence. Also refreshes the dashboard slot so the bell's
+     * endpoint fingerprints (and a pruned dead endpoint) are current.
+     */
+    public class PushTestAction : StatumAction {
+
+        private TaskService task_service = inject<TaskService>();
+        private SaddleService saddle_service = inject<SaddleService>();
+        private TimelineService timeline_service = inject<TimelineService>();
+        private InvercargillSql.Orm.OrmSession db = inject<InvercargillSql.Orm.OrmSession>();
+        private PushService push = inject<PushService>();
+        private UserService users = inject<UserService>();
+
+        public override async DirectiveBuilder handle() throws GLib.Error {
+            var guard = yield SpryAuth.require_login(directives(), request, users);
+            if (guard != null) {
+                return (!)guard;
+            }
+
+            var me = SpryAuth.current(request);
+            try {
+                var report = yield push.send_test(db, me.user_id);
+                string message;
+                string kind;
+                if (report.failed == 0 && report.pruned == 0) {
+                    kind = "success";
+                    message = report.delivered > 1
+                        ? @"Test notification sent to $(report.delivered) browsers"
+                        : "Test notification sent";
+                } else if (report.delivered > 0) {
+                    kind = "error";
+                    message = @"Test notification reached $(report.delivered) browser(s)"
+                        + @" but $(report.failed) failed";
+                    if (report.first_failure.length > 0) {
+                        message += @" ($(report.first_failure))";
+                    }
+                } else {
+                    kind = "error";
+                    message = "Test notification failed";
+                    if (report.first_failure.length > 0) {
+                        message += @": $(report.first_failure)";
+                    }
+                    if (report.pruned > 0) {
+                        message += @" — $(report.pruned) dead endpoint(s) removed";
+                    }
+                }
+                var state = yield DashboardState.build(state_service, task_service,
+                    saddle_service, timeline_service, db, action_registry,
+                    push, me.user_id);
+                return directives().notify(kind, message).set(state);
+            } catch (Error e) {
+                return directives().notify("error",
+                    @"Could not send the test notification: $(e.message)");
+            }
+        }
+    }
+
     public class ToggleTaskNotifyAction : SealedTaskAction {
 
         private InvercargillSql.Orm.OrmSession db = Inversion.inject<InvercargillSql.Orm.OrmSession>();

src/entrypoints/HomeEntrypoint.vala

@@ -97,6 +97,8 @@ namespace ClankerSaddle {
         public bool push_subscribed { get; set; }
         public Statum.Model.ActionDto? push_subscribe { get; set; }
         public Statum.Model.ActionDto? push_unsubscribe { get; set; }
+        /** Queues a REAL test push to this user (the bell's probe). */
+        public Statum.Model.ActionDto? push_test { get; set; }
     }
 
     /**
@@ -266,12 +268,24 @@ namespace ClankerSaddle {
                 pub.refresh = ((!)actions).author<RefreshDashboardAction>("GET");
                 pub.push_subscribe = ((!)actions).author<PushSubscribeAction>();
                 pub.push_unsubscribe = ((!)actions).author<PushUnsubscribeAction>();
+                pub.push_test = ((!)actions).author<PushTestAction>();
             }
-            // Web Push bell state (plan 11 task E): the VAPID key the
-            // browser needs + whether this user already subscribed.
+            // Web Push bell state (plan 11 task E; per-browser since
+            // the user-global flag lied to second browsers): the VAPID
+            // key the browser needs, whether this user subscribed
+            // anywhere (convenience), and this user's endpoint
+            // FINGERPRINTS — the client compares its own subscription's
+            // fingerprint against the list, so each browser shows its
+            // own truth.
             pub.push_vapid_key = PushService.vapid_public_key;
+            // Not a DashboardPublic property: GObjectMapping cannot
+            // serialise array-shaped values, so the fingerprint list
+            // rides the props dictionary like every other list here.
+            string[] push_fingerprints = new string[0];
             if (push != null && viewer_id > 0) {
                 pub.push_subscribed = yield ((!)push).user_subscribed(db, viewer_id);
+                push_fingerprints = yield ((!)push)
+                    .user_endpoint_fingerprints(db, viewer_id);
             }
 
             var activity = new Vector<DashboardActivityView>();
@@ -294,6 +308,7 @@ namespace ClankerSaddle {
 
             var props = (PropertyDictionary) GObjectMapping.to_properties(pub);
             props["saddles"] = StateJson.list<DashboardSaddleView>(saddle_rows.to_immutable_buffer());
+            props["push_endpoints"] = StateJson.strings(push_fingerprints);
             props["activity"] = StateJson.list<DashboardActivityView>(activity.to_immutable_buffer());
             props["active_tasks"] = StateJson.list<DashboardActiveView>(active.to_immutable_buffer());
             props["pinned"] = StateJson.list<DashboardBlockView>(pinned_sorted.to_immutable_buffer());

src/main.vala

@@ -139,6 +139,7 @@ int main(string[] args) {
         statum.action<RefreshGoalDataAction>();
         statum.action<PushSubscribeAction>();
         statum.action<PushUnsubscribeAction>();
+        statum.action<PushTestAction>();
         statum.action<ToggleTaskNotifyAction>();
         statum.action<RefreshTaskDataAction>();
         statum.action<RefreshDashboardAction>();

src/pages/home.html

@@ -108,13 +108,121 @@
                 }).catch(function (error) { fail(error); });
             }
 
+            // ---- per-browser bell state (the user-global flag lied) ----
+            // The dashboard exposes the user's endpoint FINGERPRINTS;
+            // this browser's own pushManager subscription is hashed
+            // the same way and looked up in the list. Each browser
+            // then shows its own truth: subscribed here ("Turn off")
+            // vs subscribed elsewhere only ("Enable" + a count). The
+            // client-side check is authoritative — the server flag is
+            // just a convenience for other consumers.
+            var browserSubscribed = null; // null = still asking the browser
+            var hereNow = false;          // the displayed verdict (click acts on it)
+
+            function sha256Hex16(text) {
+                // Mirrors PushService.endpoint_fingerprint: first 8
+                // bytes of the SHA-256, hex — crypto.subtle needs a
+                // secure context, exactly like pushManager itself.
+                var data = new TextEncoder().encode(text);
+                return crypto.subtle.digest('SHA-256', data).then(function (digest) {
+                    var bytes = new Uint8Array(digest).slice(0, 8);
+                    var hex = '';
+                    for (var index = 0; index < bytes.length; index++) {
+                        hex += ('0' + bytes[index].toString(16)).slice(-2);
+                    }
+                    return hex;
+                });
+            }
+
+            function myFingerprint() {
+                if (!('serviceWorker' in navigator) || !('PushManager' in window)) {
+                    return Promise.resolve(null);
+                }
+                return navigator.serviceWorker.getRegistration('/').then(function (registration) {
+                    if (!registration) return null;
+                    return registration.pushManager.getSubscription().then(function (sub) {
+                        return sub ? sha256Hex16(sub.endpoint) : null;
+                    });
+                }).catch(function () { return null; });
+            }
+
+            function otherBrowserCount() {
+                // The dashboard's fingerprints minus this browser's own.
+                var mine = browserSubscribed;
+                var list = dashboard().push_endpoints || [];
+                if (mine === null || mine === undefined) return list.length;
+                var others = 0;
+                for (var index = 0; index < list.length; index++) {
+                    if (list[index] !== mine) others++;
+                }
+                return others;
+            }
+
+            function refreshBell() {
+                return myFingerprint().then(function (mine) {
+                    browserSubscribed = mine;
+                    var list = dashboard().push_endpoints || [];
+                    var card = document.querySelector('.push-bell-card');
+                    if (!card) return;
+                    var here = mine !== null && list.indexOf(mine) !== -1;
+                    var others = otherBrowserCount();
+                    var onBtn = card.querySelector('[data-bell-on]');
+                    var offBtn = card.querySelector('[data-bell-off]');
+                    var help = card.querySelector('[data-bell-help]');
+                    var testBtn = card.querySelector('[data-push-test]');
+                    hereNow = here;
+                    if (onBtn) onBtn.hidden = here;
+                    if (offBtn) offBtn.hidden = !here;
+                    if (testBtn) testBtn.hidden = !here;
+                    if (help) {
+                        if (here) {
+                            help.textContent = others > 0
+                                ? '— this browser receives question, RFC, review and waiting-on-drafts alerts; '
+                                  + others + ' other browser(s) are also subscribed; "Notify me" per task adds its start, failure and completion'
+                                : '— this browser receives question, RFC, review and waiting-on-drafts alerts; "Notify me" per task adds its start, failure and completion';
+                        } else if (list.length > 0) {
+                            help.textContent = '— you are subscribed on ' + list.length
+                                + ' other browser(s); enable here too to be notified in this browser';
+                        } else {
+                            help.textContent = '— be told when an agent asks a question, raises an RFC or review';
+                        }
+                    }
+                });
+            }
+
+            function sendTest() {
+                var state = dashboard();
+                if (!state.push_test) {
+                    fail('not available on this server');
+                    return;
+                }
+                statum.act(state.push_test, {}).catch(function (error) { fail(error); });
+            }
+
+            // The dashboard state refreshes on every pulse; the bell's
+            // per-browser verdict must be re-derived each time (the
+            // browser's own subscription may have changed under us).
+            statum.addStateListener('dashboard', refreshBell);
+            refreshBell();
+
             document.addEventListener('click', function (event) {
                 if (!event.target.closest) return;
+                var card = event.target.closest('.push-bell-card');
+                if (!card) return;
                 var bell = event.target.closest('[data-push-bell]');
-                if (!bell) return;
-                event.preventDefault();
-                if (dashboard().push_subscribed) disablePush();
-                else enablePush();
+                var test = event.target.closest('[data-push-test]');
+                if (bell) {
+                    event.preventDefault();
+                    // Follow the DISPLAYED verdict: a browser holding a
+                    // subscription the server no longer knows (a pruned
+                    // row) shows Enable — and enabling re-registers the
+                    // existing subscription, healing it.
+                    if (hereNow) disablePush();
+                    else enablePush();
+                } else if (test) {
+                    event.preventDefault();
+                    sendTest();
+                }
             });
         })();
     }
@@ -130,17 +238,24 @@
                  browser (permission + service worker + VAPID key) or
                  turn it off. Block events notify every subscribed user;
                  per-task "Notify me" adds start/failure/completion. -->
+            <!-- The notification bell (plan 11 task E; per-browser
+                 since the user-global flag lied to second browsers):
+                 THIS browser's own subscription decides Enable vs Turn
+                 off (the JS fingerprints the browser's endpoint and
+                 matches it against the dashboard's list), the helper
+                 text counts the user's OTHER subscribed browsers, and
+                 "Send test notification" fires the real end-to-end
+                 probe through this user's subscriptions. -->
             <section class="card push-bell-card">
                 <form class="inline">
-                    <button type="button" data-push-bell class="primary"
-                            stm-if="!dashboard.push_subscribed">Enable notifications</button>
-                    <button type="button" data-push-bell
-                            stm-if="dashboard.push_subscribed">Turn off notifications</button>
-                    <span class="muted" stm-if="!dashboard.push_subscribed">— be told when an agent asks a
+                    <button type="button" data-push-bell data-bell-on class="primary"
+                            hidden>Enable notifications</button>
+                    <button type="button" data-push-bell data-bell-off
+                            hidden>Turn off notifications</button>
+                    <button type="button" data-push-test
+                            hidden>Send test notification</button>
+                    <span class="muted" data-bell-help>— be told when an agent asks a
                         question, raises an RFC or review</span>
-                    <span class="muted" stm-if="dashboard.push_subscribed">— this browser receives question,
-                        RFC, review and waiting-on-drafts alerts; turn on "Notify me" per task for its
-                        start, failure and completion</span>
                 </form>
             </section>
 

test/bundles/push.py

@@ -376,3 +376,147 @@ def run(ctx):
     act(auth, dash["push_unsubscribe"], {"endpoint": endpoint_main})
     check(subscription_count(endpoint_main) == 0,
           "unsubscribing removes the endpoint's row")
+
+    # ------------------------------------------------------------------
+    # Per-browser bell + truthful delivery + test notification
+    # (the notifications-QoL task): the user-global flag lied to a
+    # second browser, refused deliveries were silently "delivered",
+    # and there was no end-to-end probe.
+    # ------------------------------------------------------------------
+
+    def fingerprints_of(auth_handle):
+        state = slot_public(auth_handle, "/", "dashboard")
+        return state.get("push_endpoints") or []
+
+    def fingerprint_of(endpoint):
+        return hashlib.sha256(endpoint.encode()).hexdigest()[:16]
+
+    print("== per-browser bell: fingerprint state shape ==")
+    # The retry fixture left admin subscribed to /flaky; drop it so
+    # this section owns exactly the subscriptions it asserts on.
+    act(auth, dash["push_unsubscribe"], {"endpoint": endpoint_flaky})
+    # Fresh admin subscriptions on TWO fake endpoints = two browsers.
+    push2 = FakePushService()
+    push2.serve()
+    endpoint_a = push2.url("/a")
+    endpoint_b = push2.url("/b")
+    browser_a = ec.generate_private_key(ec.SECP256R1())
+    browser_a_public = browser_a.public_key().public_bytes(
+        Encoding.X962, PublicFormat.UncompressedPoint)
+    browser_a_auth = os.urandom(16)
+    act(auth, dash["push_subscribe"],
+        {"endpoint": endpoint_a, "p256dh": b64url(browser_a_public),
+         "auth": b64url(browser_a_auth)})
+    browser_b = ec.generate_private_key(ec.SECP256R1())
+    browser_b_public = browser_b.public_key().public_bytes(
+        Encoding.X962, PublicFormat.UncompressedPoint)
+    browser_b_auth = os.urandom(16)
+    act(auth, dash["push_subscribe"],
+        {"endpoint": endpoint_b, "p256dh": b64url(browser_b_public),
+         "auth": b64url(browser_b_auth)})
+    dash = slot_public(auth, "/", "dashboard")
+    prints = dash.get("push_endpoints") or []
+    check(isinstance(prints, list) and len(prints) == 2,
+          "two subscriptions => both listed for the user (%s)" % prints)
+    check(fingerprint_of(endpoint_a) in prints
+          and fingerprint_of(endpoint_b) in prints,
+          "each entry is the sha256/16-hex fingerprint of its endpoint")
+    check(endpoint_a not in json.dumps(dash),
+          "the raw endpoint URL never leaves the server")
+    check(bool(dash.get("push_test")) and bool(dash.get("push_subscribe")),
+          "the dashboard carries the test + subscribe actions")
+
+    print("== per-browser bell: second browser still offered Enable ==")
+    # The old user-global flag showed "Turn off" in EVERY browser once
+    # any subscription existed; the per-browser client check keys on
+    # the fingerprint list instead. A third endpoint the server does
+    # NOT know = an unsubscribed browser of the same user: the state
+    # that drives the client verdict must not contain its fingerprint.
+    endpoint_c = push2.url("/c")
+    check(fingerprint_of(endpoint_c) not in fingerprints_of(auth),
+          "an unknown browser's fingerprint is absent — the bell shows Enable there")
+    # ...while the subscribed browsers still see themselves present.
+    check(fingerprint_of(endpoint_a) in fingerprints_of(auth)
+          and fingerprint_of(endpoint_b) in fingerprints_of(auth),
+          "the two subscribed browsers each see themselves in the list")
+
+    print("== test notification: real end-to-end round-trip ==")
+    a_before = len(push2.hits("/a"))
+    b_before = len(push2.hits("/b"))
+    result = act(auth, dash["push_test"], {})
+    message = action_notify(result) or ""
+    check(wait_for(lambda: len(push2.hits("/a")) > a_before
+                   and len(push2.hits("/b")) > b_before, timeout=15,
+                   what="test notification POSTs") is not None,
+          "push_test fires a REAL push to both of the user's endpoints")
+    _, test_payload = decrypt_webpush(push2.hits("/a")[-1], browser_a,
+                                      browser_a_public, browser_a_auth)
+    check(test_payload["title"] == "Test notification"
+          and test_payload["url"] == "/",
+          "the test payload decrypts with title + url (goes to the dashboard)"
+          " (%s)" % test_payload)
+    check("2 browser" in message,
+          "the action response reports both deliveries (%s)" % message)
+    set_types = [d.get("type") for d in result if d.get("type") == "set"]
+    check(len(set_types) >= 1, "the action also refreshes the dashboard slot")
+
+    print("== test notification: failure surfaces the real reason ==")
+    push2.status_for["/b"] = 401
+    b_before = len(push2.hits("/b"))
+    result = act(auth, dash["push_test"], {})
+    message = action_notify(result) or ""
+    check("reached 1 browser" in message and "HTTP 401" in message,
+          "a refused endpoint reports reached/failed + the status (%s)" % message)
+    check(len(push2.hits("/b")) > b_before, "the refused endpoint was still POSTed")
+
+    print("== truthful delivery: 4xx is refused, retried, never delivered ==")
+    # A refused (401) endpoint must NOT count as delivered; the queue
+    # retries on later ticks and — after MAX_ATTEMPTS — gives up WITH a
+    # warning instead of silently claiming success. The subscription
+    # row survives (a VAPID key rotation may heal it).
+    row_before = subscription_count(endpoint_b)
+    b_before = len(push2.hits("/b"))
+    raise_global_block("Refused delivery question?")
+    check(wait_for(lambda: len(push2.hits("/b")) > b_before, timeout=15,
+                   what="first attempt at the refused endpoint") is not None,
+          "the event still reaches the refused endpoint")
+    def give_up_in_log():
+        try:
+            with open(os.path.join(WORK, "app.log")) as handle:
+                return "gave up after %d attempts" % 5 in handle.read()
+        except IOError:
+            return False
+    check(wait_for(give_up_in_log, timeout=30, what="give-up warning in app.log")
+          is not None,
+          "after MAX_ATTEMPTS the give-up warning lands in the log (never silent)")
+    check(subscription_count(endpoint_b) == row_before,
+          "a refused endpoint is NOT pruned (it may recover)")
+    check(len(fingerprints_of(auth)) == 2,
+          "the bell state still lists both endpoints after give-up")
+
+    print("== truthful delivery: 404/410 pruning regression ==")
+    push2.status_for["/b"] = 410
+    b_before = len(push2.hits("/b"))
+    raise_global_block("Prune after refusal question?")
+    check(wait_for(lambda: len(push2.hits("/b")) > b_before, timeout=15,
+                   what="POST to the now-gone endpoint") is not None,
+          "the event reaches the endpoint before its 410")
+    check(wait_for(lambda: subscription_count(endpoint_b) == 0, timeout=15,
+                   what="410 endpoint pruned after refusal episode") is not None,
+          "a 410 endpoint is pruned even after an earlier refusal episode")
+    check(wait_for(lambda: len(fingerprints_of(auth)) == 1, timeout=15,
+                   what="bell state drops the pruned endpoint") is not None,
+          "the refreshed bell state drops the pruned endpoint")
+
+    print("== per-browser bell: markup + client verdict wiring ==")
+    status, home_page = http("GET", "/", timeout=15)
+    check(status == 200 and "data-push-test" in home_page
+          and "data-bell-on" in home_page and "data-bell-off" in home_page,
+          "the bell card ships the per-browser buttons + test button")
+    check("push_endpoints" in home_page and "crypto.subtle.digest" in home_page,
+          "the client fingerprints its own endpoint and matches the list")
+
+    print("== cleanup: unsubscribe the probe fixtures ==")
+    act(auth, dash["push_unsubscribe"], {"endpoint": endpoint_a})
+    check(subscription_count(endpoint_a) == 0,
+          "unsubscribing removes the probe's endpoint row")

test/harness.py

@@ -139,6 +139,7 @@ Speed pass (plan 11, task 0):
   SADDLE_STALE_S=3, plus the fake daemon's scripted timings — the
   wire-traffic shapes stay realistic, just faster.
 """
+import hashlib
 import json
 import os
 import re