harness: sequential is the default runner — the unbounded parallel fan-out (13 apps + fake daemons at once) OOM'd the machine; --parallel [N] opts back into shards capped at N in flight (default 4), --bundle is unchanged, and the meson timeout covers the summed bundle times; users 39/0, dispatch 97/0 sequential

meson.build

@@ -252,14 +252,14 @@ clanker_saddle_exe = executable('clanker-saddle',
 # =============================================================================
 # Tests: the web-stack suite is a package (plan 16 R5, arch 18) —
 # test/web_stack_test.py is the entry shim meson invokes; the shared
-# machinery (fake saddle daemon, wire helpers, fixtures, the parallel
-# runner) lives in test/harness.py and one module per scenario bundle
-# in test/bundles/. Each bundle runs as its own shard process (own app,
-# daemons and SQLite scratch) and the runner aggregates the verdicts
-# into this single test's result.
+# machinery (fake saddle daemon, wire helpers, fixtures, the runner)
+# lives in test/harness.py and one module per scenario bundle
+# in test/bundles/. The default run is SEQUENTIAL (the unbounded
+# parallel fan-out OOM'd the machine; --parallel N opts back in with a
+# concurrency cap), so the timeout covers the summed bundle times.
 # =============================================================================
 python_test = import('python').find_installation('python3')
 test('web-stack', python_test,
     args: [files('test/web_stack_test.py'), clanker_saddle_exe],
-    timeout: 900,
+    timeout: 3600,
 )

test/harness.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 """The web-stack test harness (plan 16 R5, arch 18): everything the
-scenario bundles share, plus the parallel runner.
+scenario bundles share, plus the runner (sequential by default,
+`--parallel [N]` for capped parallel shards).
 
 Layout: test/harness.py (this file — the FakeSaddle daemon, the
 http/act/login/slot_public wire helpers, wait_for/quiet_window/db/check,
@@ -121,10 +122,11 @@ Speed pass (plan 11, task 0):
   bundles (pages, dispatch, subtasks, lineage, allocation, lifecycle,
   answers), each running in its own process against its own app
   instance, fake daemons and SQLite database. The default invocation
-  runs all bundles in parallel and aggregates the verdicts and check
-  counts into one exit code — meson sees a single green/red test.
-  `--bundle NAME` runs one shard for debugging; WEBSTACK_SEQUENTIAL=1
-  runs them in-process, one after another.
+  runs the bundles SEQUENTIALLY, in-process, one after another — the
+  unbounded parallel fan-out (one app + fake daemons per shard, all at
+  once) OOM'd the machine. `--bundle NAME` runs one shard for
+  debugging; `--parallel [N]` opts back into parallel shards, at most N
+  at a time (default 4).
 - Within a bundle, scenario groups SHARE the app+daemon instance and
   carry per-group state as database rows (task/session/block
   fixtures), never process restarts; the one deliberate restart (the
@@ -1358,33 +1360,50 @@ def run_bundle(name):
     return failures
 
 
-def run_all_parallel():
-    """The default meson entry: one subprocess per bundle (each with its
-    own app, daemons and database), verdicts aggregated into one exit
-    code and one combined check-count summary."""
+def run_all_parallel(limit):
+    """The explicit opt-in: one subprocess per bundle (each with its own
+    app, daemons and database), at most `limit` in flight at a time
+    (every shard is an app plus fake daemons — an unbounded fan-out
+    OOMs the machine), verdicts aggregated into one exit code and one
+    combined check-count summary."""
     base = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
     log_dir = os.path.join(os.path.dirname(base) or "/tmp/kilo",
                            "webstack-shards")
     os.makedirs(log_dir, exist_ok=True)
-    procs = {}
     started = time.monotonic()
-    for name in BUNDLES_ORDER:
-        out_path = os.path.join(log_dir, "bundle-%s.log" % name)
-        handle = open(out_path, "w")
-        procs[name] = (subprocess.Popen(
-            [sys.executable, "-u",
-             os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                          "web_stack_test.py"), APP, "--bundle", name],
-            stdout=handle, stderr=subprocess.STDOUT), handle, out_path)
+    limit = max(1, int(limit))
+    pending = list(BUNDLES_ORDER)
+    running = []
+    finished = []
+    while pending or running:
+        while pending and len(running) < limit:
+            name = pending.pop(0)
+            out_path = os.path.join(log_dir, "bundle-%s.log" % name)
+            handle = open(out_path, "w")
+            proc = subprocess.Popen(
+                [sys.executable, "-u",
+                 os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                              "web_stack_test.py"), APP, "--bundle", name],
+                stdout=handle, stderr=subprocess.STDOUT)
+            running.append((name, proc, handle, out_path, time.monotonic()))
+        time.sleep(0.2)
+        still = []
+        for name, proc, handle, out_path, spawned in running:
+            if proc.poll() is None:
+                if time.monotonic() - spawned > SHARD_TIMEOUT_S:
+                    proc.kill()
+                    proc.wait()
+                else:
+                    still.append((name, proc, handle, out_path, spawned))
+                continue
+            handle.close()
+            finished.append((name, proc, out_path))
+        running = still
     total_pass = total_fail = 0
     broken = []
-    for name, (proc, handle, out_path) in procs.items():
-        try:
-            proc.wait(timeout=SHARD_TIMEOUT_S)
-        except subprocess.TimeoutExpired:
-            proc.kill()
-            proc.wait()
-        handle.close()
+    by_name = {name: (proc, out_path) for name, proc, out_path in finished}
+    for name in BUNDLES_ORDER:
+        proc, out_path = by_name[name]
         text = open(out_path).read()
         match = re.search(r"(\d+) passed, (\d+) failed", text)
         took = re.search(r"\[bundle %s\] ([0-9.]+)s" % name, text)
@@ -1417,13 +1436,16 @@ def main():
     argv = sys.argv[2:]
     if "--bundle" in argv:
         sys.exit(1 if run_bundle(argv[argv.index("--bundle") + 1]) else 0)
-    if os.environ.get("WEBSTACK_SEQUENTIAL"):
-        # Debug path: every bundle in-process, one after another.
-        for name in BUNDLES_ORDER:
-            if run_bundle(name):
-                return 1
-        return 0
-    return run_all_parallel()
+    if "--parallel" in argv:
+        at = argv.index("--parallel")
+        limit = 4
+        if len(argv) > at + 1 and argv[at + 1].isdigit():
+            limit = int(argv[at + 1])
+        return run_all_parallel(limit)
+    for name in BUNDLES_ORDER:
+        if run_bundle(name):
+            return 1
+    return 0
 
 
 if __name__ == "__main__":