Initial release: SpecMem harness (code only, credentials-free)

.gitignore

@@ -0,0 +1,14 @@
+__pycache__/
+*.pyc
+.DS_Store
+
+# benchmark data is downloaded, never committed (see data/README.md)
+data/*
+!data/README.md
+
+# experiment outputs
+results/
+
+# credentials — never commit keys; pass them via environment variables
+*.env
+secrets/

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,169 @@
+# SpecMem: Accelerating Agentic Tool Calling via Live Memory Management
+
+[![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/)
+[![Paper](https://img.shields.io/badge/paper-under_review-b31b1b.svg)](#citation)
+
+Agents spend much of their latency decoding tool calls token by token, yet the
+calls a user needs are highly repetitive across sessions. **SpecMem** turns
+that repetition into speed: it keeps a small, per-user, capacity-bounded memory
+of past tool calls that is updated *live* as the agent runs, and retrieves the
+closest past call as a **draft** for speculative decoding. The served model
+verifies the draft, so outputs are exactly those of standard decoding — wrong
+drafts cost only compute, never correctness.
+
+The key finding is that **liveness drives the gain**: a store that keeps
+ingesting and evicting stays fresh as the query distribution drifts, while a
+frozen datastore (built once, then fixed) degrades over sessions.
+
+**Highlights**
+
+- **Live per-user memory as a drafter.** Top-1 cosine retrieval over
+  lightweight query embeddings (`all-MiniLM-L6-v2`, CPU), per-user
+  partitioning, LRU eviction at a small fixed capacity, online write-back
+  after every verified call. No training, no extra GPU.
+- **End-to-end wall-clock speedups** of 1.62x / 1.74x / 1.18x / 1.69x over
+  vanilla autoregressive decoding on API-Bank, ToolAlpaca, BFCL v4, and
+  ToolBench, matching or exceeding a faithful frozen-datastore
+  (ToolSpec-style) baseline on all four.
+- **Verified across serving stacks and architectures:** `gpt-oss-120b` (MoE)
+  and `gemma-4-31B-it` (dense) on sglang, `Nemotron-3-Super-120B`
+  (hybrid-SSM) on vLLM — anything with an OpenAI-compatible endpoint works.
+- **Safety-aware speculation:** an idempotency gate defers speculative
+  *execution* of irreversible tools (payments, deletes), keeping the speedup
+  while avoiding side effects a verifier cannot undo.
+
+## How it works
+
+```
+user query ──> embed ──> per-user memory (capacity-bounded, LRU)
+                              │  top-1 cosine ≥ τ
+                              ▼
+                        drafted tool call ──> served model verifies
+                              ▲                  (token-level accept)
+                              │
+                        write-back of the verified call (live update)
+```
+
+Every query is answered once by the served model with greedy decoding (the
+*target*). Each memory policy ("arm") drafts from its own store and is scored
+by the token-level longest common prefix between its draft and the target —
+the accepted-token count a speculative decoder would realize. The compared
+arms:
+
+| Arm (code name) | Description |
+|---|---|
+| `no_memory` | schema-only draft; lower bound |
+| `static_global` | one global store frozen after warmup (ToolSpec-style) |
+| `personal_memory` | **SpecMem**: per-user, live-updating, capacity-bounded |
+| `toolspec` | faithful ToolSpec reimplementation (frozen kNN-vote + schema FSM) |
+
+## Installation
+
+```bash
+git clone <this-repo> specmem && cd specmem
+pip install -r requirements.txt
+bash scripts/download_data.sh        # fetches BFCL, Seal-Tools, ToolAlpaca, API-Bank
+```
+
+Benchmark data is downloaded from the official sources, never redistributed
+here; two datasets need a small manual step (ToolBench, tau2-bench) — see
+[`data/README.md`](data/README.md).
+
+## Quickstart
+
+**1. Serve a tool-calling model** behind any OpenAI-compatible endpoint, e.g.
+
+```bash
+python -m sglang.launch_server --model-path openai/gpt-oss-120b --port 30000
+```
+
+**2. Run the main acceptance experiment** (3 arms x 40 users x 12 sessions,
+3 stream seeds — the paper's headline setting):
+
+```bash
+python -m harness.run_accept \
+    --users 40 --tasks-per-user 15 --sessions 12 --queries-per-session 6 \
+    --capacity 48 --n-seeds 3 --url http://localhost:30000/v1 --tag main
+```
+
+Targets are cached by exact query string, so re-runs and all memory-arm
+replays are GPU-free. Results land in `results/main_accept_results.json`
+(per-session and overall MAT / accepted fraction / exact rate).
+
+Add `--benchmark sealtools` for Seal-Tools. For other served models, point
+`--url`/`--model` at the endpoint and `--model-path` (or the
+`SPECMEM_TOKENIZER` env var) at the model's tokenizer so acceptance is
+measured in that model's own tokens.
+
+## Reproducing the paper
+
+| Experiment | Command |
+|---|---|
+| Main acceptance table (BFCL / Seal-Tools) | `python -m harness.run_accept ...` (above) |
+| 4-benchmark main table + wall-clock speedups | `python -m harness.phase4_maintable` |
+| Freshness-over-sessions curve | `python -m harness.phase4_partb`, then `python -m harness.phase4_freshness_fig` |
+| Memory-capacity sweep | `python -m harness.capacity_sweep` |
+| Ablations (eviction, sharing, perturbation) | `python -m harness.run_ablation` |
+| Warmup-fraction sweep | `python -m harness.review_r1_warmup` |
+| Provenance (shared vs per-user) | `python -m harness.review_r2_provenance` |
+| Retrieval-threshold sweep | `python -m harness.review_r3_confidence` |
+| Reset / TTL memory-policy arms | `python -m harness.reset_arm`, `python -m harness.ttl_arm` |
+| Suffix-decoding baseline | `python -m harness.phase4_suffixdecoding_maintable` |
+| Throughput / overlap under load | `python -m harness.phase4_throughput`, `python -m harness.phase4_overlap` |
+| Speculative-execution safety gate | `python -m harness.safety` |
+| Bootstrap confidence intervals | `python -m harness.bootstrap_ci` |
+| tau2-bench live traces + scoring | `python -m harness.tau2_live generate / extract / score` |
+
+The tau2-bench `generate` mode runs the served model as the agent against a
+live GPT-4.1 user simulator and requires `OPENAI_API_KEY` (and optionally
+`OPENAI_BASE_URL`) in the environment, plus a
+[tau2-bench](https://github.com/sierra-research/tau2-bench) install
+(`TAU2_BIN`, `TAU2_DATA_DIR`). Credentials are read from environment
+variables only and a leak check aborts if a key ever appears in an artifact.
+
+## Repository layout
+
+```
+harness/            all experiment code (run as python -m harness.<module>)
+  memory.py         memory arms: NoMemory, StaticGlobal, PersonalMemory (SpecMem),
+                    ToolSpecBaseline, suffix-decoding baseline
+  simulate.py       multi-session, multi-user query-stream generator
+  data.py           benchmark loaders (BFCL, Seal-Tools, ToolAlpaca, API-Bank,
+                    ToolBench, tau2)
+  client.py         OpenAI-compatible client + tool-call parsers (harmony, XML)
+  metrics.py        canonicalization + token-LCP acceptance scoring
+  run_accept.py     main 3-arm acceptance experiment
+  ...               see the table above for the per-experiment entry points
+scripts/            data download
+data/               benchmark data (downloaded; see data/README.md)
+results/            experiment outputs (created at runtime)
+```
+
+## Environment variables
+
+| Variable | Purpose | Default |
+|---|---|---|
+| `TOOL_SERVER_URL` | served-model endpoint | `http://localhost:30000/v1` |
+| `SPECMEM_TOKENIZER` | tokenizer for the accept metric | `openai/gpt-oss-120b` |
+| `OPENAI_API_KEY` / `OPENAI_BASE_URL` | tau2 user-simulator credentials | — |
+| `TAU2_BIN` / `TAU2_DATA_DIR` | tau2-bench CLI and data locations | `tau2` / — |
+
+## Citation
+
+The paper is currently under review. If you use this code, please cite:
+
+```bibtex
+@article{specmem2026,
+  title  = {SpecMem: Accelerating Agentic Tool Calling via Live Memory Management},
+  author = {Anonymous},
+  note   = {Under review},
+  year   = {2026}
+}
+```
+
+## License
+
+This repository is released under the [Apache License 2.0](LICENSE).
+Benchmark datasets and served models keep their own licenses (see
+[`data/README.md`](data/README.md)).

data/README.md

@@ -0,0 +1,48 @@
+# Benchmark data
+
+We do not redistribute any third-party benchmark. Run
+
+```bash
+bash scripts/download_data.sh
+```
+
+from the repository root to fetch everything that has a direct official URL.
+Expected layout after download:
+
+```
+data/
+  bfcl/        BFCL_v4_simple_python.json  BFCL_v4_multiple.json  BFCL_v4_parallel.json
+  sealtools/   tool.jsonl  test_in_domain.jsonl
+  toolalpaca/  eval_simulated.json  eval_real.json
+  apibank/     level-1-api.json  level-2-api.json
+  toolbench/   test_instruction/G1_instruction.json ... (manual, see below)
+  tau2/        tools_airline.json  tools_retail.json  tools_telecom.json (generated, see below)
+```
+
+## Provenance and licenses
+
+| Dataset | Source | License |
+|---|---|---|
+| BFCL v4 | [ShishirPatil/gorilla](https://github.com/ShishirPatil/gorilla) (`berkeley-function-call-leaderboard/bfcl_eval/data`) | Apache-2.0 |
+| Seal-Tools | [fairyshine/Seal-Tools](https://github.com/fairyshine/Seal-Tools) (`Seal-Tools_Dataset`) | Apache-2.0 |
+| ToolAlpaca | [tangqiaoyu/ToolAlpaca](https://github.com/tangqiaoyu/ToolAlpaca) (`data/`) | Apache-2.0 |
+| API-Bank | [HF: liminghao1630/API-Bank](https://huggingface.co/datasets/liminghao1630/API-Bank) (`test-data/`) | MIT |
+| ToolBench | [OpenBMB/ToolBench](https://github.com/OpenBMB/ToolBench) official Google Drive release | Apache-2.0 |
+| tau2-bench | [sierra-research/tau2-bench](https://github.com/sierra-research/tau2-bench) | MIT |
+
+## Manual steps
+
+**ToolBench** (only needed for the 4-benchmark main table,
+`harness/phase4_maintable.py`): download the data archive linked from the
+[OpenBMB/ToolBench README](https://github.com/OpenBMB/ToolBench) (Google
+Drive) and copy `data/test_instruction/G{1,2,3}_*.json` to
+`data/toolbench/test_instruction/`.
+
+**tau2-bench** (only needed for the tau2 experiments): the harness needs the
+per-domain tool registries `data/tau2/tools_{domain}.json`. These are
+generated from the tau2-bench repo's domain toolkit sources
+(`src/tau2/domains/<domain>/tools.py`) with `harness/tau2_extract.py`, which
+parses the `@is_tool` methods into BFCL-style JSON schemas. Live trace
+generation additionally requires installing
+[tau2-bench](https://github.com/sierra-research/tau2-bench) itself (the
+`tau2` CLI) — see `harness/tau2_live.py`.

harness/arch_fig.py

@@ -0,0 +1,186 @@
+"""Conceptual/architecture figures for the paper (no model server needed).
+
+Produces two vector figures under paper/figures/:
+  * architecture.pdf/png  -- the three memory arms feeding a draft-then-verify
+                             speculative decoder (no-memory vs frozen global
+                             datastore vs our persistent per-user evicting store).
+  * safety_gate.pdf/png   -- the speculative-execution decision flow contrasting
+                             naive-exec, confidence-gate, and idempotency-gate.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from matplotlib.patches import FancyBboxPatch, FancyArrowPatch
+
+ROOT = Path(__file__).resolve().parent.parent
+FIGS = ROOT / "figures"
+
+BLUE = "#1b6ca8"
+RED = "#d1495b"
+GREY = "#888888"
+GREEN = "#2e8b57"
+DARK = "#2b2b2b"
+
+
+def _box(ax, x, y, w, h, text, fc="white", ec=DARK, fs=8.5, lw=1.3, tc=DARK,
+         style="round,pad=0.02,rounding_size=0.06", bold=False):
+    p = FancyBboxPatch((x, y), w, h, boxstyle=style, fc=fc, ec=ec, lw=lw,
+                       mutation_aspect=1.0, zorder=2)
+    ax.add_patch(p)
+    ax.text(x + w / 2, y + h / 2, text, ha="center", va="center", fontsize=fs,
+            color=tc, zorder=3, fontweight="bold" if bold else "normal",
+            wrap=True)
+    return p
+
+
+def _arrow(ax, xy1, xy2, color=DARK, lw=1.4, style="-|>", ls="-", rad=0.0):
+    a = FancyArrowPatch(xy1, xy2, arrowstyle=style, mutation_scale=12,
+                        color=color, lw=lw, linestyle=ls, zorder=1,
+                        connectionstyle=f"arc3,rad={rad}")
+    ax.add_patch(a)
+
+
+def architecture():
+    fig, ax = plt.subplots(figsize=(10.5, 4.6))
+    ax.set_xlim(0, 10.5)
+    ax.set_ylim(0, 4.6)
+    ax.axis("off")
+
+    # user query stream (sessions)
+    _box(ax, 0.15, 2.0, 1.5, 0.8,
+         "User $u$ query\n(session $s$)", fc="#eef3f7", fs=8.5)
+    ax.text(0.9, 1.75, "recurring, evolving\nper-user tool use",
+            ha="center", va="top", fontsize=6.8, color=GREY, style="italic")
+
+    # three arms as stacked draft sources
+    arm_x = 2.35
+    arms = [
+        (3.55, GREY, "No memory",
+         "schema-only draft\n(zero history)"),
+        (2.30, RED, "Static datastore (ToolSpec-style)",
+         "one global store,\nbuilt once then FROZEN"),
+        (1.05, BLUE, "Ours: persistent per-user memory",
+         "grows across sessions,\nLRU/LFU eviction,\npersonalized retrieval"),
+    ]
+    for y, color, title, sub in arms:
+        _box(ax, arm_x, y, 3.0, 1.05, "", ec=color, lw=1.6,
+             fc=color + "14" if False else "white")
+        ax.text(arm_x + 0.12, y + 0.86, title, ha="left", va="center",
+                fontsize=8.2, color=color, fontweight="bold")
+        ax.text(arm_x + 0.12, y + 0.36, sub, ha="left", va="center",
+                fontsize=7.2, color=DARK)
+        _arrow(ax, (1.65, 2.4), (arm_x, y + 0.55), color=color, lw=1.2,
+               rad=0.05)
+
+    # draft -> speculative decoder
+    dec_x = 6.15
+    _box(ax, dec_x, 1.75, 1.95, 1.3,
+         "Draft-then-verify\nspeculative\ndecoder", fc="#fff7e6",
+         ec="#c8860a", fs=8.5, bold=True)
+    for y, color, *_ in arms:
+        _arrow(ax, (arm_x + 3.0, y + 0.55), (dec_x, 2.4), color=color, lw=1.1,
+               ls="--", rad=-0.05)
+    ax.text(dec_x + 0.98, 3.12, "draft tool call", ha="center", va="bottom",
+            fontsize=7, color="#c8860a")
+
+    # served target model
+    _box(ax, dec_x - 0.1, 0.35, 2.15, 0.85,
+         "Served target model\n(gpt-oss-120b, greedy)", fc="#f2f2f2", fs=7.6)
+    _arrow(ax, (dec_x + 0.95, 1.75), (dec_x + 0.95, 1.2), color=DARK, lw=1.2)
+    _arrow(ax, (dec_x + 1.15, 1.2), (dec_x + 1.15, 1.75), color=DARK, lw=1.2)
+    ax.text(dec_x + 1.35, 1.47, "verify\n(token LCP)", ha="left", va="center",
+            fontsize=6.6, color=DARK)
+
+    # accepted output + metric
+    _box(ax, 8.55, 1.75, 1.8, 1.3,
+         "Accepted tokens\n= mean accepted\ntokens (MAT)", fc="#eaf5ee",
+         ec=GREEN, fs=8.2, tc=DARK)
+    _arrow(ax, (dec_x + 1.95, 2.4), (8.55, 2.4), color=GREEN, lw=1.6)
+
+    # observe/write-back loop (target grows the personal store)
+    _arrow(ax, (dec_x + 0.05, 1.9), (arm_x + 3.0, 1.5), color=BLUE, lw=1.1,
+           ls=":", rad=0.28)
+    ax.text(5.1, 1.05, "observe target $\\rightarrow$ write back\n"
+            "(personal store only; static store frozen after warmup)",
+            ha="center", va="center", fontsize=6.8, color=BLUE, style="italic")
+
+    ax.set_title("Persistent, personalized tool-call memory feeding a "
+                 "speculative decoder", fontsize=10.5, fontweight="bold")
+    fig.tight_layout()
+    FIGS.mkdir(parents=True, exist_ok=True)
+    fig.savefig(FIGS / "architecture.pdf")
+    fig.savefig(FIGS / "architecture.png", dpi=150)
+    plt.close(fig)
+    print("wrote architecture.{pdf,png}")
+
+
+def safety_gate():
+    fig, ax = plt.subplots(figsize=(9.5, 4.2))
+    ax.set_xlim(0, 9.5)
+    ax.set_ylim(0, 4.2)
+    ax.axis("off")
+
+    _box(ax, 0.2, 1.75, 1.75, 0.9,
+         "Memory proposes\na drafted\ntool call", fc="#eef3f7", fs=8.2)
+
+    # decision diamond-ish: idempotent?
+    _box(ax, 2.35, 1.75, 1.6, 0.9, "Tool\nidempotent?", fc="#fff7e6",
+         ec="#c8860a", fs=8.4, bold=True)
+    _arrow(ax, (1.95, 2.2), (2.35, 2.2))
+
+    # idempotent -> always speculatively execute (safe)
+    _box(ax, 4.35, 3.0, 2.4, 0.85,
+         "Speculatively EXECUTE\n(wrong $\\Rightarrow$ just re-run)",
+         fc="#eaf5ee", ec=GREEN, fs=8.0)
+    _arrow(ax, (3.95, 2.5), (4.35, 3.35), color=GREEN, rad=0.15)
+    ax.text(4.15, 3.05, "yes", fontsize=7.5, color=GREEN, ha="left")
+
+    # non-idempotent -> policy fork
+    ax.text(4.15, 1.55, "no", fontsize=7.5, color=RED, ha="left")
+    _box(ax, 4.35, 0.35, 2.4, 2.2, "", ec=RED, lw=1.4, fc="#fdeef1")
+    ax.text(5.55, 2.35, "Non-idempotent tool\n(policy decides)",
+            ha="center", va="center", fontsize=8.0, color=RED,
+            fontweight="bold")
+    _arrow(ax, (3.95, 1.95), (4.35, 1.6), color=RED, rad=-0.12)
+
+    ax.text(4.5, 1.75, "naive:", fontsize=7.6, color=DARK, ha="left",
+            fontweight="bold")
+    ax.text(4.95, 1.75, "execute anyway", fontsize=7.6, color=RED, ha="left")
+    ax.text(4.5, 1.32, "conf-gate:", fontsize=7.6, color=DARK, ha="left",
+            fontweight="bold")
+    ax.text(5.15, 1.32, "execute if conf$\\geq\\tau$", fontsize=7.6,
+            color="#c8860a", ha="left")
+    ax.text(4.5, 0.89, "ours:", fontsize=7.6, color=DARK, ha="left",
+            fontweight="bold")
+    ax.text(4.9, 0.89, "NEVER execute;", fontsize=7.6, color=GREEN, ha="left")
+    ax.text(4.9, 0.58, "draft only, wait for verify", fontsize=7.0,
+            color=GREEN, ha="left")
+
+    # outcomes
+    _box(ax, 7.15, 2.55, 2.15, 0.95,
+         "Irreversible harm\nif draft wrong\n(severity-weighted cost)",
+         fc="#fdeef1", ec=RED, fs=7.6, tc=RED)
+    _box(ax, 7.15, 0.65, 2.15, 0.95,
+         "Zero cost, keeps\nlatency win on\nsafe tools", fc="#eaf5ee",
+         ec=GREEN, fs=7.8, tc=DARK)
+    _arrow(ax, (6.75, 1.65), (7.15, 2.9), color=RED, rad=0.1, lw=1.2)
+    _arrow(ax, (6.75, 0.95), (7.15, 1.1), color=GREEN, rad=-0.05, lw=1.4)
+
+    ax.set_title("Idempotency-gated speculative execution: naive and "
+                 "confidence gates still leak irreversible cost",
+                 fontsize=10.0, fontweight="bold")
+    fig.tight_layout()
+    FIGS.mkdir(parents=True, exist_ok=True)
+    fig.savefig(FIGS / "safety_gate.pdf")
+    fig.savefig(FIGS / "safety_gate.png", dpi=150)
+    plt.close(fig)
+    print("wrote safety_gate.{pdf,png}")
+
+
+if __name__ == "__main__":
+    architecture()
+    safety_gate()

harness/bootstrap_ci.py

@@ -0,0 +1,109 @@
+"""Bootstrap/paired CIs + variance decomposition (external-review item 5).
+
+Deterministically replays the phase-2 seed-0 stream against CACHED targets
+(no model calls) to obtain per-instance paired (static, personal) accepts,
+then reports:
+  - paired bootstrap 95% CI for the post-warmup MAT gap and % gap
+  - per-user, per-session, per-signature-task gap distributions
+Writes results/phase2_bootstrap_ci.json.
+"""
+from __future__ import annotations
+
+import json
+import random
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .data import load_bfcl
+from .memory import Embedder, PersonalMemory, StaticGlobal
+from .run_accept import MODEL_PATH, _parse_target
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+
+
+def main():
+    metrics.get_tokenizer(MODEL_PATH)
+    tasks = load_bfcl()
+    embedder = Embedder()
+    instances = build_users(tasks, n_users=40, tasks_per_user=15,
+                            n_sessions=12, queries_per_session=6, seed=0)
+    instances.sort(key=lambda x: (x.session, x.user_id))
+    targets = json.loads((RESULTS / "phase2_targets_seed0.json").read_text())
+
+    static, personal = StaticGlobal(), PersonalMemory(capacity=48,
+                                                      eviction="lru")
+    rows = []          # (user, session, sig, a_static, a_personal)
+    cur = -1
+    for ins in instances:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur:
+            cur = ins.session
+            if cur == 1:
+                static.freeze()
+        a_s = metrics.score(static.draft(ins.query, ins.functions,
+                                         ins.user_id, embedder),
+                            tgt)["accept_length"]
+        a_p = metrics.score(personal.draft(ins.query, ins.functions,
+                                           ins.user_id, embedder),
+                            tgt)["accept_length"]
+        rows.append((ins.user_id, ins.session, ins.signature_id, a_s, a_p))
+        cname, cargs = _parse_target(tgt)
+        for a in (static, personal):
+            a.observe(ins.query, ins.functions, ins.user_id, cname, cargs,
+                      embedder)
+        if ins.session == 0:
+            personal.seed_shared(ins.query, cname, cargs, embedder)
+
+    post = [r for r in rows if r[1] > 0]
+    n = len(post)
+    mat_s = sum(r[3] for r in post) / n
+    mat_p = sum(r[4] for r in post) / n
+
+    rng = random.Random(0)
+    B = 10_000
+    gaps, pct = [], []
+    for _ in range(B):
+        idx = [rng.randrange(n) for _ in range(n)]
+        s = sum(post[i][3] for i in idx) / n
+        p = sum(post[i][4] for i in idx) / n
+        gaps.append(p - s)
+        pct.append(100 * (p - s) / s)
+    gaps.sort(); pct.sort()
+    ci = lambda xs: (round(xs[int(0.025 * B)], 3), round(xs[int(0.975 * B)], 3))
+
+    def group_gaps(key):
+        g = defaultdict(lambda: [0.0, 0.0, 0])
+        for r in post:
+            k = key(r); g[k][0] += r[3]; g[k][1] += r[4]; g[k][2] += 1
+        vals = sorted((v[1] - v[0]) / v[2] for v in g.values())
+        m = len(vals)
+        return {"n_groups": m,
+                "mean_gap": round(sum(vals) / m, 3),
+                "min": round(vals[0], 3), "p25": round(vals[m // 4], 3),
+                "median": round(vals[m // 2], 3),
+                "p75": round(vals[3 * m // 4], 3), "max": round(vals[-1], 3),
+                "groups_with_negative_gap": sum(1 for v in vals if v < 0)}
+
+    out = {
+        "config": {"seed": 0, "targets": "phase2_targets_seed0.json (cached)",
+                   "n_post_warmup_paired": n, "bootstrap_resamples": B},
+        "MAT": {"static": round(mat_s, 3), "personal": round(mat_p, 3),
+                "gap": round(mat_p - mat_s, 3),
+                "gap_pct": round(100 * (mat_p - mat_s) / mat_s, 2)},
+        "paired_bootstrap_95CI": {"gap_MAT": ci(gaps), "gap_pct": ci(pct)},
+        "per_user_gap": group_gaps(lambda r: r[0]),
+        "per_session_gap": group_gaps(lambda r: r[1]),
+        "per_task_gap": group_gaps(lambda r: r[2]),
+    }
+    (RESULTS / "phase2_bootstrap_ci.json").write_text(json.dumps(out,
+                                                                 indent=2))
+    print(json.dumps(out, indent=2))
+
+
+if __name__ == "__main__":
+    main()

harness/capacity_sweep.py

@@ -0,0 +1,79 @@
+"""Capacity sweep for the personal evicting memory (reviewer ask, round 7).
+
+Where does eviction start to cost acceptance? Replays the phase-2 seed-0
+instance stream (identical construction; cached greedy targets, so no GPU/model
+calls) through PersonalMemory at several capacities C plus the unbounded
+variant, and reports post-warmup MAT per capacity. Per-user load is
+tasks_per_user=15 signature tasks and 8+66=74 queries across 12 sessions, so
+C=48 barely binds while small C forces heavy eviction.
+
+Usage: python -m harness.capacity_sweep  (from code/)
+Writes results/phase2_capacity_sweep.json.
+"""
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .data import load_bfcl
+from .memory import Embedder, PersonalMemory, PersonalNoEvict
+from .run_accept import MODEL_PATH, _parse_target
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+
+CAPACITIES = [4, 8, 16, 32, 48]
+
+
+def main():
+    metrics.get_tokenizer(MODEL_PATH)
+    tasks = load_bfcl()
+    embedder = Embedder()
+    instances = build_users(tasks, n_users=40, tasks_per_user=15,
+                            n_sessions=12, queries_per_session=6, seed=0)
+    instances.sort(key=lambda x: (x.session, x.user_id))
+    targets = json.loads(
+        (RESULTS / "phase2_targets_seed0.json").read_text())
+
+    arms = [PersonalMemory(capacity=c, eviction="lru") for c in CAPACITIES]
+    labels = [f"C={c}" for c in CAPACITIES] + ["unbounded"]
+    arms.append(PersonalNoEvict())
+
+    agg = {lab: defaultdict(list) for lab in labels}
+    for ins in instances:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        for lab, a in zip(labels, arms):
+            draft = a.draft(ins.query, ins.functions, ins.user_id, embedder)
+            agg[lab][ins.session].append(metrics.score(draft, tgt))
+        name, argd = _parse_target(tgt)
+        for lab, a in zip(labels, arms):
+            a.observe(ins.query, ins.functions, ins.user_id, name, argd,
+                      embedder)
+            if isinstance(a, PersonalMemory) and ins.session == 0:
+                a.seed_shared(ins.query, name, argd, embedder)
+
+    out = {}
+    for lab in labels:
+        scores = [x for s, xs in agg[lab].items() if s > 0 for x in xs]
+        n = len(scores)
+        out[lab] = {
+            "n": n,
+            "MAT": round(sum(x["accept_length"] for x in scores) / n, 3),
+            "exact_rate": round(sum(1 for x in scores if x["exact"]) / n, 4),
+        }
+    result = {"config": {"users": 40, "tasks_per_user": 15, "sessions": 12,
+                         "queries_per_session": 6, "seed": 0,
+                         "targets": "phase2_targets_seed0.json (cached)"},
+              "post_warmup_MAT_by_capacity": out}
+    (RESULTS / "phase2_capacity_sweep.json").write_text(
+        json.dumps(result, indent=2))
+    print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

harness/client.py

@@ -0,0 +1,202 @@
+"""Thin client over the sglang OpenAI-compatible endpoint for gpt-oss-120b.
+
+Given a query and the offered tool schemas, we ask the genuinely-served model
+to emit a tool call and return the (name, arguments) it produced. This is the
+*target* generation used by the acceptance metric. Generation is greedy
+(temperature 0) so the target is deterministic -- the correct reference for a
+speculative decoder that verifies greedily.
+"""
+from __future__ import annotations
+
+import json
+import re
+import time
+from typing import Any
+
+import os
+
+import requests
+
+# The sglang server runs on the GPU node; from the GPU node itself localhost
+# works, but from the head/launch node it must be addressed by host. Honor an
+# env override so the harness runs from either place.
+DEFAULT_URL = os.environ.get("TOOL_SERVER_URL", "http://localhost:30000/v1")
+
+# gpt-oss harmony tool-call markup, e.g.:
+#   ...to=functions.triangle_properties.get <|constrain|>json<|message|>{...}<|call|>
+_HARMONY_NAME = re.compile(r"to=functions\.([A-Za-z0-9_.\-]+)")
+_HARMONY_ARGS = re.compile(r"<\|message\|>(.*?)<\|call\|>", re.DOTALL)
+
+
+def parse_harmony_content(content: str) -> dict[str, Any] | None:
+    """Fallback: extract the first tool call from raw harmony content.
+
+    sglang's HarmonyParser occasionally leaves the call in `content` instead of
+    populating `tool_calls`; this recovers the genuine call the model emitted.
+    """
+    if not content or "to=functions." not in content:
+        return None
+    nm = _HARMONY_NAME.search(content)
+    if not nm:
+        return None
+    am = _HARMONY_ARGS.search(content, nm.end())
+    raw = am.group(1).strip() if am else "{}"
+    try:
+        args = json.loads(raw) if raw else {}
+    except json.JSONDecodeError:
+        args = {"__raw__": raw}
+    return {"name": nm.group(1), "arguments": args}
+
+
+# Nemotron-H / Llama-style XML tool-call markup, emitted in `content` when the
+# server-side parser (hermes) does not recognize it, e.g.:
+#   <tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>...
+_XML_FUNC = re.compile(r"<function=([A-Za-z0-9_.\-]+)\s*>")
+_XML_PARAM = re.compile(r"<parameter=([A-Za-z0-9_.\-]+)\s*>(.*?)</parameter>",
+                        re.DOTALL)
+
+
+def parse_xml_content(content: str) -> dict[str, Any] | None:
+    """Fallback: extract an XML-style ``<function=..><parameter=..>`` tool call.
+
+    Used for models (e.g. Nemotron-3-Super) whose native tool-call format the
+    served parser leaves in ``content``. Values are kept as the model's literal
+    strings; since draft and target pass through the same parser, only their
+    mutual agreement matters for the acceptance metric.
+    """
+    if not content or "<function=" not in content:
+        return None
+    fm = _XML_FUNC.search(content)
+    if not fm:
+        return None
+    # scope parameters to this function block if a closing tag exists
+    end = content.find("</function>", fm.end())
+    block = content[fm.end():end if end != -1 else None]
+    args: dict[str, Any] = {}
+    for pm in _XML_PARAM.finditer(block):
+        val = pm.group(2).strip()
+        # coerce obvious scalars so canonicalization matches JSON tool_calls
+        low = val.lower()
+        if low in ("true", "false"):
+            args[pm.group(1)] = (low == "true")
+        else:
+            try:
+                args[pm.group(1)] = int(val)
+            except ValueError:
+                try:
+                    args[pm.group(1)] = float(val)
+                except ValueError:
+                    args[pm.group(1)] = val
+    return {"name": fm.group(1), "arguments": args}
+
+
+# BFCL uses Python-style type names; JSON Schema needs these mappings.
+_TYPE_MAP = {"dict": "object", "float": "number", "integer": "integer",
+             "tuple": "array", "list": "array", "string": "string",
+             "boolean": "boolean", "bool": "boolean", "int": "integer",
+             "number": "number", "array": "array", "object": "object"}
+
+
+def _sanitize_schema(node: Any) -> Any:
+    """Recursively convert BFCL Python types into valid JSON Schema."""
+    if isinstance(node, dict):
+        out = {}
+        for k, v in node.items():
+            if k == "type" and isinstance(v, str):
+                if v == "any":
+                    continue                       # unconstrained -> omit type
+                out[k] = _TYPE_MAP.get(v, v)
+            else:
+                out[k] = _sanitize_schema(v)
+        # a "tuple"/"array" with no item schema still needs items for strict
+        # validators; leave as-is otherwise.
+        return out
+    if isinstance(node, list):
+        return [_sanitize_schema(x) for x in node]
+    return node
+
+
+def to_openai_tools(functions: list[dict[str, Any]]) -> list[dict]:
+    """Convert BFCL function schemas to OpenAI tool schema."""
+    tools = []
+    for f in functions:
+        params = f.get("parameters", {}) or {"type": "object", "properties": {}}
+        params = _sanitize_schema(dict(params))
+        if params.get("type") in (None, "dict"):
+            params["type"] = "object"
+        tools.append({
+            "type": "function",
+            "function": {
+                "name": f["name"],
+                "description": f.get("description", ""),
+                "parameters": params,
+            },
+        })
+    return tools
+
+
+class ToolClient:
+    def __init__(self, url: str = DEFAULT_URL, model: str = "gpt-oss-120b",
+                 timeout: float = 120.0):
+        self.url = url.rstrip("/")
+        self.model = model
+        self.timeout = timeout
+
+    def ping(self) -> bool:
+        try:
+            r = requests.get(f"{self.url}/models", timeout=5)
+            return r.status_code == 200
+        except Exception:
+            return False
+
+    def generate_call(self, query: str, functions: list[dict[str, Any]],
+                      retries: int = 3) -> dict[str, Any] | None:
+        """Return {'name':..., 'arguments':{...}} for the model's tool call.
+
+        Returns None if the model declined to call a tool or on hard failure.
+        """
+        tools = to_openai_tools(functions)
+        payload = {
+            "model": self.model,
+            "messages": [
+                {"role": "system", "content":
+                 "You are a function-calling agent. Call exactly one of the "
+                 "provided tools to satisfy the user's request."},
+                {"role": "user", "content": query},
+            ],
+            "tools": tools,
+            # gpt-oss harmony parser rejects tool_choice="required"
+            # (structure_info conflict); it natively emits tool calls with auto.
+            "tool_choice": "auto",
+            "temperature": 0.0,
+            "max_tokens": 512,
+        }
+        last_err = None
+        for attempt in range(retries):
+            try:
+                r = requests.post(f"{self.url}/chat/completions", json=payload,
+                                  timeout=self.timeout)
+                if r.status_code != 200:
+                    last_err = f"http {r.status_code}: {r.text[:200]}"
+                    time.sleep(1.5 * (attempt + 1))
+                    continue
+                msg = r.json()["choices"][0]["message"]
+                tcs = msg.get("tool_calls") or []
+                if not tcs:
+                    # parser left the call in content -> recover it ourselves.
+                    # Try gpt-oss harmony markup first, then Nemotron/Llama XML.
+                    content = msg.get("content") or ""
+                    return (parse_harmony_content(content)
+                            or parse_xml_content(content))
+                fn = tcs[0]["function"]
+                args = fn.get("arguments", "{}")
+                if isinstance(args, str):
+                    try:
+                        args = json.loads(args) if args.strip() else {}
+                    except json.JSONDecodeError:
+                        args = {"__raw__": args}
+                return {"name": fn["name"], "arguments": args}
+            except Exception as e:  # noqa: BLE001
+                last_err = str(e)
+                time.sleep(1.5 * (attempt + 1))
+        raise RuntimeError(f"generate_call failed after {retries}: {last_err}")

harness/crossmodel.py

@@ -0,0 +1,109 @@
+"""Cross-model generalization summary (Phase 3).
+
+Reads the per-model acceptance result JSONs (gpt-oss Phase-2 headline + any
+Phase-3 models) and emits:
+  * results/phase3_crossmodel.json  -- machine-readable comparison
+  * paper/figures/crossmodel.{pdf,png} -- grouped bar of personal vs static MAT
+  * a LaTeX-ready table printed to stdout
+
+No model is re-run here; this only aggregates existing result files. Missing
+model files are skipped (so it works whether or not Nemotron served).
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+FIGS = ROOT / "figures"
+
+# (display name, results json). row order = table order (gpt-oss first).
+MODELS = [
+    ("gpt-oss-120b", RESULTS / "phase2_accept_results.json"),
+    ("gemma-4-31B-it", RESULTS / "phase3_gemma_accept_results.json"),
+    ("Nemotron-3-Super-120B", RESULTS / "phase3_nemotron_accept_results.json"),
+]
+
+
+def _row(path: Path):
+    d = json.loads(path.read_text())
+    ow = d["overall_post_warmup"]
+    per = d["summary"]  # per-session, for tail gap
+    sessions = sorted(int(s) for s in per["personal_memory"].keys())
+    last = str(sessions[-1])
+    def mat(arm, block=ow, key=None):
+        return block[arm]["MAT"] if key is None else block[arm][key]["MAT"]
+    stat, pers, nomem = mat("static_global"), mat("personal_memory"), mat("no_memory")
+    gap = 100.0 * (pers - stat) / stat
+    tail_stat = per["static_global"][last]["MAT"]
+    tail_pers = per["personal_memory"][last]["MAT"]
+    tail_gap = 100.0 * (tail_pers - tail_stat) / tail_stat
+    return {
+        "no_memory": round(nomem, 2),
+        "static_global": round(stat, 2),
+        "personal_memory": round(pers, 2),
+        "gap_pct": round(gap, 1),
+        "tail_gap_pct": round(tail_gap, 1),
+        "personal_seed_std": ow["personal_memory"].get("MAT_seed_std"),
+        "n": ow["personal_memory"]["n"],
+    }
+
+
+def main():
+    out = {}
+    for name, path in MODELS:
+        if path.exists():
+            out[name] = _row(path)
+            print(f"[ok] {name}: {out[name]}")
+        else:
+            print(f"[skip] {name}: {path.name} not found")
+    (RESULTS / "phase3_crossmodel.json").write_text(json.dumps(out, indent=2))
+
+    # LaTeX table body
+    print("\n% --- LaTeX table rows (personal vs static vs none, +gap) ---")
+    for name, r in out.items():
+        print(f"{name} & {r['no_memory']:.2f} & {r['static_global']:.2f} & "
+              f"{r['personal_memory']:.2f} & $+{r['gap_pct']:.0f}\\%$ & "
+              f"$+{r['tail_gap_pct']:.0f}\\%$ \\\\")
+
+    # grouped bar figure
+    try:
+        import matplotlib
+        matplotlib.use("Agg")
+        import matplotlib.pyplot as plt
+        import numpy as np
+        names = list(out.keys())
+        x = np.arange(len(names))
+        w = 0.26
+        nomem = [out[n]["no_memory"] for n in names]
+        stat = [out[n]["static_global"] for n in names]
+        pers = [out[n]["personal_memory"] for n in names]
+        fig, ax = plt.subplots(figsize=(7.2, 3.6))
+        ax.bar(x - w, nomem, w, label="No memory", color="#9e9e9e")
+        ax.bar(x, stat, w, label="Static datastore", color="#4C72B0")
+        ax.bar(x + w, pers, w, label="Personal evicting (ours)", color="#C44E52")
+        ymax = max(pers) * 1.30  # headroom for labels + legend
+        ax.set_ylim(0, ymax)
+        for xi, n in zip(x, names):
+            ax.text(xi + w, out[n]["personal_memory"] + ymax * 0.015,
+                    f"+{out[n]['gap_pct']:.0f}%", ha="center", fontsize=8,
+                    color="#C44E52", fontweight="bold")
+        ax.set_xticks(x)
+        ax.set_xticklabels(names, fontsize=9)
+        ax.set_ylabel("Mean accepted tokens (post-warmup)")
+        ax.set_title("Personalized evicting memory generalizes across served models")
+        ax.legend(fontsize=8, loc="upper center", ncol=3, frameon=False,
+                  bbox_to_anchor=(0.5, 1.0))
+        ax.grid(axis="y", alpha=0.3)
+        fig.tight_layout()
+        FIGS.mkdir(parents=True, exist_ok=True)
+        fig.savefig(FIGS / "crossmodel.pdf")
+        fig.savefig(FIGS / "crossmodel.png", dpi=150)
+        print(f"\n[fig] wrote {FIGS/'crossmodel.pdf'}")
+    except Exception as e:  # noqa: BLE001
+        print(f"[fig] skipped: {e}")
+
+
+if __name__ == "__main__":
+    main()

harness/data.py

@@ -0,0 +1,352 @@
+"""BFCL v4 loading + synthetic per-user session construction.
+
+BFCL examples are almost all distinct functions, so
+there is essentially no natural cross-session repetition. To test the
+personalization/persistence claim we synthesize realistic repetition: each
+simulated user is assigned a small set of *signature tasks* (real BFCL
+examples), and across ordered sessions they re-issue those tasks with perturbed
+numeric argument values -- i.e. the same tool used again with new inputs, which
+is exactly the recurring per-user tool usage the memory is meant to exploit.
+
+Crucially, we do NOT fabricate the target tool call: every (possibly perturbed)
+query is sent to the genuinely served gpt-oss-120b model, and the model's real
+generation is the target. The perturbation only changes the natural-language
+query; the model decides what to emit.
+"""
+from __future__ import annotations
+
+import json
+import random
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+DATA_DIR = Path(__file__).resolve().parent.parent / "data" / "bfcl"
+SEALTOOLS_DIR = Path(__file__).resolve().parent.parent / "data" / "sealtools"
+TAU2_DIR = Path(__file__).resolve().parent.parent / "data" / "tau2"
+TOOLALPACA_DIR = Path(__file__).resolve().parent.parent / "data" / "toolalpaca"
+APIBANK_DIR = Path(__file__).resolve().parent.parent / "data" / "apibank"
+TOOLBENCH_DIR = Path(__file__).resolve().parent.parent / "data" / "toolbench"
+
+
+def _toolbench_schema(api: dict) -> dict[str, Any]:
+    props, required = {}, []
+    for p in (api.get("required_parameters") or []):
+        props[p["name"]] = {"type": _SEAL_TYPE_MAP.get(str(p.get("type", "str")).lower(),
+                                                        "string"),
+                            "description": str(p.get("description", ""))[:160]}
+        if props[p["name"]]["type"] not in ("string", "integer", "number",
+                                            "boolean", "array", "object"):
+            props[p["name"]]["type"] = "string"
+        required.append(p["name"])
+    for p in (api.get("optional_parameters") or []):
+        t = _SEAL_TYPE_MAP.get(str(p.get("type", "str")).lower(), "string")
+        if t not in ("string", "integer", "number", "boolean", "array", "object"):
+            t = "string"
+        props[p["name"]] = {"type": t,
+                            "description": str(p.get("description", ""))[:160]}
+    name = str(api.get("api_name", "api")).strip().replace(" ", "_")
+    return {"name": name or "api",
+            "description": str(api.get("api_description", ""))[:300],
+            "parameters": {"type": "dict", "properties": props,
+                           "required": required}}
+
+
+def load_toolbench(splits: tuple[str, ...] = ("G1_instruction", "G2_instruction",
+                                             "G3_instruction")) -> list[Task]:
+    """ToolBench (Qin et al., 2023, arXiv:2307.16789) as Tasks, from the REAL
+    dataset (OpenBMB/ToolBench Google-Drive `data/test_instruction/`). Each item
+    has a natural-language `query` and an `api_list` of candidate tools with
+    required/optional parameter schemas."""
+    tasks: list[Task] = []
+    for split in splits:
+        f = TOOLBENCH_DIR / "test_instruction" / f"{split}.json"
+        if not f.exists():
+            continue
+        for r in json.loads(f.read_text()):
+            q = (r.get("query") or "").strip()
+            apis = r.get("api_list") or []
+            if not q or len(q) < 8 or not apis:
+                continue
+            funcs = [_toolbench_schema(a) for a in apis]
+            # drop dup / empty-name schemas
+            seen, uniq = set(), []
+            for fn in funcs:
+                if fn["name"] and fn["name"] not in seen:
+                    seen.add(fn["name"])
+                    uniq.append(fn)
+            if not uniq:
+                continue
+            tid = f"toolbench_{split}_{r.get('query_id')}"
+            tasks.append(Task(id=tid, query=q, functions=uniq, origin_id=tid))
+    return tasks
+
+
+def _toolalpaca_params(desc: str) -> dict[str, Any]:
+    """Parse the 'Parameters: {...}' JSON-ish blob out of a ToolAlpaca function
+    description into BFCL-style {type, description} properties. Best-effort:
+    ToolAlpaca param values are free text ('string. One of: [...]'), so we keep
+    the leading type word and stash the rest as the description."""
+    m = re.search(r"Parameters:\s*(\{.*?\})\s*(?:\nOutput|$)", desc, re.S)
+    props: dict[str, Any] = {}
+    if not m:
+        return props
+    try:
+        raw = json.loads(m.group(1))
+    except Exception:
+        return props
+    for pname, pdesc in raw.items():
+        head = str(pdesc).split(".")[0].strip().lower()
+        typ = head if head in ("string", "integer", "number", "boolean",
+                               "array", "object") else "string"
+        prop: dict[str, Any] = {"type": typ, "description": str(pdesc)[:200]}
+        if typ == "array":
+            prop["items"] = {"type": "string"}
+        props[pname] = prop
+    return props
+
+
+def load_toolalpaca(splits: tuple[str, ...] = ("eval_simulated",
+                                              "eval_real")) -> list[Task]:
+    """ToolAlpaca (Tang et al., 2023) as Tasks. Each API's Instructions become
+    queries; the API's Function_Description entries become the tool registry
+    for that query. Downloaded from github.com/tangqiaoyu/ToolAlpaca."""
+    tasks: list[Task] = []
+    for split in splits:
+        f = TOOLALPACA_DIR / f"{split}.json"
+        if not f.exists():
+            continue
+        for tool in json.loads(f.read_text()):
+            fdesc = tool.get("Function_Description") or {}
+            if not isinstance(fdesc, dict):
+                continue
+            funcs = []
+            for fname, d in fdesc.items():
+                if fname in ("components", "Response"):     # doc noise
+                    continue
+                funcs.append({
+                    "name": fname,
+                    "description": str(d).split("\n")[0][:300],
+                    "parameters": {"type": "dict",
+                                   "properties": _toolalpaca_params(str(d)),
+                                   "required": []}})
+            if not funcs:
+                continue
+            name = tool.get("Name", "api")
+            for i, instr in enumerate(tool.get("Instructions", []) or []):
+                if not instr or len(instr) < 8:
+                    continue
+                tid = f"toolalpaca_{split}_{name}_{i}"
+                tasks.append(Task(id=tid, query=instr, functions=funcs,
+                                  origin_id=tid))
+    return tasks
+
+
+@dataclass
+class Task:
+    id: str
+    query: str                      # natural-language user request
+    functions: list[dict[str, Any]]  # tool schemas offered for this query
+    origin_id: str                  # BFCL id this was derived from
+
+
+def _load_jsonl(path: Path) -> list[dict]:
+    return [json.loads(l) for l in path.read_text().splitlines() if l.strip()]
+
+
+def load_bfcl(categories: tuple[str, ...] = ("simple_python", "multiple",
+                                             "parallel")) -> list[Task]:
+    fmap = {
+        "simple_python": "BFCL_v4_simple_python.json",
+        "multiple": "BFCL_v4_multiple.json",
+        "parallel": "BFCL_v4_parallel.json",
+    }
+    tasks: list[Task] = []
+    for cat in categories:
+        rows = _load_jsonl(DATA_DIR / fmap[cat])
+        for r in rows:
+            # question is [[{role, content}, ...]] -- take first user turn.
+            turns = r["question"][0]
+            user_msg = next((m["content"] for m in turns if m["role"] == "user"),
+                            turns[0]["content"])
+            tasks.append(Task(id=r["id"], query=user_msg,
+                              functions=r["function"], origin_id=r["id"]))
+    return tasks
+
+
+_SEAL_TYPE_MAP = {"str": "string", "int": "integer", "float": "number",
+                  "bool": "boolean", "list": "array", "dict": "object"}
+
+
+def _seal_schema(tool: dict) -> dict[str, Any]:
+    """Convert one Seal-Tools registry entry to a BFCL-style function schema.
+
+    Keep only {type, description} per property: Seal-Tools specs carry
+    non-standard keywords (e.g. stringified enums) that strict JSON-schema
+    validators in serving stacks reject.
+    """
+    props = {}
+    for pname, spec in (tool.get("parameters") or {}).items():
+        typ = _SEAL_TYPE_MAP.get(spec.get("type", "str"),
+                                 spec.get("type", "string"))
+        if typ not in ("string", "integer", "number", "boolean",
+                       "array", "object"):
+            typ = "string"
+        clean: dict[str, Any] = {"type": typ,
+                                 "description": str(spec.get("description",
+                                                             ""))}
+        if typ == "array":
+            clean["items"] = {"type": "string"}
+        props[pname] = clean
+    return {
+        "name": tool["api_name"],
+        "description": tool.get("api_description", ""),
+        "parameters": {"type": "dict", "properties": props,
+                       "required": tool.get("required", [])},
+    }
+
+
+def _apibank_schema(spec: dict) -> dict[str, Any]:
+    props = {}
+    for pname, p in (spec.get("input_parameters") or {}).items():
+        typ = _SEAL_TYPE_MAP.get(str(p.get("type", "str")).lower(),
+                                 str(p.get("type", "string")).lower())
+        if typ not in ("string", "integer", "number", "boolean",
+                       "array", "object"):
+            typ = "string"
+        prop: dict[str, Any] = {"type": typ,
+                                "description": str(p.get("description", ""))[:200]}
+        if typ == "array":
+            prop["items"] = {"type": "string"}
+        props[pname] = prop
+    return {"name": spec["name"], "description": str(spec.get("description", ""))[:300],
+            "parameters": {"type": "dict", "properties": props,
+                           "required": list(props.keys())}}
+
+
+def load_apibank(levels: tuple[str, ...] = ("level-1", "level-2")) -> list[Task]:
+    """API-Bank (Li et al., 2023, arXiv:2304.08244) as Tasks. Each item's
+    `input` dialogue is the query; the API specs embedded in `instruction`
+    ('API descriptions: {json}\\n{json}...') become the tool registry.
+    Downloaded from HF liminghao1630/API-Bank test-data."""
+    tasks: list[Task] = []
+    seen = set()
+    for lvl in levels:
+        f = APIBANK_DIR / f"{lvl}-api.json"
+        if not f.exists():
+            continue
+        for r in json.loads(f.read_text()):
+            instr = r.get("instruction", "")
+            query = (r.get("input") or "").strip()
+            if not query or len(query) < 8:
+                continue
+            # parse the embedded API-description JSON objects
+            funcs = []
+            body = instr.split("API descriptions:", 1)
+            if len(body) == 2:
+                for line in body[1].splitlines():
+                    line = line.strip()
+                    if line.startswith('{') and '"name"' in line:
+                        try:
+                            spec = json.loads(line)
+                            funcs.append(_apibank_schema(spec))
+                        except Exception:
+                            pass
+            if not funcs:
+                continue
+            tid = f"apibank_{lvl}_{r.get('file','')}_{r.get('id')}"
+            if tid in seen:
+                continue
+            seen.add(tid)
+            tasks.append(Task(id=tid, query=query, functions=funcs,
+                              origin_id=tid))
+    return tasks
+
+
+def load_sealtools(n_distractors: int = 3) -> list[Task]:
+    """Load the Seal-Tools in-domain test split (Wu et al., 2024) as Tasks.
+
+    Second benchmark for the acceptance experiment: same Task interface as
+    load_bfcl(), so the simulated-user construction and the 3-arm replay are
+    IDENTICAL to the BFCL runs. Each example keeps its first gold API's schema
+    and adds ``n_distractors`` deterministic distractor schemas (stable hash of
+    the example id), shuffled deterministically so the gold schema's position
+    carries no signal. Only ~49% of Seal-Tools queries contain perturbable
+    numerals (vs. most BFCL queries), so the per-user recurrence statistics are
+    natively different from BFCL -- more exact repeats, less argument drift.
+    """
+    import hashlib
+    registry = {t["api_name"]: t
+                for t in _load_jsonl(SEALTOOLS_DIR / "tool.jsonl")}
+    names = sorted(registry)
+    tasks: list[Task] = []
+    for r in _load_jsonl(SEALTOOLS_DIR / "test_in_domain.jsonl"):
+        gold = r["calling"][0]["api"]
+        if gold not in registry:      # all resolve in practice; guard anyway
+            continue
+        h = int(hashlib.sha256(r["id"].encode()).hexdigest(), 16)
+        rng = random.Random(h)
+        distractors = [n for n in rng.sample(names, n_distractors + 1)
+                       if n != gold][:n_distractors]
+        funcs = [_seal_schema(registry[n]) for n in [gold] + distractors]
+        rng.shuffle(funcs)
+        tasks.append(Task(id=r["id"], query=r["query"], functions=funcs,
+                          origin_id=r["id"]))
+    return tasks
+
+
+def load_tau2(pool_size: int = 900, seed: int = 20260716) -> list[Task]:
+    """Load tau2-bench frozen-trajectory decision points (Barres et al., 2025).
+
+    Third benchmark: multi-turn conversational decision points extracted from
+    tau2-bench's shipped reference trajectories (see tau2_extract.py for the
+    frozen-conversation design that keeps the 3-arm comparison controlled).
+    Each Task's query is a rendered transcript prefix; functions are the full
+    tool registry of the task's domain (14 airline / 16 retail tools), so the
+    model must also pick the right tool, not just fill arguments. A
+    deterministic sample of ``pool_size`` keeps the pool comparable to the
+    BFCL (800) / Seal-Tools (700) pools.
+    """
+    tools = {dom: json.loads((TAU2_DIR / f"tools_{dom}.json").read_text())
+             for dom in ("airline", "retail")}
+    rows = _load_jsonl(TAU2_DIR / "decision_points.jsonl")
+    rng = random.Random(seed)
+    rng.shuffle(rows)
+    rows = rows[:pool_size]
+    return [Task(id=r["id"], query=r["query"],
+                 functions=tools[r["domain"]], origin_id=r["id"])
+            for r in rows]
+
+
+_NUM_RE = re.compile(r"(?<![\w.])(-?\d+(?:\.\d+)?)(?![\w.])")
+
+
+def perturb_numeric(query: str, rng: random.Random) -> str:
+    """Replace standalone numbers in a query with new values of similar scale.
+
+    Returns the query unchanged if it contains no substitutable numbers (in
+    which case the task recurs as an exact repeat, which is also realistic).
+    """
+    matches = list(_NUM_RE.finditer(query))
+    if not matches:
+        return query
+    out, last = [], 0
+    for m in matches:
+        out.append(query[last:m.start()])
+        tok = m.group(1)
+        if "." in tok:
+            base = float(tok)
+            lo, hi = max(0.1, base * 0.4), base * 1.9 + 1.0
+            out.append(f"{rng.uniform(lo, hi):.1f}")
+        else:
+            base = int(tok)
+            if abs(base) <= 1:            # keep tiny counts (0/1/2) stable
+                out.append(tok)
+            else:
+                lo, hi = max(2, int(abs(base) * 0.4)), int(abs(base) * 1.9) + 2
+                val = rng.randint(lo, hi)
+                out.append(str(-val if base < 0 else val))
+        last = m.end()
+    out.append(query[last:])
+    return "".join(out)

harness/memory.py

@@ -0,0 +1,432 @@
+"""Three tool-call-memory arms.
+
+An arm exposes two operations:
+  - draft(query, functions, user_id) -> canonical draft string
+  - observe(query, functions, user_id, target_name, target_args) -> None
+      record the genuine target so future drafts can reuse it.
+
+Arms:
+  1. NoMemory      -- schema-only draft (best you can do with zero history).
+  2. StaticGlobal  -- ToolSpec-style: one global datastore, built during warmup
+                      and then FROZEN (no growth, no eviction, no per-user view).
+  3. PersonalMemory (ours) -- per-user store that grows across sessions, with an
+                      eviction policy (LRU or LFU) capping per-user size, and
+                      personalized retrieval (query only the user's own store,
+                      backing off to a small shared store when empty).
+
+Retrieval is top-1 cosine similarity over sentence-transformer embeddings of the
+query text. The drafted call is the canonicalized (name, arguments) of the most
+similar past observation.
+"""
+from __future__ import annotations
+
+import re
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+
+from .metrics import canonical_call_str
+
+
+# --------------------------------------------------------------------------- #
+# Embedding backend
+# --------------------------------------------------------------------------- #
+class Embedder:
+    def __init__(self, name: str = "sentence-transformers/all-MiniLM-L6-v2"):
+        from sentence_transformers import SentenceTransformer
+        self.model = SentenceTransformer(name, device="cpu")
+        self._cache: dict[str, np.ndarray] = {}
+
+    def embed(self, text: str) -> np.ndarray:
+        v = self._cache.get(text)
+        if v is None:
+            v = self.model.encode(text, normalize_embeddings=True)
+            v = np.asarray(v, dtype=np.float32)
+            self._cache[text] = v
+        return v
+
+
+def schema_draft(functions: list[dict[str, Any]]) -> str:
+    """Zero-history draft: the first tool's signature with placeholder args."""
+    if not functions:
+        return ""
+    f = functions[0]
+    props = (f.get("parameters", {}) or {}).get("properties", {}) or {}
+    required = (f.get("parameters", {}) or {}).get("required", []) or list(props)
+    args = {k: None for k in required}
+    return canonical_call_str(f["name"], args)
+
+
+# --------------------------------------------------------------------------- #
+# Datastore entry
+# --------------------------------------------------------------------------- #
+@dataclass
+class Entry:
+    emb: np.ndarray
+    call: str          # canonical target call string
+    freq: int = 1      # times this (query-region) has been reinforced
+
+
+def _best_match(emb: np.ndarray, entries: list[Entry]) -> tuple[int, float]:
+    if not entries:
+        return -1, -1.0
+    mat = np.stack([e.emb for e in entries])   # (N, d), rows unit-norm
+    sims = mat @ emb                            # cosine (emb unit-norm)
+    i = int(np.argmax(sims))
+    return i, float(sims[i])
+
+
+# --------------------------------------------------------------------------- #
+# Arms
+# --------------------------------------------------------------------------- #
+class NoMemory:
+    name = "no_memory"
+
+    def draft(self, query, functions, user_id, embedder) -> str:
+        return schema_draft(functions)
+
+    def observe(self, *a, **k):
+        return None
+
+
+class StaticGlobal:
+    """ToolSpec-style frozen global datastore."""
+    name = "static_global"
+
+    def __init__(self):
+        self.entries: list[Entry] = []
+        self.frozen = False
+
+    def freeze(self):
+        self.frozen = True
+
+    def draft(self, query, functions, user_id, embedder) -> str:
+        emb = embedder.embed(query)
+        i, _ = _best_match(emb, self.entries)
+        if i < 0:
+            return schema_draft(functions)
+        return self.entries[i].call
+
+    def observe(self, query, functions, user_id, name, args, embedder) -> None:
+        if self.frozen:
+            return
+        self.entries.append(Entry(embedder.embed(query),
+                                  canonical_call_str(name, args)))
+
+
+class PersonalMemory:
+    """Ours: per-user growing store + eviction + personalization."""
+    name = "personal_memory"
+
+    def __init__(self, capacity: int = 32, eviction: str = "lru",
+                 sim_threshold: float = 0.35):
+        self.capacity = capacity
+        self.eviction = eviction              # "lru" | "lfu"
+        self.sim_threshold = sim_threshold
+        # per-user ordered store (insertion/most-recent-use order for LRU)
+        self.stores: dict[str, "OrderedDict[int, Entry]"] = {}
+        self.shared: list[Entry] = []         # cold-start backoff
+        self._next_id = 0
+
+    def _store(self, user_id) -> "OrderedDict[int, Entry]":
+        return self.stores.setdefault(user_id, OrderedDict())
+
+    def draft(self, query, functions, user_id, embedder) -> str:
+        emb = embedder.embed(query)
+        store = self._store(user_id)
+        entries = list(store.values())
+        i, sim = _best_match(emb, entries)
+        if i >= 0 and sim >= self.sim_threshold:
+            key = list(store.keys())[i]
+            entry = store[key]
+            if self.eviction == "lru":        # mark most-recently-used
+                store.move_to_end(key)
+            entry.freq += 1
+            return entry.call
+        # personal store empty / too dissimilar -> shared backoff
+        j, sj = _best_match(emb, self.shared)
+        if j >= 0 and sj >= self.sim_threshold:
+            return self.shared[j].call
+        return schema_draft(functions)
+
+    def observe(self, query, functions, user_id, name, args, embedder) -> None:
+        emb = embedder.embed(query)
+        call = canonical_call_str(name, args)
+        store = self._store(user_id)
+        eid = self._next_id
+        self._next_id += 1
+        store[eid] = Entry(emb, call)
+        store.move_to_end(eid)
+        self._evict(store)
+
+    def seed_shared(self, query, name, args, embedder) -> None:
+        self.shared.append(Entry(embedder.embed(query),
+                                 canonical_call_str(name, args)))
+
+    def _evict(self, store: "OrderedDict[int, Entry]") -> None:
+        while len(store) > self.capacity:
+            if self.eviction == "lru":
+                store.popitem(last=False)     # drop least-recently-used
+            elif self.eviction == "lfu":
+                k = min(store, key=lambda x: store[x].freq)
+                del store[k]
+            else:
+                store.popitem(last=False)
+
+    def total_entries(self) -> int:
+        return sum(len(s) for s in self.stores.values())
+
+
+# --------------------------------------------------------------------------- #
+# Ablation arms (2x2: personalized? x evicting?) -- see run_ablation.py.
+# These decompose PersonalMemory's gain over StaticGlobal into the contribution
+# of per-user partitioning vs. the contribution of online-growth+eviction.
+# --------------------------------------------------------------------------- #
+class PersonalNoEvict(PersonalMemory):
+    """[+personalization, -eviction]: per-user store that grows online but is
+    never bounded/evicted. Isolates how much of ours' gain is eviction."""
+    name = "personal_noevict"
+
+    def __init__(self, sim_threshold: float = 0.35):
+        # capacity = +inf so _evict() never fires.
+        super().__init__(capacity=10**9, eviction="lru",
+                         sim_threshold=sim_threshold)
+
+
+class GlobalEvict:
+    """[-personalization, +online-growth+eviction]: a single GLOBAL store (not
+    per-user) that keeps ingesting after warmup and LRU-evicts at a total
+    capacity. Isolates how much of ours' gain is per-user partitioning: it
+    differs from PersonalMemory only in that retrieval ignores user id."""
+    name = "global_evict"
+
+    def __init__(self, capacity: int = 1920, sim_threshold: float = 0.35):
+        self.capacity = capacity
+        self.sim_threshold = sim_threshold
+        self.store: "OrderedDict[int, Entry]" = OrderedDict()
+        self._next_id = 0
+
+    def draft(self, query, functions, user_id, embedder) -> str:
+        emb = embedder.embed(query)
+        entries = list(self.store.values())
+        i, sim = _best_match(emb, entries)
+        if i >= 0 and sim >= self.sim_threshold:
+            key = list(self.store.keys())[i]
+            self.store.move_to_end(key)          # LRU touch
+            self.store[key].freq += 1
+            return self.store[key].call
+        return schema_draft(functions)
+
+    def observe(self, query, functions, user_id, name, args, embedder) -> None:
+        emb = embedder.embed(query)
+        eid = self._next_id
+        self._next_id += 1
+        self.store[eid] = Entry(emb, canonical_call_str(name, args))
+        self.store.move_to_end(eid)
+        while len(self.store) > self.capacity:
+            self.store.popitem(last=False)       # drop least-recently-used
+
+    def seed_shared(self, *a, **k):
+        return None
+
+
+# --------------------------------------------------------------------------- #
+# Phase 4.4 — faithful stronger ToolSpec-style baseline
+# --------------------------------------------------------------------------- #
+def _schema_scaffold(functions, name) -> str:
+    """Schema-aware structural draft for a NAMED function: the function's
+    required argument keys in canonical (sorted) order with placeholder values.
+    This is the structurally-valid fallback a schema-aware FSM emits when
+    retrieval is not confident enough to commit a concrete prior call."""
+    fn = next((f for f in functions if f.get("name") == name), None)
+    if fn is None:
+        return schema_draft(functions)
+    params = fn.get("parameters", {}) or {}
+    props = params.get("properties", {}) or {}
+    required = params.get("required", []) or list(props)
+    return canonical_call_str(name, {k: None for k in required})
+
+
+class ToolSpecBaseline:
+    """Faithful ToolSpec reproduction (Xia et al., 2026): a *frozen global*
+    retrieval store (no eviction, no personalization — the ToolSpec regime)
+    with the two ToolSpec mechanisms the simple ``StaticGlobal`` proxy omits:
+
+      1. **Confidence-gated retrieval.** Return the nearest stored call verbatim
+         only while its similarity clears ``sim_lo``; ``StaticGlobal`` instead
+         returns its single nearest neighbour unconditionally, so on a cold /
+         far query it drafts a wholly unrelated call.
+      2. **Schema-aware fallback (FSM surrogate).** On a cold miss, rather than
+         emitting a random far neighbour we emit a *structurally valid* draft
+         for the nearest neighbour's function (its required-arg scaffold in
+         canonical order) — the acceptance a schema-constrained decoder
+         guarantees on the call's structural tokens even without a value hit.
+         This makes the arm **strictly at least as strong as ``StaticGlobal``**:
+         identical on confident hits, better on cold misses.
+
+    ToolSpec has no public code, so the FSM is approximated by this schema-aware
+    scaffold. We also tested a
+    ``k``-NN summed-similarity vote on the target *function* (retrieval-augmented
+    denoising); it *degraded* MAT on these traces because the highly skewed
+    telecom workload (one diagnostic call dominates) lets the majority function
+    override correct top-1 picks — reported honestly in the write-up, and NOT
+    used here. Everything else (frozen, global, eviction-free) matches ToolSpec
+    and is deliberately NOT personalized — the property under test.
+    """
+    name = "toolspec"
+
+    def __init__(self, sim_lo: float = 0.30):
+        self.entries: list[Entry] = []
+        self.frozen = False
+        self.sim_lo = sim_lo
+        self._names: list[str] = []          # parallel function name per entry
+
+    def freeze(self):
+        self.frozen = True
+
+    def draft(self, query, functions, user_id, embedder) -> str:
+        if not self.entries:
+            return schema_draft(functions)
+        emb = embedder.embed(query)
+        i, sim = _best_match(emb, self.entries)
+        if sim >= self.sim_lo:
+            return self.entries[i].call              # confident retrieval hit
+        return _schema_scaffold(functions, self._names[i])  # schema-aware miss
+
+    def observe(self, query, functions, user_id, name, args, embedder) -> None:
+        if self.frozen:
+            return
+        self.entries.append(Entry(embedder.embed(query),
+                                  canonical_call_str(name, args)))
+        self._names.append(name)
+
+    def seed_shared(self, *a, **k):
+        return None
+
+
+# --------------------------------------------------------------------------- #
+# SuffixDecoding baseline (Oliaro et al., NeurIPS 2025; arXiv 2411.04975)
+# --------------------------------------------------------------------------- #
+_TOK_RE = re.compile(r"\w+|[^\w\s]")
+_SEP = " "
+
+
+def _tokenize(text: str) -> tuple[str, ...]:
+    """Word/punctuation-level tokens (lowercased). A deliberate approximation of
+    SuffixDecoding's model-BPE tokens: the mechanistic contrast under test is
+    *exact token matching vs. embedding similarity*, which this preserves; the
+    exact subword vocabulary is not what distinguishes the two arms."""
+    return tuple(_TOK_RE.findall(text.lower()))
+
+
+def _suffix_key(tokens: tuple[str, ...]) -> str:
+    """Separator-delimited form so whole-token substring tests never match
+    across partial tokens (every boundary is a _SEP)."""
+    return _SEP + _SEP.join(tokens) + _SEP
+
+
+def _longest_suffix_match(q: tuple[str, ...], stored_key: str,
+                          floor: int) -> int:
+    """Longest k>floor such that the k-token *suffix* of the current query q is
+    a contiguous whole-token substring of a stored sequence (its suffix key).
+    Returns 0 if no suffix longer than `floor` matches. This is SuffixDecoding's
+    'walk the tree to the node matching the context suffix' step, adapted to our
+    per-request query context.
+
+    Uses binary search: the predicate ``q[-k:] is a substring of stored`` is
+    monotonic in k (if the k-token suffix matches, every shorter suffix does),
+    so we find the largest matching k in O(log|q|) containment tests rather than
+    O(|q|) — essential because dialogue-context queries run to hundreds of
+    tokens."""
+    best = 0
+    lo, hi = floor + 1, len(q)
+    while lo <= hi:
+        mid = (lo + hi) // 2
+        cand = _SEP + _SEP.join(q[-mid:]) + _SEP
+        if cand in stored_key:
+            best = mid
+            lo = mid + 1
+        else:
+            hi = mid - 1
+    return best
+
+
+class SuffixDecodingBaseline:
+    """Faithful adaptation of **SuffixDecoding** (Oliaro et al., *SuffixDecoding:
+    Extreme Speculative Decoding for Emerging AI Applications*, NeurIPS 2025
+    Spotlight; arXiv 2411.04975) as a retrieval arm.
+
+    SuffixDecoding keeps a **global suffix tree accumulated from previous
+    requests' token streams** (live/growing across the deployment, so request N
+    benefits from request N-1), matches the **suffix of the current context**
+    against the tree at each step, and speculates the highest-frequency
+    continuation with adaptive length. It is **token-level (exact match),
+    global, and NOT personalized**; the tree is **size-capped** (~10.75 B/token,
+    ~31 days on a 144 GB host), not unbounded.
+
+    We reproduce that regime and swap **exactly one mechanism** vs. our own
+    ``GlobalEvict`` arm: retrieval is **longest-token-suffix match** on the query
+    instead of embedding cosine similarity. Everything else — global live
+    write-back, canonicalization, a size cap, no per-user partitioning — is held
+    identical, so the comparison isolates *retrieval mechanism* (exact token
+    match vs. semantic embedding), not confounds like different data pools or
+    personalization. Frequency and recency break ties in match length, mirroring
+    SuffixDecoding's frequency-ranked tree paths.
+
+    Faithful vs. adapted:
+      - Faithful: global, live-growing, size-capped, non-personalized store;
+        exact token-suffix matching; frequency-ranked selection.
+      - Adapted: we match the *query* token context (which selects a tool call
+        in our one-call-per-request setting) rather than a running generation,
+        and our token-LCP acceptance already truncates the speculated call at the
+        first mismatch, subsuming SuffixDecoding's adaptive speculation length.
+        Tokenization is word/punct level, not the model BPE (approximation).
+    """
+    name = "suffixdecoding"
+
+    def __init__(self, capacity: int = 1920, min_match: int = 1):
+        # capacity matches GlobalEvict's total footprint (U*C = 40*48) for a
+        # same-size comparison; min_match=1 lets any shared suffix token yield a
+        # (frequency-ranked) speculation, as SuffixDecoding's tree always does.
+        self.capacity = capacity
+        self.min_match = min_match
+        # eid -> [q_tokens, suffix_key, call, freq]
+        self.store: "OrderedDict[int, list]" = OrderedDict()
+        self._next_id = 0
+
+    def draft(self, query, functions, user_id, embedder) -> str:
+        q = _tokenize(query)
+        if not q:
+            return schema_draft(functions)
+        floor = self.min_match - 1
+        best_key, best_eid = None, None    # rank by (match_len, freq), then recency
+        for eid, rec in self.store.items():
+            m = _longest_suffix_match(q, rec[1], floor)   # true length (fixed floor)
+            if m < self.min_match:
+                continue
+            key = (m, rec[3])
+            if best_key is None or key >= best_key:        # >= => later (more recent) wins ties
+                best_key, best_eid = key, eid
+        if best_eid is None:
+            return schema_draft(functions)
+        rec = self.store[best_eid]
+        rec[3] += 1                       # frequency reinforcement
+        self.store.move_to_end(best_eid)  # recency touch
+        return rec[2]
+
+    def observe(self, query, functions, user_id, name, args, embedder) -> None:
+        q = _tokenize(query)
+        eid = self._next_id
+        self._next_id += 1
+        self.store[eid] = [q, _suffix_key(q), canonical_call_str(name, args), 1]
+        self.store.move_to_end(eid)
+        while len(self.store) > self.capacity:
+            self.store.popitem(last=False)   # drop oldest (size cap)
+
+    def seed_shared(self, *a, **k):
+        return None
+
+    def total_entries(self) -> int:
+        return len(self.store)

harness/metrics.py

@@ -0,0 +1,84 @@
+"""Speculative-decoding acceptance metric.
+
+Given a *draft* tool call string and the model's genuine *target* tool call
+string, both serialized in the same canonical form, we tokenize each with the
+served model's HF tokenizer and compute the token-level longest-common-prefix
+(LCP) accept length -- exactly the quantity a speculative decoder would accept
+when it proposes the draft and the target model verifies it greedily.
+
+We report:
+  - accept_length : # tokens of the draft accepted before the first mismatch.
+  - target_len    : # tokens in the target (upper bound on accept_length).
+  - accepted_frac : accept_length / target_len (in [0, 1]).
+
+This is a *proxy* for true spec-decoding: we compare canonicalized tool-call
+strings rather than the raw harmony token stream. Because every arm and every
+session is scored through the identical canonicalizer + tokenizer, cross-arm
+and cross-session comparisons (the actual claims of the paper) are valid; only
+the absolute MAT should be read as a proxy. This caveat is documented in the
+paper's limitations section.
+"""
+from __future__ import annotations
+
+import json
+from functools import lru_cache
+from typing import Any
+
+
+def canonical_call_str(name: str, arguments: dict[str, Any]) -> str:
+    """Serialize a tool call to a deterministic canonical string.
+
+    Keys are sorted so that the draft and target are compared on identical
+    surface form; argument values are JSON-encoded compactly. Both drafts and
+    targets pass through this same function, so LCP is well defined and fair.
+    """
+    try:
+        args = {k: arguments[k] for k in sorted(arguments.keys())}
+    except Exception:
+        args = arguments
+    return json.dumps({"name": name, "arguments": args}, sort_keys=True,
+                      ensure_ascii=False, separators=(", ", ": "))
+
+
+_TOKENIZER = None
+
+
+def get_tokenizer(model_path: str):
+    global _TOKENIZER
+    if _TOKENIZER is None:
+        from transformers import AutoTokenizer
+        try:
+            _TOKENIZER = AutoTokenizer.from_pretrained(model_path)
+        except (ValueError, OSError, KeyError):
+            # some models (e.g. Nemotron-H) ship a custom tokenizer class
+            _TOKENIZER = AutoTokenizer.from_pretrained(
+                model_path, trust_remote_code=True)
+    return _TOKENIZER
+
+
+@lru_cache(maxsize=200_000)
+def _encode(text: str) -> tuple[int, ...]:
+    assert _TOKENIZER is not None, "call get_tokenizer(model_path) first"
+    return tuple(_TOKENIZER.encode(text, add_special_tokens=False))
+
+
+def accept_length(draft: str, target: str) -> tuple[int, int]:
+    """Return (accept_length, target_len) in tokens via token-level LCP."""
+    dt = _encode(draft)
+    tt = _encode(target)
+    n = 0
+    for a, b in zip(dt, tt):
+        if a != b:
+            break
+        n += 1
+    return n, len(tt)
+
+
+def score(draft: str, target: str) -> dict[str, float]:
+    acc, tlen = accept_length(draft, target)
+    return {
+        "accept_length": acc,
+        "target_len": tlen,
+        "accepted_frac": (acc / tlen) if tlen else 0.0,
+        "exact": bool(tlen and acc == tlen),
+    }

harness/phase4_freshness_fig.py

@@ -0,0 +1,67 @@
+"""Render the freshness-over-time figure from
+results/phase4_freshness_curve.json -> figures/freshness_curve.pdf."""
+import json
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def main():
+    r = json.loads((ROOT / "results" / "phase4_freshness_curve.json").read_text())
+    c = r["per_session_MAT"]
+    S = list(range(12))
+
+    def y(arm):
+        return [c[arm][str(s)] for s in S]
+
+    fig, ax = plt.subplots(1, 2, figsize=(9.4, 3.3),
+                           gridspec_kw={"width_ratios": [1.5, 1]})
+
+    # left: per-session MAT
+    ax[0].axvspan(-0.4, 0.4, color="0.92", zorder=0)
+    ax[0].text(0.0, 4.6, "warmup", ha="center", va="bottom", fontsize=7.5,
+               color="0.4")
+    ax[0].plot(S, y("personal_memory"), "-o", color="#0E7C86", lw=2.2, ms=4,
+               label="SpecMem (ours, live)")
+    ax[0].plot(S, y("static_global"), "-s", color="#B9820B", lw=1.8, ms=3.5,
+               label="static datastore (frozen)")
+    ax[0].plot(S, y("no_memory"), "-^", color="#9198A1", lw=1.4, ms=3,
+               label="Vanilla AR (no memory)")
+    # shade the widening gap after warmup
+    ax[0].fill_between(S[1:], y("static_global")[1:], y("personal_memory")[1:],
+                       color="#0E7C86", alpha=0.10, zorder=0)
+    ax[0].set_xlabel("session index")
+    ax[0].set_ylabel("mean accepted tokens (MAT)")
+    ax[0].set_xticks(S)
+    ax[0].set_ylim(0, 34)
+    ax[0].legend(fontsize=7.6, loc="lower right", framealpha=0.9)
+    ax[0].set_title("Freshness over time ($\\tau^2$-bench, 3 domains)",
+                    fontsize=9.5)
+    ax[0].grid(True, alpha=0.25)
+
+    # right: growing relative advantage
+    rel = r["personal_over_static_pct_by_session"]
+    xs = sorted(int(k) for k in rel)
+    ax[1].plot(xs, [rel[str(k)] for k in xs], "-o", color="#0E7C86", lw=2.0,
+               ms=4)
+    ax[1].set_xlabel("session index")
+    ax[1].set_ylabel("SpecMem over frozen (%)")
+    ax[1].set_xticks(xs)
+    ax[1].set_ylim(0, 40)
+    ax[1].set_title("Advantage grows as\nnovel calls accumulate", fontsize=9.5)
+    ax[1].grid(True, alpha=0.25)
+
+    fig.tight_layout()
+    out = ROOT / "figures" / "freshness_curve.pdf"
+    out.parent.mkdir(exist_ok=True)
+    fig.savefig(out, bbox_inches="tight")
+    fig.savefig(out.with_suffix(".png"), dpi=200, bbox_inches="tight")
+    print("wrote", out)
+
+
+if __name__ == "__main__":
+    main()

harness/phase4_maintable.py

@@ -0,0 +1,277 @@
+"""Phase 4 MAIN TABLE (Part A) — 4 arms x 4 real datasets, full metrics.
+
+Rows/arms (reuse existing, no reimplementation):
+  no_memory      = Vanilla AR (speedup reference)
+  toolspec       = faithful ToolSpec (Phase 4.4: confidence-gated retrieval + FSM)
+  static_global  = ToolSpec + static memory (simple frozen proxy)
+  personal_memory= SpecMem (ours: live, evicting, per-user)
+
+Columns/datasets (all REAL): API-Bank, ToolAlpaca, BFCLv4, ToolBench.
+
+Per (arm, dataset) cell we record:
+  1. MAT (mean accepted tokens)          -- replay
+  2. tokens/s (real decode throughput)   -- wall-clock timing
+  3. e2e wall-clock speedup vs Vanilla AR -- wall-clock timing (p50)
+  4. retrieval / write-back overhead ms  -- timed embed+lookup / embed+insert
+  5. memory size (entries) at run end     -- replay
+  6. cold-start vs long-term MAT curve    -- per-session MAT (replay)
+
+Targets are the served gpt-oss-120b's greedy calls (real, cached per dataset+seed).
+Run from the repo root:  python -m harness.phase4_maintable --url http://localhost:30000/v1
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import time
+from collections import defaultdict
+from pathlib import Path
+
+import requests
+
+from . import metrics
+from .client import ToolClient
+from .data import (load_apibank, load_bfcl, load_toolalpaca, load_toolbench)
+from .memory import (Embedder, NoMemory, PersonalMemory, StaticGlobal,
+                     ToolSpecBaseline)
+from .run_accept import _parse_target, generate_targets
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+# Tokenizer for the token-LCP accept metric: HF hub id by default;
+# override with a local snapshot path if running offline.
+MODEL_PATH = os.environ.get("SPECMEM_TOKENIZER", "openai/gpt-oss-120b")
+DATASETS = {"apibank": load_apibank, "toolalpaca": load_toolalpaca,
+            "bfcl": load_bfcl, "toolbench": load_toolbench}
+ARMS = ["no_memory", "toolspec", "static_global", "personal_memory"]
+
+
+def _make_arms(cap):
+    return [NoMemory(), ToolSpecBaseline(), StaticGlobal(),
+            PersonalMemory(capacity=cap, eviction="lru")]
+
+
+def _mem_size(arm):
+    if isinstance(arm, PersonalMemory):
+        return sum(len(s) for s in arm.stores.values())
+    if isinstance(arm, (StaticGlobal, ToolSpecBaseline)):
+        return len(arm.entries)
+    return 0
+
+
+def _replay(instances, targets, emb, cap):
+    """3-arm+baseline replay; return per-arm MAT, per-session MAT, mem size."""
+    arms = _make_arms(cap)
+    agg = {a.name: defaultdict(list) for a in arms}
+    cur = -1
+    for ins in instances:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur:
+            cur = ins.session
+            if cur == 1:
+                for a in arms:
+                    if hasattr(a, "freeze"):
+                        a.freeze()
+        for a in arms:
+            agg[a.name][ins.session].append(metrics.score(
+                a.draft(ins.query, ins.functions, ins.user_id, emb), tgt))
+        cn, ca = _parse_target(tgt)
+        for a in arms[1:]:
+            a.observe(ins.query, ins.functions, ins.user_id, cn, ca, emb)
+            if isinstance(a, PersonalMemory) and ins.session == 0:
+                a.seed_shared(ins.query, cn, ca, emb)
+    out = {}
+    for a in arms:
+        v = agg[a.name]
+        post = [x for s, xs in v.items() if s > 0 for x in xs]
+        by_sess = {s: round(sum(x["accept_length"] for x in xs) / len(xs), 3)
+                   for s, xs in sorted(v.items()) if xs}
+        out[a.name] = {
+            "MAT": round(sum(x["accept_length"] for x in post) /
+                         max(1, len(post)), 3),
+            "accepted_frac": round(sum(x["accepted_frac"] for x in post) /
+                                   max(1, len(post)), 3),
+            "by_session": by_sess, "mem_entries": _mem_size(a)}
+    return out
+
+
+def _timed_gen(chat, prompt, ntok):
+    t0 = time.perf_counter()
+    r = requests.post(chat, json={"model": "gpt-oss-120b", "temperature": 0.0,
+                                  "max_tokens": ntok, "ignore_eos": True,
+                                  "messages": [{"role": "user",
+                                                "content": prompt}]}, timeout=180)
+    dt = (time.perf_counter() - t0) * 1000
+    r.raise_for_status()
+    return dt
+
+
+def _wallclock(instances, targets, emb, cap, chat, sample, seed):
+    """Real spec-decode wall-clock per arm on a sample: p50/p95 + speedup +
+    tokens/s. Rebuild arm accept-lengths at each sampled point via replay."""
+    arms = _make_arms(cap)
+    pts, cur = [], -1
+    for ins in instances:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur:
+            cur = ins.session
+            if cur == 1:
+                for a in arms:
+                    if hasattr(a, "freeze"):
+                        a.freeze()
+        if ins.session > 0:
+            row = {"query": ins.query, "T": metrics.accept_length(tgt, tgt)[1]}
+            for a in arms:
+                row[a.name] = metrics.accept_length(
+                    a.draft(ins.query, ins.functions, ins.user_id, emb), tgt)[0]
+            pts.append(row)
+        cn, ca = _parse_target(tgt)
+        for a in arms[1:]:
+            a.observe(ins.query, ins.functions, ins.user_id, cn, ca, emb)
+            if isinstance(a, PersonalMemory) and ins.session == 0:
+                a.seed_shared(ins.query, cn, ca, emb)
+    rng = random.Random(seed)
+    rng.shuffle(pts)
+    pts = [p for p in pts if p["T"] >= 2][:sample]
+    verify = sorted(_timed_gen(chat, pts[i]["query"][:1500], 1)
+                    for i in range(min(12, len(pts))))[6 // 2 or 0]
+    lat = defaultdict(list)
+    toks = defaultdict(list)
+    for row in pts:
+        prompt, T = row["query"][:1500], max(1, row["T"])
+        need = {T} | {max(1, T - row[a]) for a in ARMS}
+        tc = {n: _timed_gen(chat, prompt, n) for n in need}
+        lat["baseline"].append(tc[T])
+        for a in ARMS:
+            ms = verify + tc[max(1, T - row[a])]
+            lat[a].append(ms)
+            toks[a].append(T / (ms / 1000.0))      # effective tokens/s
+    def p(xs, q):
+        xs = sorted(xs)
+        return xs[min(len(xs) - 1, int(q * len(xs)))]
+    base_p50 = p(lat["baseline"], 0.5)
+    out = {}
+    for a in ARMS:
+        out[a] = {"p50_ms": round(p(lat[a], 0.5), 1),
+                  "p95_ms": round(p(lat[a], 0.95), 1),
+                  "tokens_per_s": round(sum(toks[a]) / len(toks[a]), 1),
+                  "speedup_vs_vanilla": round(
+                      p(lat["no_memory"], 0.5) / p(lat[a], 0.5), 3)}
+    out["_baseline_p50_ms"] = round(base_p50, 1)
+    return out
+
+
+def _overhead(emb, chat):
+    """Real retrieval (embed+NN over 48-entry store) and write-back (embed+
+    insert) latency, ms/query, measured separately."""
+    from .memory import Entry, _best_match
+    import numpy as np
+    store = [Entry(emb.embed(f"seed query {i}"), "x") for i in range(48)]
+    rlat, wlat = [], []
+    for i in range(150):
+        q = f"overhead probe query number {i} with args {i*7}"
+        t0 = time.perf_counter(); e = emb.embed(q); _best_match(e, store)
+        rlat.append((time.perf_counter() - t0) * 1000)
+        t0 = time.perf_counter(); e2 = emb.embed(q + " wb"); store.append(Entry(e2, "y"))
+        wlat.append((time.perf_counter() - t0) * 1000)
+    return {"retrieval_ms_mean": round(sum(rlat) / len(rlat), 2),
+            "writeback_ms_mean": round(sum(wlat) / len(wlat), 2)}
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--url", default="http://localhost:30000/v1")
+    p.add_argument("--model", default="gpt-oss-120b")
+    p.add_argument("--datasets", nargs="+", default=list(DATASETS))
+    p.add_argument("--seeds", nargs="+", type=int, default=[0, 1, 2])
+    p.add_argument("--wallclock-sample", type=int, default=80)
+    p.add_argument("--tasks-per-user", type=int, default=10)
+    p.add_argument("--capacity", type=int, default=48)
+    p.add_argument("--workers", type=int, default=16)
+    args = p.parse_args()
+    chat = args.url.rstrip("/") + "/chat/completions"
+    metrics.get_tokenizer(MODEL_PATH)
+    emb = Embedder()
+    client = ToolClient(url=args.url, model=args.model)
+    assert client.ping(), f"served model not reachable at {args.url}"
+    overhead = _overhead(emb, chat)
+
+    table = {}
+    for ds in args.datasets:
+        tasks = DATASETS[ds]()
+        n_users = min(40, len(tasks) // args.tasks_per_user)
+        cfg = {"n_users": n_users, "tasks_per_user": args.tasks_per_user,
+               "pool": len(tasks)}
+        print(f"\n=== {ds}: {len(tasks)} tasks -> {n_users} users x "
+              f"{args.tasks_per_user} ===", flush=True)
+        per_seed = defaultdict(dict)
+        by_session_acc = defaultdict(lambda: defaultdict(list))
+        for sd in args.seeds:
+            inst = build_users(tasks, n_users=n_users,
+                               tasks_per_user=args.tasks_per_user, n_sessions=12,
+                               queries_per_session=6, seed=sd)
+            inst.sort(key=lambda x: (x.session, x.user_id))
+            cache_f = RESULTS / f"phase4_mt_targets_{ds}_seed{sd}.json"
+            cache = json.loads(cache_f.read_text()) if cache_f.exists() else {}
+            fmap = {i.query: i.functions for i in inst}
+            miss = [type("S", (), {"query": q, "functions": fmap[q]})()
+                    for q in {i.query for i in inst} if q not in cache]
+            if miss:
+                print(f"  [seed {sd}] generating {len(miss)} targets ...", flush=True)
+                cache.update(generate_targets(client, miss, workers=args.workers))
+                cache_f.write_text(json.dumps(cache))
+            res = _replay(inst, cache, emb, args.capacity)
+            for a in ARMS:
+                per_seed[a][sd] = res[a]["MAT"]
+                for s, m in res[a]["by_session"].items():
+                    by_session_acc[a][s].append(m)
+            if sd == args.seeds[0]:
+                mem = {a: res[a]["mem_entries"] for a in ARMS}
+                wc = _wallclock(inst, cache, emb, args.capacity, chat,
+                                args.wallclock_sample, sd)
+            print(f"  [seed {sd}] MAT: " +
+                  " ".join(f"{a}={res[a]['MAT']}" for a in ARMS), flush=True)
+        import statistics as st
+        cell = {}
+        for a in ARMS:
+            mats = [per_seed[a][sd] for sd in args.seeds]
+            cell[a] = {
+                "MAT_mean": round(st.mean(mats), 3),
+                "MAT_std": round(st.pstdev(mats), 3),
+                "tokens_per_s": wc[a]["tokens_per_s"],
+                "speedup_vs_vanilla": wc[a]["speedup_vs_vanilla"],
+                "wallclock_p50_ms": wc[a]["p50_ms"],
+                "wallclock_p95_ms": wc[a]["p95_ms"],
+                "mem_entries": mem[a],
+                "by_session_MAT": {s: round(sum(v) / len(v), 3)
+                                   for s, v in sorted(by_session_acc[a].items())},
+            }
+        table[ds] = {"config": cfg, "baseline_p50_ms": wc["_baseline_p50_ms"],
+                     "cells": cell}
+
+    out = {"overhead": overhead, "arms": ARMS,
+           "datasets": list(args.datasets), "table": table,
+           "note": ("wall-clock via faithful external spec-decode loop, real "
+                    "sglang timers (ignore_eos), NOT engine-integrated; targets "
+                    "= served gpt-oss-120b greedy; BFCL is v4 (superset of the "
+                    "v2 ToolSpec used); ToolBench from OpenBMB Drive.")}
+    (RESULTS / "phase4_main_table.json").write_text(json.dumps(out, indent=2))
+    print("\n=== SPEEDUP vs Vanilla AR (p50) ===")
+    hdr = "arm".ljust(16) + "".join(d[:9].ljust(11) for d in args.datasets)
+    print(hdr)
+    for a in ARMS:
+        row = a.ljust(16) + "".join(
+            f"{table[d]['cells'][a]['speedup_vs_vanilla']}x".ljust(11)
+            for d in args.datasets)
+        print(row)
+
+
+if __name__ == "__main__":
+    main()

harness/phase4_overlap.py

@@ -0,0 +1,141 @@
+"""Phase 4.3 — shared-user-task personalization overlap sweep.
+
+Stress-tests the personalization claim (per-user memory vs. a live global/shared
+store) as users' task sets *overlap*. Workload: every user has ``tasks_per_user``
+signature tasks with a FIXED per-user argument realization (user A always NYC,
+user B always Boston), consistent across the user's sessions. A fraction
+``overlap_frac`` of those tasks come from ONE shared template pool that every
+user reuses (same templates, different per-user args); the rest are private and
+disjoint. As overlap rises, a global store sees many users' different calls for
+the same template — does per-user memory still win, or does a shared store catch
+up?
+
+Two arms, both live + evicting (only partitioning differs), so the contrast is
+personalization alone:
+  - global_evict  : one live global store (no per-user view)
+  - personal_memory (ours): per-user live evicting store
+
+Targets are the served model's greedy calls (real, generated once per unique
+query on the migrated node and cached per overlap level). Run from ``code/``:
+    python -m harness.phase4_overlap --url http://localhost:30000/v1
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .client import ToolClient
+from .data import load_bfcl
+from .memory import Embedder, GlobalEvict, PersonalMemory
+from .run_accept import _parse_target, generate_targets
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+# Tokenizer for the token-LCP accept metric: HF hub id by default;
+# override with a local snapshot path if running offline.
+MODEL_PATH = os.environ.get("SPECMEM_TOKENIZER", "openai/gpt-oss-120b")
+
+
+class Slot:
+    """generate_targets expects objects with .query and .functions; the
+    functions registry is identical across BFCL tasks' first-tool schema, so we
+    attach it lazily below."""
+    __slots__ = ("query", "functions")
+
+    def __init__(self, query, functions=None):
+        self.query = query
+        self.functions = functions
+
+
+def _replay(instances, targets, embedder, capacity):
+    arms = [GlobalEvict(capacity=40 * capacity),
+            PersonalMemory(capacity=capacity, eviction="lru")]
+    agg = {a.name: defaultdict(list) for a in arms}
+    for ins in instances:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        for a in arms:
+            agg[a.name][ins.session].append(metrics.score(
+                a.draft(ins.query, ins.functions, ins.user_id, embedder), tgt))
+        cn, ca = _parse_target(tgt)
+        for a in arms:
+            a.observe(ins.query, ins.functions, ins.user_id, cn, ca, embedder)
+    out = {}
+    for n, v in agg.items():
+        post = [x for s, xs in v.items() if s > 0 for x in xs]
+        out[n] = round(sum(x["accept_length"] for x in post) /
+                       max(1, len(post)), 3)
+    return out
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--url", default="http://localhost:30000/v1")
+    p.add_argument("--model", default="gpt-oss-120b")
+    p.add_argument("--workers", type=int, default=16)
+    p.add_argument("--overlaps", nargs="+", type=float,
+                   default=[0.0, 0.25, 0.5, 0.75, 1.0])
+    p.add_argument("--seeds", nargs="+", type=int, default=[0, 1, 2])
+    p.add_argument("--capacity", type=int, default=48)
+    args = p.parse_args()
+
+    metrics.get_tokenizer(MODEL_PATH)
+    tasks = load_bfcl()
+    embedder = Embedder()
+    client = ToolClient(url=args.url, model=args.model)
+    if not client.ping():
+        raise SystemExit(f"served model not reachable at {args.url}")
+
+    sweep = {}
+    for ov in args.overlaps:
+        per_seed = {"global_evict": [], "personal_memory": []}
+        for sd in args.seeds:
+            instances = build_users(
+                tasks, n_users=40, tasks_per_user=15, n_sessions=12,
+                queries_per_session=6, seed=sd, overlap_frac=ov,
+                user_consistent=True)
+            # attach the (shared) functions registry to Slots for target gen
+            fmap = {ins.query: ins.functions for ins in instances}
+            cache_f = RESULTS / f"phase4_overlap_targets_ov{ov}_seed{sd}.json"
+            cache = json.loads(cache_f.read_text()) if cache_f.exists() else {}
+            miss = [Slot(q, fmap[q]) for q in {i.query for i in instances}
+                    if q not in cache]
+            if miss:
+                cache.update(generate_targets(client, miss, workers=args.workers))
+                cache_f.write_text(json.dumps(cache))
+            res = _replay(instances, cache, embedder, args.capacity)
+            per_seed["global_evict"].append(res["global_evict"])
+            per_seed["personal_memory"].append(res["personal_memory"])
+            print(f"ov={ov} seed={sd}: {res}", flush=True)
+        import statistics as st
+        g = st.mean(per_seed["global_evict"])
+        pm = st.mean(per_seed["personal_memory"])
+        sweep[ov] = {
+            "global_evict": round(g, 3),
+            "personal_memory": round(pm, 3),
+            "personal_advantage_pct": round(100 * (pm - g) / g, 1),
+            "per_seed": per_seed}
+        print(f"== ov={ov}: global {g:.3f} | personal {pm:.3f} | "
+              f"adv {sweep[ov]['personal_advantage_pct']}% ==", flush=True)
+
+    out = {"config": {"users": 40, "tasks_per_user": 15, "sessions": 12,
+                      "capacity": args.capacity, "seeds": args.seeds,
+                      "user_consistent_args": True,
+                      "arms": "global_evict (live shared) vs personal_memory (ours)"},
+           "sweep": sweep}
+    (RESULTS / "phase4_personalization_overlap.json").write_text(
+        json.dumps(out, indent=2))
+    print(json.dumps({ov: {"global": s["global_evict"],
+                           "personal": s["personal_memory"],
+                           "adv%": s["personal_advantage_pct"]}
+                      for ov, s in sweep.items()}, indent=1))
+
+
+if __name__ == "__main__":
+    main()

harness/phase4_partb.py

@@ -0,0 +1,103 @@
+"""Phase 4 MAIN TABLE Part B — freshness-over-time curve on tau2-bench.
+
+Packages the existing multi-session tau2-bench longitudinal result as an explicit
+freshness curve: per-session MAT for each arm (no_memory / toolspec /
+static_global / personal_memory), x = session index, one line per arm. This is
+the plot that shows WHY persistence is necessary: the fixed store degrades in
+relative terms as novel per-user calls accumulate while the live store holds.
+
+CPU-only: replays the frozen tau2 decision points against their cached on-policy
+trace targets (no server calls). Emits results/phase4_freshness_curve.json.
+
+Run from the repo root:  python -m harness.phase4_partb
+"""
+from __future__ import annotations
+
+import json
+import os
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .data import Task
+from .memory import (Embedder, NoMemory, PersonalMemory, StaticGlobal,
+                     ToolSpecBaseline)
+from .run_accept import _parse_target
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+# Tokenizer for the token-LCP accept metric: HF hub id by default;
+# override with a local snapshot path if running offline.
+MODEL_PATH = os.environ.get("SPECMEM_TOKENIZER", "openai/gpt-oss-120b")
+DOMAINS = ("airline", "retail", "telecom")
+ARMS = ["no_memory", "toolspec", "static_global", "personal_memory"]
+
+
+def main():
+    metrics.get_tokenizer(MODEL_PATH)
+    dp = [json.loads(l) for l in
+          (RESULTS / "tau2_live_decision_points.jsonl").read_text().splitlines()]
+    tools = {d: json.loads((ROOT / "data" / "tau2" /
+                            f"tools_{d}.json").read_text()) for d in DOMAINS}
+    tasks = [Task(id=r["id"], query=r["query"], functions=tools[r["domain"]],
+                  origin_id=r["id"]) for r in dp]
+    targets = {r["query"]: r["target"] for r in dp}
+    emb = Embedder()
+
+    per_seed = {a: defaultdict(list) for a in ARMS}   # arm -> session -> [seed MAT]
+    for sd in (0, 1, 2):
+        inst = build_users(tasks, n_users=40, tasks_per_user=15, n_sessions=12,
+                           queries_per_session=6, seed=sd, perturb_prob=0.0)
+        inst.sort(key=lambda x: (x.session, x.user_id))
+        arms = [NoMemory(), ToolSpecBaseline(), StaticGlobal(),
+                PersonalMemory(capacity=48, eviction="lru")]
+        agg = {a.name: defaultdict(list) for a in arms}
+        cur = -1
+        for ins in inst:
+            tgt = targets.get(ins.query)
+            if tgt is None:
+                continue
+            if ins.session != cur:
+                cur = ins.session
+                if cur == 1:
+                    for a in arms:
+                        if hasattr(a, "freeze"):
+                            a.freeze()
+            for a in arms:
+                agg[a.name][ins.session].append(metrics.score(
+                    a.draft(ins.query, ins.functions, ins.user_id, emb), tgt))
+            cn, ca = _parse_target(tgt)
+            for a in arms[1:]:
+                a.observe(ins.query, ins.functions, ins.user_id, cn, ca, emb)
+                if isinstance(a, PersonalMemory) and ins.session == 0:
+                    a.seed_shared(ins.query, cn, ca, emb)
+        for a in arms:
+            for s, xs in agg[a.name].items():
+                per_seed[a.name][s].append(
+                    sum(x["accept_length"] for x in xs) / len(xs))
+        print(f"seed {sd} done", flush=True)
+
+    curve = {a: {str(s): round(sum(v) / len(v), 3)
+                 for s, v in sorted(per_seed[a].items())} for a in ARMS}
+    # relative freshness: personal advantage over static per session (post-warmup)
+    rel = {}
+    for s in sorted(per_seed["personal_memory"]):
+        if s == 0:
+            continue
+        pm = sum(per_seed["personal_memory"][s]) / len(per_seed["personal_memory"][s])
+        sg = sum(per_seed["static_global"][s]) / len(per_seed["static_global"][s])
+        rel[str(s)] = round(100 * (pm - sg) / sg, 1)
+    out = {"arms": ARMS, "per_session_MAT": curve,
+           "personal_over_static_pct_by_session": rel,
+           "note": ("tau2-bench 3-domain frozen decision points, cached "
+                    "on-policy trace targets, 3 seeds; session 0 = warmup. "
+                    "Fixed stores (toolspec/static) plateau/degrade while "
+                    "personal (live) holds -> the freshness curve.")}
+    (RESULTS / "phase4_freshness_curve.json").write_text(json.dumps(out, indent=2))
+    print(json.dumps({"per_session_MAT": curve,
+                      "personal_over_static_pct_by_session": rel}, indent=1))
+
+
+if __name__ == "__main__":
+    main()

harness/phase4_suffixdecoding.py

@@ -0,0 +1,139 @@
+"""Phase 4 — SuffixDecoding baseline validation on tau2-bench (cheap CPU check).
+
+Adds a faithful SuffixDecoding arm (Oliaro et al., NeurIPS 2025; arXiv 2411.04975)
+to the acceptance harness and compares it against the arms that isolate the
+question it answers:
+
+  no_memory      frozen floor (schema draft)
+  toolspec       faithful ToolSpec (frozen, global, confidence-gated + FSM)
+  static_global  frozen global proxy
+  global_evict   LIVE, global, EMBEDDING-similarity retrieval  <-- contrast A
+  suffixdecoding LIVE, global, TOKEN-SUFFIX-match retrieval     <-- contrast A
+  personal_memory ours: LIVE, per-user, embedding retrieval     <-- contrast B
+
+Contrast A (global_evict vs suffixdecoding): both live+global+size-capped, they
+differ ONLY in retrieval mechanism -> does embedding similarity add value beyond
+mere liveness, or is token matching enough?
+Contrast B (suffixdecoding/global_evict vs personal_memory): does per-user
+partitioning add value on top of a live store?
+
+CPU-only: replays the frozen tau2 decision points against cached on-policy trace
+targets (no server). Emits results/phase4_suffixdecoding.json.
+
+Run from the repo root:  python -m harness.phase4_suffixdecoding
+"""
+from __future__ import annotations
+
+import json
+import os
+import statistics as st
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .data import Task
+from .memory import (Embedder, GlobalEvict, NoMemory, PersonalMemory,
+                     StaticGlobal, SuffixDecodingBaseline, ToolSpecBaseline)
+from .run_accept import _parse_target
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+# Tokenizer for the token-LCP accept metric: HF hub id by default;
+# override with a local snapshot path if running offline.
+MODEL_PATH = os.environ.get("SPECMEM_TOKENIZER", "openai/gpt-oss-120b")
+DOMAINS = ("airline", "retail", "telecom")
+ARMS = ["no_memory", "toolspec", "static_global", "global_evict",
+        "suffixdecoding", "personal_memory"]
+
+
+def _make_arms():
+    # global arms sized to the same total footprint U*C = 40*48 = 1920.
+    return [NoMemory(), ToolSpecBaseline(), StaticGlobal(),
+            GlobalEvict(capacity=1920),
+            SuffixDecodingBaseline(capacity=1920),
+            PersonalMemory(capacity=48, eviction="lru")]
+
+
+def main():
+    metrics.get_tokenizer(MODEL_PATH)
+    dp = [json.loads(l) for l in
+          (RESULTS / "tau2_live_decision_points.jsonl").read_text().splitlines()]
+    tools = {d: json.loads((ROOT / "data" / "tau2" /
+                            f"tools_{d}.json").read_text()) for d in DOMAINS}
+    tasks = [Task(id=r["id"], query=r["query"], functions=tools[r["domain"]],
+                  origin_id=r["id"]) for r in dp]
+    targets = {r["query"]: r["target"] for r in dp}
+    emb = Embedder()
+
+    per_seed = {a: defaultdict(list) for a in ARMS}   # arm -> session -> [seed MAT]
+    post_seed = {a: [] for a in ARMS}                 # arm -> [seed post-warmup MAT]
+    for sd in (0, 1, 2):
+        inst = build_users(tasks, n_users=40, tasks_per_user=15, n_sessions=12,
+                           queries_per_session=6, seed=sd, perturb_prob=0.0)
+        inst.sort(key=lambda x: (x.session, x.user_id))
+        arms = _make_arms()
+        agg = {a.name: defaultdict(list) for a in arms}
+        cur = -1
+        for ins in inst:
+            tgt = targets.get(ins.query)
+            if tgt is None:
+                continue
+            if ins.session != cur:
+                cur = ins.session
+                if cur == 1:
+                    for a in arms:
+                        if hasattr(a, "freeze"):
+                            a.freeze()
+            for a in arms:
+                agg[a.name][ins.session].append(metrics.score(
+                    a.draft(ins.query, ins.functions, ins.user_id, emb), tgt))
+            cn, ca = _parse_target(tgt)
+            for a in arms[1:]:
+                a.observe(ins.query, ins.functions, ins.user_id, cn, ca, emb)
+                if isinstance(a, PersonalMemory) and ins.session == 0:
+                    a.seed_shared(ins.query, cn, ca, emb)
+        for a in arms:
+            for s, xs in agg[a.name].items():
+                per_seed[a.name][s].append(
+                    sum(x["accept_length"] for x in xs) / len(xs))
+            post = [x for s, xs in agg[a.name].items() if s > 0 for x in xs]
+            post_seed[a.name].append(
+                sum(x["accept_length"] for x in post) / len(post))
+        print(f"seed {sd} done", flush=True)
+
+    curve = {a: {str(s): round(sum(v) / len(v), 3)
+                 for s, v in sorted(per_seed[a].items())} for a in ARMS}
+    postmat = {a: round(sum(v) / len(v), 3) for a, v in post_seed.items()}
+    poststd = {a: round(st.pstdev(v), 3) if len(v) > 1 else 0.0
+               for a, v in post_seed.items()}
+
+    def rel(x, base):
+        return round(100 * (postmat[x] - postmat[base]) / postmat[base], 1)
+
+    contrasts = {
+        "suffixdecoding_over_static_pct": rel("suffixdecoding", "static_global"),
+        "global_evict_over_static_pct": rel("global_evict", "static_global"),
+        "personal_over_static_pct": rel("personal_memory", "static_global"),
+        "embedding_vs_token_gap_pct_of_static": round(
+            rel("global_evict", "static_global")
+            - rel("suffixdecoding", "static_global"), 1),
+        "personal_over_suffixdecoding_pct": rel("personal_memory",
+                                                "suffixdecoding"),
+        "personal_over_global_evict_pct": rel("personal_memory", "global_evict"),
+    }
+    out = {"arms": ARMS, "post_warmup_MAT": postmat, "post_warmup_seed_std": poststd,
+           "per_session_MAT": curve, "contrasts": contrasts,
+           "note": ("tau2-bench 3-domain frozen decision points, cached on-policy "
+                    "trace targets, 3 seeds; session 0 = warmup. global_evict and "
+                    "suffixdecoding are both LIVE + global + size-capped 1920, "
+                    "differing ONLY in retrieval (embedding cosine vs token-suffix "
+                    "match) -> isolates retrieval mechanism. All numbers are real "
+                    "replay outputs; no tuning to a target outcome.")}
+    (RESULTS / "phase4_suffixdecoding.json").write_text(json.dumps(out, indent=2))
+    print(json.dumps({"post_warmup_MAT": postmat, "contrasts": contrasts},
+                     indent=1))
+
+
+if __name__ == "__main__":
+    main()

harness/phase4_suffixdecoding_maintable.py

@@ -0,0 +1,147 @@
+"""Phase 4 — SuffixDecoding across the MAIN TABLE's 4 benchmarks (CPU, cached).
+
+Generalizes the tau2-bench SuffixDecoding check (phase4_suffixdecoding.py) to the
+four standard benchmarks (API-Bank, ToolAlpaca, BFCL, ToolBench) using the SAME
+multi-session workload construction and the SAME cached greedy targets the main
+table already uses (results/phase4_mt_targets_{ds}_seed{sd}.json) — so no server
+and no re-decoding are needed; MAT is directly comparable to phase4_main_table.json.
+
+Arms (post-warmup MAT, 3 seeds):
+  static_global  frozen (ToolSpec regime)
+  global_evict   LIVE global, EMBEDDING-similarity retrieval
+  suffixdecoding LIVE global, TOKEN-SUFFIX-match retrieval (SuffixDecoding)
+  personal_memory ours (LIVE per-user, embedding)
+
+Isolates the same question as the tau2 run across four independent benchmarks:
+does the freshness gain depend on the retrieval mechanism (embedding vs exact
+token match), or only on the store being live?
+
+Run from the repo root:  python -m harness.phase4_suffixdecoding_maintable
+"""
+from __future__ import annotations
+
+import json
+import os
+import statistics as st
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .data import load_apibank, load_bfcl, load_toolalpaca, load_toolbench
+from .memory import (Embedder, GlobalEvict, NoMemory, PersonalMemory,
+                     StaticGlobal, SuffixDecodingBaseline, ToolSpecBaseline)
+from .run_accept import _parse_target
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+# Tokenizer for the token-LCP accept metric: HF hub id by default;
+# override with a local snapshot path if running offline.
+MODEL_PATH = os.environ.get("SPECMEM_TOKENIZER", "openai/gpt-oss-120b")
+DATASETS = {"apibank": load_apibank, "toolalpaca": load_toolalpaca,
+            "bfcl": load_bfcl, "toolbench": load_toolbench}
+# Lean arm set: the embedding-vs-token retrieval contrast (global_evict vs
+# suffixdecoding) is settled on tau2-bench (phase4_suffixdecoding.json); here we
+# corroborate across the 4 standard benchmarks that a LIVE token-match store
+# recovers the freshness gain vs the frozen store, like our embedding store.
+#   no_memory      floor          (schema draft)
+#   static_global  frozen ref     (== toolspec on these workloads)
+#   suffixdecoding LIVE token-match (SuffixDecoding)
+#   personal_memory ours          (LIVE per-user embedding)
+ARMS = ["no_memory", "static_global", "suffixdecoding", "personal_memory"]
+TASKS_PER_USER = 10
+CAP = 48
+
+
+def _make_arms(footprint):
+    return [NoMemory(), StaticGlobal(),
+            SuffixDecodingBaseline(capacity=footprint),
+            PersonalMemory(capacity=CAP, eviction="lru")]
+
+
+def _replay(inst, targets, emb, footprint):
+    arms = _make_arms(footprint)
+    agg = {a.name: defaultdict(list) for a in arms}
+    cur = -1
+    for ins in inst:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur:
+            cur = ins.session
+            if cur == 1:
+                for a in arms:
+                    if hasattr(a, "freeze"):
+                        a.freeze()
+        for a in arms:
+            agg[a.name][ins.session].append(metrics.score(
+                a.draft(ins.query, ins.functions, ins.user_id, emb), tgt))
+        cn, ca = _parse_target(tgt)
+        for a in arms[1:]:
+            a.observe(ins.query, ins.functions, ins.user_id, cn, ca, emb)
+            if isinstance(a, PersonalMemory) and ins.session == 0:
+                a.seed_shared(ins.query, cn, ca, emb)
+    out = {}
+    for a in arms:
+        post = [x for s, xs in agg[a.name].items() if s > 0 for x in xs]
+        out[a.name] = round(sum(x["accept_length"] for x in post) /
+                            max(1, len(post)), 3)
+    return out
+
+
+def main():
+    metrics.get_tokenizer(MODEL_PATH)
+    emb = Embedder()
+    table = {}
+    for ds, loader in DATASETS.items():
+        tasks = loader()
+        n_users = min(40, len(tasks) // TASKS_PER_USER)
+        footprint = n_users * CAP           # same total size for both global arms
+        per_seed = {a: [] for a in ARMS}
+        for sd in (0, 1, 2):
+            cache_f = RESULTS / f"phase4_mt_targets_{ds}_seed{sd}.json"
+            if not cache_f.exists():
+                print(f"  [{ds} seed {sd}] MISSING cache -> skip", flush=True)
+                continue
+            targets = json.loads(cache_f.read_text())
+            inst = build_users(tasks, n_users=n_users,
+                               tasks_per_user=TASKS_PER_USER, n_sessions=12,
+                               queries_per_session=6, seed=sd)
+            inst.sort(key=lambda x: (x.session, x.user_id))
+            res = _replay(inst, targets, emb, footprint)
+            for a in ARMS:
+                per_seed[a].append(res[a])
+            print(f"  [{ds} seed {sd}] " +
+                  " ".join(f"{a}={res[a]}" for a in ARMS), flush=True)
+        cells = {a: {"MAT_mean": round(st.mean(per_seed[a]), 3),
+                     "MAT_std": round(st.pstdev(per_seed[a]), 3)
+                     if len(per_seed[a]) > 1 else 0.0}
+                 for a in ARMS if per_seed[a]}
+        sg = cells["static_global"]["MAT_mean"]
+        for a in ARMS:
+            if a in cells and sg:
+                cells[a]["rel_over_static_pct"] = round(
+                    100 * (cells[a]["MAT_mean"] - sg) / sg, 1)
+        table[ds] = {"n_users": n_users, "footprint": footprint, "cells": cells}
+        print(f"=== {ds} done ===", flush=True)
+
+    out = {"arms": ARMS, "datasets": list(DATASETS), "table": table,
+           "note": ("Post-warmup MAT (sessions 1-11), 3 seeds, cached greedy "
+                    "targets identical to phase4_main_table.json; CPU replay, no "
+                    "server. global_evict and suffixdecoding are both LIVE + "
+                    "global + size-capped to n_users*48, differing ONLY in "
+                    "retrieval (embedding cosine vs exact token-suffix match). "
+                    "Real replay outputs; no tuning to a target outcome.")}
+    (RESULTS / "phase4_suffixdecoding_maintable.json").write_text(
+        json.dumps(out, indent=2))
+    print("\n=== post-warmup MAT (mean over 3 seeds) ===")
+    hdr = "arm".ljust(16) + "".join(d[:9].ljust(11) for d in DATASETS)
+    print(hdr)
+    for a in ARMS:
+        print(a.ljust(16) + "".join(
+            f"{table[d]['cells'][a]['MAT_mean']}".ljust(11)
+            for d in DATASETS if a in table[d]["cells"]))
+
+
+if __name__ == "__main__":
+    main()

harness/phase4_throughput.py

@@ -0,0 +1,117 @@
+"""Phase 4.2 — concurrency / throughput under real load.
+
+Drives CONCURRENT (not sequential) requests against the serving stack and
+measures how aggregate throughput scales with concurrency, per arm, to answer:
+does the memory arm's decode-step advantage hold, shrink, or reverse under
+contention?
+
+Method: reuse the faithful external spec-decode accounting (Phase 4.1) — each
+arm's per-request work is decoding ``T - L`` tokens (accept length ``L`` from
+the real served targets). For each arm and concurrency level ``C`` we fire the
+sampled requests through a ``C``-worker pool against the live sglang server
+(real batched serving, ``ignore_eos`` forces exact token counts) and record
+aggregate tokens/s and per-request latency. The no-memory arm decodes ~``T``
+tokens/request; ours decodes ~``T-L`` with ``L`` large, so under batching it
+should sustain higher throughput.
+
+Run from the repo root:  python -m harness.phase4_throughput --url http://localhost:30000/v1
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import time
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import requests
+
+from . import metrics
+from .phase4_wallclock import _collect_points, MODEL_PATH
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+ARMS = ["no_memory", "static_global", "personal_memory"]
+
+
+def _gen(chat, prompt, ntok, T):
+    t0 = time.perf_counter()
+    r = requests.post(chat, json={"model": "gpt-oss-120b", "temperature": 0.0,
+                                  "max_tokens": ntok, "ignore_eos": True,
+                                  "messages": [{"role": "user",
+                                                "content": prompt}]}, timeout=300)
+    dt = time.perf_counter() - t0
+    r.raise_for_status()
+    return dt, T
+
+
+def _run_arm(chat, jobs, concurrency):
+    """Fire (prompt, decode_ntok, target_T) jobs through a C-worker pool.
+    Each request PRODUCES its full target (``T`` tokens) whether by decoding or
+    by accepting the draft's prefix; the memory arm decodes fewer (``T-L``) but
+    still produces ``T``, so the correct throughput counts effective OUTPUT
+    tokens (``T``), not decode work. We also report requests/s."""
+    lat, out_toks = [], 0
+    t0 = time.perf_counter()
+    with ThreadPoolExecutor(max_workers=concurrency) as ex:
+        for dt, T in ex.map(lambda j: _gen(chat, j[0], j[1], j[2]), jobs):
+            lat.append(dt * 1000)
+            out_toks += T
+    wall = time.perf_counter() - t0
+    lat.sort()
+    return {"eff_output_tokens_per_s": round(out_toks / wall, 1),
+            "requests_per_s": round(len(jobs) / wall, 2),
+            "wall_s": round(wall, 2),
+            "req_p50_ms": round(lat[len(lat) // 2], 1),
+            "req_p95_ms": round(lat[min(len(lat) - 1, int(0.95 * len(lat)))], 1)}
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--url", default="http://localhost:30000/v1")
+    p.add_argument("--domains", nargs="+",
+                   default=["airline", "retail", "telecom"])
+    p.add_argument("--concurrency", nargs="+", type=int, default=[1, 8, 32])
+    p.add_argument("--sample", type=int, default=96)
+    p.add_argument("--seed", type=int, default=0)
+    args = p.parse_args()
+    chat = args.url.rstrip("/") + "/chat/completions"
+    metrics.get_tokenizer(MODEL_PATH)
+
+    pts = _collect_points(args.domains)
+    rng = random.Random(args.seed)
+    rng.shuffle(pts)
+    pts = [x for x in pts if x["T"] >= 2][:args.sample]
+    print(f"[throughput] {len(pts)} sampled requests", flush=True)
+
+    # precompute per-arm job list: (prompt, decode_ntok=T-L, target_T)
+    jobs = {a: [(x["query"][:1500], max(1, x["T"] - x[a]), x["T"]) for x in pts]
+            for a in ARMS}
+    mean_tok = {a: round(sum(n for _, n, _ in jobs[a]) / len(jobs[a]), 1)
+                for a in ARMS}
+
+    out = {"concurrency_levels": args.concurrency, "arms": ARMS,
+           "n_requests": len(pts), "mean_decode_tokens_per_req": mean_tok,
+           "results": defaultdict(dict)}
+    for C in args.concurrency:
+        for a in ARMS:
+            r = _run_arm(chat, jobs[a], C)
+            out["results"][str(C)][a] = r
+            print(f"  C={C:2d} {a:16s} out={r['eff_output_tokens_per_s']:8.1f} tok/s "
+                  f"req/s={r['requests_per_s']:.2f} p50={r['req_p50_ms']:.0f}ms "
+                  f"p95={r['req_p95_ms']:.0f}ms", flush=True)
+    # personal over no_memory effective-output throughput at each C
+    out["personal_over_vanilla_throughput"] = {
+        str(C): round(out["results"][str(C)]["personal_memory"]["eff_output_tokens_per_s"] /
+                      out["results"][str(C)]["no_memory"]["eff_output_tokens_per_s"], 3)
+        for C in args.concurrency}
+    out["results"] = dict(out["results"])
+    (RESULTS / "phase4_throughput.json").write_text(json.dumps(out, indent=2))
+    print("\npersonal/vanilla throughput ratio by concurrency:",
+          out["personal_over_vanilla_throughput"])
+
+
+if __name__ == "__main__":
+    main()

harness/phase4_wallclock.py

@@ -0,0 +1,161 @@
+"""Phase 4.1 — deployed wall-clock speedup via a faithful external spec-decode
+loop with REAL timers (not engine-integrated).
+
+Wiring a retrieval draft into vLLM/sglang's internal speculative-decoding hook
+is out of reach in the available time, so — as the directive permits — we
+reproduce the accept/reject/re-decode loop end-to-end against the live served
+model with real wall-clock timers, and label it plainly as an external harness.
+
+Spec-decode accounting (single retrieval draft per call): the target model runs
+ONE verification forward over the drafted tool call, accepts its ``L``-token
+correct prefix (token-LCP against the greedy target, exactly the MAT metric),
+then autoregressively decodes the remaining ``T-L`` target tokens. So the target
+performs ~``(T-L)`` sequential forwards plus one verify, vs. ``T`` for a
+no-speculation baseline. We MEASURE the real per-request wall-clock of generating
+``T`` tokens (baseline) and ``T-L`` tokens (each arm) from the actual decision-
+point prompt on the served gpt-oss-120b — the content is irrelevant, only the
+decode-step count and real server timing matter — and add one measured verify
+forward. Reports p50/p95 latency per arm and end-to-end speedup vs no-memory.
+
+Run from the repo root:  python -m harness.phase4_wallclock --url http://localhost:30000/v1
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import time
+from collections import defaultdict
+from pathlib import Path
+
+import requests
+
+from . import metrics
+from .data import Task
+from .memory import Embedder, NoMemory, PersonalMemory, StaticGlobal
+from .run_accept import _parse_target
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+# Tokenizer for the token-LCP accept metric: HF hub id by default;
+# override with a local snapshot path if running offline.
+MODEL_PATH = os.environ.get("SPECMEM_TOKENIZER", "openai/gpt-oss-120b")
+
+
+def _collect_points(domains):
+    """Replay the live tau2 decision points; record, per post-warmup point,
+    each arm's token-accept length against the served target."""
+    dp = [json.loads(l) for l in
+          (RESULTS / "tau2_live_decision_points.jsonl").read_text().splitlines()
+          if json.loads(l)["domain"] in domains]
+    tools = {d: json.loads((ROOT / "data" / "tau2" /
+                            f"tools_{d}.json").read_text()) for d in domains}
+    tasks = [Task(id=r["id"], query=r["query"], functions=tools[r["domain"]],
+                  origin_id=r["id"]) for r in dp]
+    targets = {r["query"]: r["target"] for r in dp}
+    emb = Embedder()
+    inst = build_users(tasks, n_users=40, tasks_per_user=15, n_sessions=12,
+                       queries_per_session=6, seed=0, perturb_prob=0.0)
+    inst.sort(key=lambda x: (x.session, x.user_id))
+    arms = [NoMemory(), StaticGlobal(), PersonalMemory(capacity=48, eviction="lru")]
+    cur, pts = -1, []
+    for ins in inst:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur:
+            cur = ins.session
+            if cur == 1:
+                for a in arms:
+                    if hasattr(a, "freeze"):
+                        a.freeze()
+        if ins.session > 0:
+            row = {"query": ins.query, "target": tgt,
+                   "T": metrics.accept_length(tgt, tgt)[1]}
+            for a in arms:
+                row[a.name] = metrics.accept_length(
+                    a.draft(ins.query, ins.functions, ins.user_id, emb), tgt)[0]
+            pts.append(row)
+        cn, ca = _parse_target(tgt)
+        for a in arms[1:]:
+            a.observe(ins.query, ins.functions, ins.user_id, cn, ca, emb)
+            if isinstance(a, PersonalMemory) and ins.session == 0:
+                a.seed_shared(ins.query, cn, ca, emb)
+    return pts
+
+
+def _timed_gen(url, prompt, max_tok):
+    """Real wall-clock (ms) to generate exactly max_tok tokens; content unused."""
+    t0 = time.perf_counter()
+    r = requests.post(url, json={"model": "gpt-oss-120b", "temperature": 0.0,
+                                 "max_tokens": max_tok, "ignore_eos": True,
+                                 "messages": [{"role": "user", "content": prompt}]},
+                      timeout=180)
+    dt = (time.perf_counter() - t0) * 1000
+    r.raise_for_status()
+    return dt
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--url", default="http://localhost:30000/v1")
+    p.add_argument("--domains", nargs="+",
+                   default=["airline", "retail", "telecom"])
+    p.add_argument("--sample", type=int, default=120)
+    p.add_argument("--seed", type=int, default=0)
+    args = p.parse_args()
+    chat = args.url.rstrip("/") + "/chat/completions"
+
+    metrics.get_tokenizer(MODEL_PATH)
+    pts = _collect_points(args.domains)
+    rng = random.Random(args.seed)
+    rng.shuffle(pts)
+    pts = pts[: args.sample]
+    print(f"[wallclock] {len(pts)} sampled decision points", flush=True)
+
+    # one measured verify forward (prefill+1 tok) as spec-decode overhead
+    verify_ms = sorted(_timed_gen(chat, pts[i]["query"][:1500], 1)
+                       for i in range(min(15, len(pts))))
+    verify = verify_ms[len(verify_ms) // 2]
+
+    lat = defaultdict(list)      # arm -> per-request wall-clock ms
+    for k, row in enumerate(pts):
+        prompt = row["query"][:1500]
+        T = max(1, row["T"])
+        # cache decode-time per distinct token count to save calls
+        need = {T}
+        for arm in ("no_memory", "static_global", "personal_memory"):
+            need.add(max(1, T - row[arm]))
+        tcache = {n: _timed_gen(chat, prompt, n) for n in need}
+        lat["baseline_no_spec"].append(tcache[T])
+        for arm in ("no_memory", "static_global", "personal_memory"):
+            lat[arm].append(verify + tcache[max(1, T - row[arm])])
+        if (k + 1) % 20 == 0:
+            print(f"  {k+1}/{len(pts)}", flush=True)
+
+    def stats(xs):
+        xs = sorted(xs)
+        return {"p50_ms": round(xs[len(xs) // 2], 1),
+                "p95_ms": round(xs[int(0.95 * len(xs))], 1),
+                "mean_ms": round(sum(xs) / len(xs), 1)}
+
+    base = stats(lat["baseline_no_spec"])
+    out = {"config": {"domains": args.domains, "n": len(pts),
+                      "verify_forward_ms": round(verify, 1),
+                      "note": "faithful external spec-decode loop, real timers, "
+                              "NOT engine-integrated (see docstring)"},
+           "baseline_no_spec": base, "arms": {}}
+    for arm in ("no_memory", "static_global", "personal_memory"):
+        s = stats(lat[arm])
+        s["speedup_vs_no_memory_p50"] = round(
+            stats(lat["no_memory"])["p50_ms"] / s["p50_ms"], 3)
+        s["speedup_vs_baseline_p50"] = round(base["p50_ms"] / s["p50_ms"], 3)
+        out["arms"][arm] = s
+    (RESULTS / "phase4_wallclock_deployed.json").write_text(json.dumps(out, indent=2))
+    print(json.dumps(out, indent=2))
+
+
+if __name__ == "__main__":
+    main()

harness/plots.py

@@ -0,0 +1,107 @@
+"""Generate figures from results/*.json into paper/figures/."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+FIGS = ROOT / "figures"
+
+ARM_STYLE = {
+    "no_memory": ("No memory (schema draft)", "#888888", "o", "--"),
+    "static_global": ("Static datastore (ToolSpec-style)", "#d1495b", "s", "-"),
+    "personal_memory": ("Ours (personal + evict)", "#1b6ca8", "D", "-"),
+}
+
+
+def plot_acceptance(tag=""):
+    fname = f"{tag}_accept_results.json" if tag else "accept_results.json"
+    data = json.loads((RESULTS / fname).read_text())
+    summary = data["summary"]
+    n_sessions = data["config"]["sessions"]
+
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 4.2))
+    for arm, (label, color, marker, ls) in ARM_STYLE.items():
+        if arm not in summary:
+            continue
+        xs = sorted(int(s) for s in summary[arm])
+        mat = [summary[arm][str(s)]["MAT"] for s in xs]
+        frac = [summary[arm][str(s)]["accepted_frac"] for s in xs]
+        ax1.plot(xs, mat, marker=marker, ls=ls, color=color, label=label)
+        ax2.plot(xs, frac, marker=marker, ls=ls, color=color, label=label)
+
+    for ax, ylab, title in ((ax1, "Mean accepted tokens (MAT)",
+                             "Draft acceptance vs. session"),
+                            (ax2, "Accepted fraction of target",
+                             "Accepted fraction vs. session")):
+        ax.set_xlabel("Session index")
+        ax.set_ylabel(ylab)
+        ax.set_title(title)
+        ax.grid(alpha=0.3)
+        ax.axvspan(-0.4, 0.4, color="k", alpha=0.05)
+    ax1.legend(fontsize=8, loc="best")
+    ax1.annotate("warmup", (0, ax1.get_ylim()[1]*0.05), fontsize=7, ha="center")
+    fig.tight_layout()
+    FIGS.mkdir(parents=True, exist_ok=True)
+    fig.savefig(FIGS / "acceptance_by_session.pdf")
+    fig.savefig(FIGS / "acceptance_by_session.png", dpi=140)
+    print("wrote acceptance_by_session.{pdf,png}")
+
+
+def plot_safety():
+    """Two-panel safety figure over the three execution policies:
+    (left) severity-weighted cost of being wrong; (right) safe spec-executions
+    preserved (the latency win) and bad irreversible actions."""
+    data = json.loads((RESULTS / "safety_results.json").read_text())
+    policies = ["naive_exec", "conf_gate", "gated_exec"]
+    labels = ["Naive\nspec-exec", "Confidence\ngate", "Idempotency\ngate (ours)"]
+    colors = ["#d1495b", "#e8a33d", "#1b6ca8"]
+
+    wcost = [data[p]["weighted_cost"] for p in policies]
+    bad = [data[p]["bad_irreversible_actions"] for p in policies]
+    safe = [data[p]["safe_spec_executions"] for p in policies]
+    x = list(range(len(policies)))
+
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 4.1))
+
+    ax1.bar(x, wcost, color=colors)
+    for i, w in enumerate(wcost):
+        ax1.text(i, w + max(wcost) * 0.02, f"{w:.0f}", ha="center",
+                 fontsize=10, fontweight="bold", color=colors[i])
+    ax1.set_xticks(x)
+    ax1.set_xticklabels(labels, fontsize=8.5)
+    ax1.set_ylabel("Severity-weighted cost of being wrong")
+    ax1.set_title("Cost of wrong irreversible speculative executions")
+    ax1.grid(axis="y", alpha=0.3)
+
+    ax2.bar(x, safe, color="#1b6ca8", label="Safe spec-executions (latency win)")
+    ax2.bar(x, bad, bottom=safe, color="#d1495b",
+            label="Bad irreversible actions")
+    for i, b in enumerate(bad):
+        ax2.text(i, safe[i] + b + max(safe) * 0.02,
+                 f"{b:.0f} bad", ha="center", fontsize=8.5,
+                 color="#d1495b", fontweight="bold")
+    ax2.set_xticks(x)
+    ax2.set_xticklabels(labels, fontsize=8.5)
+    ax2.set_ylabel("Speculative executions (mean/stream)")
+    ax2.set_title("Latency win preserved vs. harm incurred")
+    ax2.legend(fontsize=8, loc="upper right")
+    ax2.grid(axis="y", alpha=0.3)
+
+    fig.tight_layout()
+    FIGS.mkdir(parents=True, exist_ok=True)
+    fig.savefig(FIGS / "safety.pdf")
+    fig.savefig(FIGS / "safety.png", dpi=140)
+    print("wrote safety.{pdf,png}")
+
+
+if __name__ == "__main__":
+    import sys
+    tag = sys.argv[1] if len(sys.argv) > 1 else ""
+    plot_acceptance(tag)
+    plot_safety()

harness/reset_arm.py

@@ -0,0 +1,97 @@
+"""Session-reset baseline (reviewer ask, round 7).
+
+Separates CROSS-SESSION persistence from mere WITHIN-SESSION liveness: a
+"reset" arm ingests live during a session but is restored to its end-of-warmup
+state at every session boundary. Then, post-warmup:
+    reset - static      = value of within-session liveness alone
+    personal - reset    = value of persisting across sessions
+Replays the phase-2 seed-0 stream against cached targets (no GPU/model calls).
+
+Usage: python -m harness.reset_arm  (from code/)
+Writes results/phase2_reset_arm.json.
+"""
+from __future__ import annotations
+
+import copy
+import json
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .data import load_bfcl
+from .memory import Embedder, PersonalMemory, StaticGlobal
+from .run_accept import MODEL_PATH, _parse_target
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+
+
+def main():
+    metrics.get_tokenizer(MODEL_PATH)
+    tasks = load_bfcl()
+    embedder = Embedder()
+    instances = build_users(tasks, n_users=40, tasks_per_user=15,
+                            n_sessions=12, queries_per_session=6, seed=0)
+    instances.sort(key=lambda x: (x.session, x.user_id))
+    targets = json.loads((RESULTS / "phase2_targets_seed0.json").read_text())
+
+    static = StaticGlobal()
+    personal = PersonalMemory(capacity=48, eviction="lru")
+    reset = PersonalMemory(capacity=48, eviction="lru")
+    warm_snapshot = None
+
+    agg = {n: defaultdict(list) for n in ("static", "personal", "reset")}
+    cur_session = -1
+    for ins in instances:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur_session:
+            cur_session = ins.session
+            if cur_session == 1:
+                static.freeze()
+                warm_snapshot = copy.deepcopy(reset)   # end-of-warmup state
+            elif cur_session > 1:
+                reset = copy.deepcopy(warm_snapshot)   # wipe session memory
+        for name, a in (("static", static), ("personal", personal),
+                        ("reset", reset)):
+            draft = a.draft(ins.query, ins.functions, ins.user_id, embedder)
+            agg[name][ins.session].append(metrics.score(draft, tgt))
+        cname, cargs = _parse_target(tgt)
+        for a in (static, personal, reset):
+            a.observe(ins.query, ins.functions, ins.user_id, cname, cargs,
+                      embedder)
+            if isinstance(a, PersonalMemory) and ins.session == 0:
+                a.seed_shared(ins.query, cname, cargs, embedder)
+
+    out = {}
+    for name in agg:
+        scores = [x for s, xs in agg[name].items() if s > 0 for x in xs]
+        n = len(scores)
+        out[name] = {
+            "n": n,
+            "MAT": round(sum(x["accept_length"] for x in scores) / n, 3),
+            "exact_rate": round(sum(1 for x in scores if x["exact"]) / n, 4),
+            "by_session": {str(s): round(sum(x["accept_length"] for x in xs)
+                                         / len(xs), 2)
+                           for s, xs in sorted(agg[name].items())},
+        }
+    result = {"config": {"users": 40, "tasks_per_user": 15, "sessions": 12,
+                         "queries_per_session": 6, "seed": 0, "capacity": 48,
+                         "targets": "phase2_targets_seed0.json (cached)"},
+              "arms": out,
+              "decomposition": {
+                  "within_session_liveness (reset - static)":
+                      round(out["reset"]["MAT"] - out["static"]["MAT"], 3),
+                  "cross_session_persistence (personal - reset)":
+                      round(out["personal"]["MAT"] - out["reset"]["MAT"], 3),
+              }}
+    (RESULTS / "phase2_reset_arm.json").write_text(json.dumps(result,
+                                                              indent=2))
+    print(json.dumps(result["arms"]["reset"], indent=1))
+    print(json.dumps(result["decomposition"], indent=1))
+
+
+if __name__ == "__main__":
+    main()

harness/review_r1_warmup.py

@@ -0,0 +1,144 @@
+"""Review R1 — warmup-budget sensitivity of the frozen-vs-live gap.
+
+Does the frozen store's decay just reflect a stingy warmup? We sweep the warmup
+budget: session-0 sees {50% (headline), 75%, 100%} of each user's eventual task
+set (simulate.build_users' warmup_frac). At 100% warmup the frozen store has seen
+every task template, so any residual gap is pure argument-drift (numeric-variant)
+decay; at 50% the gap also carries post-warmup novelty. Either outcome is
+informative.
+
+Arms: static_global (frozen at session 1) vs personal_memory (live). 3 seeds.
+warmup_frac=0.5 reuses the cached phase2 targets (identical build); 0.75/1.0
+generate their own greedy targets from the served model once and cache them.
+
+Run from the repo root:  python -m harness.review_r1_warmup --url http://localhost:30000/v1
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import statistics as st
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .client import ToolClient
+from .data import load_bfcl
+from .memory import Embedder, PersonalMemory, StaticGlobal
+from .run_accept import _parse_target, generate_targets
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+# Tokenizer for the token-LCP accept metric: HF hub id by default;
+# override with a local snapshot path if running offline.
+MODEL_PATH = os.environ.get("SPECMEM_TOKENIZER", "openai/gpt-oss-120b")
+WARMUPS = [0.5, 0.75, 1.0]
+SEEDS = [0, 1, 2]
+
+
+class Slot:
+    __slots__ = ("query", "functions")
+
+    def __init__(self, query, functions):
+        self.query, self.functions = query, functions
+
+
+def _replay(inst, targets, emb):
+    static, personal = StaticGlobal(), PersonalMemory(capacity=48, eviction="lru")
+    agg = {"static_global": defaultdict(list), "personal_memory": defaultdict(list)}
+    cur = -1
+    for ins in inst:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur:
+            cur = ins.session
+            if cur == 1:
+                static.freeze()
+        agg["static_global"][ins.session].append(
+            metrics.score(static.draft(ins.query, ins.functions, ins.user_id, emb), tgt))
+        agg["personal_memory"][ins.session].append(
+            metrics.score(personal.draft(ins.query, ins.functions, ins.user_id, emb), tgt))
+        cn, ca = _parse_target(tgt)
+        static.observe(ins.query, ins.functions, ins.user_id, cn, ca, emb)
+        personal.observe(ins.query, ins.functions, ins.user_id, cn, ca, emb)
+        if ins.session == 0:
+            personal.seed_shared(ins.query, cn, ca, emb)
+    out = {}
+    for a in ("static_global", "personal_memory"):
+        post = [x for s, xs in agg[a].items() if s > 0 for x in xs]
+        out[a] = sum(x["accept_length"] for x in post) / max(1, len(post))
+    return out["static_global"], out["personal_memory"]
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--url", default="http://localhost:30000/v1")
+    p.add_argument("--model", default="gpt-oss-120b")
+    p.add_argument("--workers", type=int, default=16)
+    args = p.parse_args()
+
+    metrics.get_tokenizer(MODEL_PATH)
+    tasks = load_bfcl()
+    emb = Embedder()
+    client = ToolClient(url=args.url, model=args.model)
+    assert client.ping(), f"served model not reachable at {args.url}"
+
+    sweep = {}
+    for wf in WARMUPS:
+        per_seed = {"static_global": [], "personal_memory": []}
+        for sd in SEEDS:
+            inst = build_users(tasks, n_users=40, tasks_per_user=15, n_sessions=12,
+                               queries_per_session=6, seed=sd, warmup_frac=wf)
+            inst.sort(key=lambda x: (x.session, x.user_id))
+            if wf == 0.5:
+                cache_f = RESULTS / f"phase2_targets_seed{sd}.json"
+            else:
+                cache_f = RESULTS / f"review_r1_targets_w{wf}_seed{sd}.json"
+            cache = json.loads(cache_f.read_text()) if cache_f.exists() else {}
+            fmap = {i.query: i.functions for i in inst}
+            miss = [Slot(q, fmap[q]) for q in {i.query for i in inst} if q not in cache]
+            if miss:
+                print(f"  [wf={wf} seed={sd}] generating {len(miss)} targets ...", flush=True)
+                cache.update(generate_targets(client, miss, workers=args.workers))
+                # never overwrite the shared phase2 cache; write R1-owned copy
+                outf = (RESULTS / f"review_r1_targets_w{wf}_seed{sd}.json")
+                outf.write_text(json.dumps(cache))
+            sg, pm = _replay(inst, cache, emb)
+            per_seed["static_global"].append(sg)
+            per_seed["personal_memory"].append(pm)
+            print(f"wf={wf} seed={sd}: static {sg:.3f} | personal {pm:.3f} | "
+                  f"gap {100*(pm-sg)/sg:+.1f}%", flush=True)
+        sg = st.mean(per_seed["static_global"])
+        pm = st.mean(per_seed["personal_memory"])
+        sweep[str(wf)] = {
+            "warmup_frac": wf,
+            "n_known_tasks": max(1, round(15 * wf)), "n_novel_tasks": 15 - max(1, round(15 * wf)),
+            "static_MAT": round(sg, 3), "personal_MAT": round(pm, 3),
+            "gap_pct": round(100 * (pm - sg) / sg, 1),
+            "static_std": round(st.pstdev(per_seed["static_global"]), 3),
+            "personal_std": round(st.pstdev(per_seed["personal_memory"]), 3)}
+        print(f"== wf={wf}: static {sg:.3f} | personal {pm:.3f} | "
+              f"gap {sweep[str(wf)]['gap_pct']:+}% ==", flush=True)
+
+    out = {"config": {"users": 40, "tasks_per_user": 15, "sessions": 12,
+                      "seeds": SEEDS, "arms": "static_global (frozen) vs personal_memory (live)"},
+           "sweep_by_warmup": sweep,
+           "note": ("Post-warmup MAT (sessions 1-11), 3 seeds. warmup_frac controls how "
+                    "much of each user's task set the frozen store sees at session 0. At "
+                    "1.0 the frozen store has seen every template, so residual gap is pure "
+                    "argument-drift decay; below 1.0 it also carries post-warmup novelty. "
+                    "Real replay; wf=0.5 reuses cached phase2 targets, 0.75/1.0 use their "
+                    "own served greedy targets.")}
+    (RESULTS / "review_r1_warmup_sweep.json").write_text(json.dumps(out, indent=2))
+    print("\n=== gap vs warmup fraction ===")
+    for wf in WARMUPS:
+        s = sweep[str(wf)]
+        print(f"  warmup {int(wf*100)}%: static {s['static_MAT']} | personal "
+              f"{s['personal_MAT']} | gap {s['gap_pct']:+}%")
+
+
+if __name__ == "__main__":
+    main()

harness/review_r2_provenance.py

@@ -0,0 +1,177 @@
+"""Review R2 — cross-user draft-provenance rate in the live GLOBAL store.
+
+The paper asserts that per-user partitioning buys privacy/isolation. This measures
+it: in a live global (non-personalized) store, what fraction of drafts served to
+user U were actually written by a DIFFERENT user U'? That is the leakage channel a
+per-user store closes by construction (0% — a user only ever drafts from their own
+calls). We reuse the Phase 4.3 overlap workload (user-consistent args: each user has
+fixed, distinct argument realizations, so a cross-user draft carries another user's
+concrete values) and its cached greedy targets — CPU replay, no serving.
+
+Mirrors GlobalEvict exactly (same sim_threshold, LRU, capacity) but tags each stored
+entry with its writer's user id and logs, per retrieval hit, whether writer != querier.
+
+Run from the repo root:  python -m harness.review_r2_provenance
+"""
+from __future__ import annotations
+
+import json
+import os
+import statistics as st
+from collections import OrderedDict, defaultdict
+from pathlib import Path
+
+from . import metrics
+from .data import load_bfcl
+from .memory import Embedder, Entry, _best_match, schema_draft
+from .metrics import canonical_call_str
+from .run_accept import _parse_target
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+# Tokenizer for the token-LCP accept metric: HF hub id by default;
+# override with a local snapshot path if running offline.
+MODEL_PATH = os.environ.get("SPECMEM_TOKENIZER", "openai/gpt-oss-120b")
+OVERLAPS = [0.0, 0.25, 0.5, 0.75, 1.0]
+SEEDS = [0, 1, 2]
+CAP = 48
+
+
+class ProvenanceGlobal:
+    """GlobalEvict with per-entry writer tracking (byte-identical draft/observe
+    logic; only adds provenance bookkeeping)."""
+
+    def __init__(self, capacity):
+        self.capacity = capacity
+        self.sim_threshold = 0.35
+        self.store: "OrderedDict[int, Entry]" = OrderedDict()
+        self.writer: dict[int, str] = {}
+        self._next_id = 0
+
+    def draft(self, query, functions, user_id, embedder):
+        """Returns (call, writer_uid_or_None). writer is None on a schema-draft
+        miss (no retrieval); otherwise the user who wrote the retrieved entry."""
+        emb = embedder.embed(query)
+        keys = list(self.store.keys())
+        entries = [self.store[k] for k in keys]
+        i, sim = _best_match(emb, entries)
+        if i >= 0 and sim >= self.sim_threshold:
+            key = keys[i]
+            self.store.move_to_end(key)
+            self.store[key].freq += 1
+            return self.store[key].call, self.writer[key]
+        return schema_draft(functions), None
+
+    def observe(self, query, functions, user_id, name, args, embedder):
+        emb = embedder.embed(query)
+        eid = self._next_id
+        self._next_id += 1
+        self.store[eid] = Entry(emb, canonical_call_str(name, args))
+        self.writer[eid] = user_id
+        self.store.move_to_end(eid)
+        while len(self.store) > self.capacity:
+            old, _ = self.store.popitem(last=False)
+            self.writer.pop(old, None)
+
+
+def _redact(call_str):
+    """Keep the call's structure (name + arg keys) but replace concrete argument
+    VALUES with a placeholder, so the example shows the leakage channel without
+    printing another user's real data."""
+    try:
+        obj = json.loads(call_str)
+        args = obj.get("arguments", {})
+        obj["arguments"] = {k: "<value from writer's call>" for k in args}
+        return json.dumps(obj)
+    except Exception:
+        return "<unparseable>"
+
+
+def main():
+    metrics.get_tokenizer(MODEL_PATH)
+    tasks = load_bfcl()
+    emb = Embedder()
+    sweep = {}
+    example = None
+    for ov in OVERLAPS:
+        # per-seed rates
+        hit_cross, hit_total, all_total = [], [], []
+        cross_wrongargs = []
+        for sd in SEEDS:
+            inst = build_users(tasks, n_users=40, tasks_per_user=15, n_sessions=12,
+                               queries_per_session=6, seed=sd, overlap_frac=ov,
+                               user_consistent=True)
+            inst.sort(key=lambda x: (x.session, x.user_id))
+            cache_f = RESULTS / f"phase4_overlap_targets_ov{ov}_seed{sd}.json"
+            if not cache_f.exists():
+                print(f"  MISSING {cache_f.name}; skip", flush=True)
+                continue
+            targets = json.loads(cache_f.read_text())
+            store = ProvenanceGlobal(capacity=40 * CAP)
+            n_all = n_hit = n_cross = n_cross_wrong = 0
+            for ins in inst:
+                tgt = targets.get(ins.query)
+                if tgt is None:
+                    continue
+                if ins.session > 0:                       # post-warmup drafts
+                    call, writer = store.draft(ins.query, ins.functions,
+                                               ins.user_id, emb)
+                    n_all += 1
+                    if writer is not None:
+                        n_hit += 1
+                        if writer != ins.user_id:
+                            n_cross += 1
+                            if call != tgt:               # wrong-args leak
+                                n_cross_wrong += 1
+                            if example is None and ov >= 0.75:
+                                example = {
+                                    "overlap_frac": ov, "seed": sd,
+                                    "querying_user": ins.user_id,
+                                    "writer_user": writer,
+                                    "drafted_call_redacted": _redact(call),
+                                    "note": ("global store served user U a draft "
+                                             "written by user U'; concrete arg "
+                                             "values are U''s, not U's")}
+                cn, ca = _parse_target(tgt)
+                store.observe(ins.query, ins.functions, ins.user_id, cn, ca, emb)
+            if n_hit:
+                hit_cross.append(100 * n_cross / n_hit)
+            hit_total.append(100 * n_hit / max(1, n_all))
+            all_total.append(100 * n_cross / max(1, n_all))
+            cross_wrongargs.append(100 * n_cross_wrong / max(1, n_cross))
+            print(f"ov={ov} seed={sd}: {n_cross}/{n_hit} hits cross-user "
+                  f"({100*n_cross/max(1,n_hit):.1f}%), {n_cross}/{n_all} of all "
+                  f"drafts ({100*n_cross/max(1,n_all):.1f}%)", flush=True)
+        sweep[ov] = {
+            "cross_user_pct_of_retrieval_hits": round(st.mean(hit_cross), 1) if hit_cross else 0.0,
+            "cross_user_pct_of_all_drafts": round(st.mean(all_total), 1) if all_total else 0.0,
+            "retrieval_hit_pct_of_drafts": round(st.mean(hit_total), 1) if hit_total else 0.0,
+            "cross_user_drafts_with_wrong_args_pct": round(st.mean(cross_wrongargs), 1) if cross_wrongargs else 0.0,
+        }
+        print(f"== ov={ov}: cross-user {sweep[ov]['cross_user_pct_of_retrieval_hits']}% "
+              f"of hits ==", flush=True)
+
+    out = {"config": {"users": 40, "tasks_per_user": 15, "sessions": 12,
+                      "capacity_global": 40 * CAP, "seeds": SEEDS,
+                      "workload": "Phase 4.3 user-consistent overlap (cached targets)",
+                      "personal_store_cross_user_rate_pct": 0.0,
+                      "personal_note": ("per-user store: 0% cross-user provenance by "
+                                        "construction — a user only drafts from its own "
+                                        "writes")},
+           "sweep_by_overlap": sweep,
+           "example_cross_user_draft": example,
+           "metric": ("cross-user draft-provenance rate = fraction of drafts served to "
+                      "user U whose retrieved entry was written by a different user U'; "
+                      "measured post-warmup (sessions 1-11), 3 seeds, on the live global "
+                      "store; real replay, no tuning")}
+    (RESULTS / "review_r2_crossuser_provenance.json").write_text(json.dumps(out, indent=2))
+    print("\n=== cross-user provenance rate (% of retrieval hits) by overlap ===")
+    for ov in OVERLAPS:
+        if ov in sweep:
+            print(f"  overlap {ov}: {sweep[ov]['cross_user_pct_of_retrieval_hits']}% "
+                  f"(of all drafts: {sweep[ov]['cross_user_pct_of_all_drafts']}%)")
+
+
+if __name__ == "__main__":
+    main()

harness/review_r3_confidence.py

@@ -0,0 +1,149 @@
+"""Round-3 review ask: EMPIRICAL retrieval-confidence distributions.
+
+All three round-1 reviewers objected that the safety experiment's confidence
+model is author-sampled (correct ~U[0.6,1.0], stale ~U[0.3,0.85]), so the
+"no confidence threshold separates right from wrong" conclusion could be an
+artifact of the chosen supports. This script measures the real thing on the
+full-scale BFCL replay (seed 0, cached targets, GPU-free): for every
+post-warmup draft retrieved from the frozen global store and from the personal
+store, log the retrieval cosine similarity and whether the draft was exactly
+right, then ask whether ANY similarity threshold separates correct from wrong.
+
+Output: results/review_r3_empirical_confidence.json
+Run:    OMP_NUM_THREADS=8 <python> -m harness.review_r3_confidence
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+
+from . import metrics
+from .data import load_bfcl
+from .memory import (Embedder, StaticGlobal, PersonalMemory, _best_match,
+                     schema_draft)
+from .run_accept import MODEL_PATH
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+
+
+def _stats(sims: np.ndarray) -> dict:
+    if len(sims) == 0:
+        return {"n": 0}
+    return {
+        "n": int(len(sims)),
+        "mean": round(float(sims.mean()), 4),
+        "p5": round(float(np.percentile(sims, 5)), 4),
+        "p25": round(float(np.percentile(sims, 25)), 4),
+        "median": round(float(np.percentile(sims, 50)), 4),
+        "p75": round(float(np.percentile(sims, 75)), 4),
+        "p95": round(float(np.percentile(sims, 95)), 4),
+        "max": round(float(sims.max()), 4),
+        "min": round(float(sims.min()), 4),
+    }
+
+
+def _separability(correct: np.ndarray, wrong: np.ndarray) -> dict:
+    """Can a threshold tau (fire draft only if sim >= tau) reach zero wrong
+    while keeping correct drafts? Mirrors the safety confidence-gate sweep."""
+    out = {}
+    if len(wrong) == 0 or len(correct) == 0:
+        return out
+    # AUROC of similarity as a correct-vs-wrong discriminator
+    labels = np.concatenate([np.ones(len(correct)), np.zeros(len(wrong))])
+    scores = np.concatenate([correct, wrong])
+    order = np.argsort(scores)
+    ranks = np.empty(len(scores)); ranks[order] = np.arange(1, len(scores) + 1)
+    n1, n0 = len(correct), len(wrong)
+    auroc = (ranks[labels == 1].sum() - n1 * (n1 + 1) / 2) / (n1 * n0)
+    out["auroc"] = round(float(auroc), 4)
+    # zero-wrong threshold: must clear the highest-similarity wrong draft
+    tau_zero = float(wrong.max())
+    out["tau_for_zero_wrong"] = round(tau_zero, 4)
+    out["correct_forfeited_at_tau_zero_pct"] = round(
+        100 * float((correct <= tau_zero).mean()), 1)
+    # per-threshold sweep mirroring the safety table
+    sweep = {}
+    for tau in (0.5, 0.7, 0.8, 0.9, 0.95, 0.99):
+        sweep[str(tau)] = {
+            "wrong_fired": int((wrong >= tau).sum()),
+            "wrong_fired_pct": round(100 * float((wrong >= tau).mean()), 1),
+            "correct_kept_pct": round(100 * float((correct >= tau).mean()), 1),
+        }
+    out["threshold_sweep"] = sweep
+    return out
+
+
+def run() -> None:
+    metrics.get_tokenizer(MODEL_PATH)
+    tasks = load_bfcl()
+    embedder = Embedder()
+    instances = build_users(
+        tasks, n_users=40, tasks_per_user=15, n_sessions=12,
+        queries_per_session=6, seed=0, perturb_prob=1.0,
+        arrival="spread", novel_weight=3.0, warmup_frac=0.5)
+    instances.sort(key=lambda x: (x.session, x.user_id))
+    targets = json.loads((RESULTS / "phase2_targets_seed0.json").read_text())
+
+    static, personal = StaticGlobal(), PersonalMemory(capacity=48,
+                                                      eviction="lru")
+    log = {"static_global": [], "personal_memory": []}
+    cur_session = -1
+    for ins in instances:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur_session:
+            cur_session = ins.session
+            if cur_session == 1:
+                static.freeze()
+        emb = embedder.embed(ins.query)
+        entries_by_arm = (
+            ("static_global", static.entries),
+            ("personal_memory",
+             list(personal.stores.get(ins.user_id, {}).values())))
+        for arm, entries in entries_by_arm:
+            i, sim = _best_match(emb, entries)
+            if i >= 0 and ins.session >= 1:
+                draft = entries[i].call
+                s = metrics.score(draft, tgt)
+                log[arm].append((float(sim), bool(s["exact"]),
+                                 float(s["accepted_frac"])))
+        # observe/write-back exactly as in run_ablation
+        from .run_accept import _parse_target
+        name, argd = _parse_target(tgt)
+        static.observe(ins.query, ins.functions, ins.user_id, name, argd,
+                       embedder)
+        personal.observe(ins.query, ins.functions, ins.user_id, name, argd,
+                         embedder)
+        if ins.session == 0:
+            personal.seed_shared(ins.query, name, argd, embedder)
+
+    out = {"design": {
+        "workload": "phase2 full-scale BFCL, seed 0, cached greedy targets",
+        "what": "retrieval cosine similarity vs. exact-correctness of the "
+                "retrieved draft, post-warmup (sessions 1-11) retrieval hits",
+        "correct": "draft serializes exactly to the served model's target",
+    }}
+    for arm, rows in log.items():
+        sims = np.array([r[0] for r in rows])
+        exact = np.array([r[1] for r in rows])
+        correct, wrong = sims[exact], sims[~exact]
+        out[arm] = {
+            "n_retrieval_drafts": int(len(rows)),
+            "exact_rate": round(float(exact.mean()), 4),
+            "sim_correct": _stats(correct),
+            "sim_wrong": _stats(wrong),
+            "separability": _separability(correct, wrong),
+        }
+    path = RESULTS / "review_r3_empirical_confidence.json"
+    path.write_text(json.dumps(out, indent=1))
+    print(json.dumps(out, indent=1))
+    print(f"\nWrote {path}")
+
+
+if __name__ == "__main__":
+    run()

harness/review_r3_rebuild_arm.py

@@ -0,0 +1,149 @@
+"""Round-2 review ask: periodic-rebuild baseline + cold-start backoff accounting.
+
+(a) Periodic rebuild: the deployment-realistic competitor to per-call write-back
+is a store rebuilt from accumulated logs every k sessions, frozen in between.
+Arms: static (never rebuilt), rebuild_k1/k2/k4 (global store rebuilt from ALL
+calls observed so far at every k-th session boundary), personal (ours, live).
+Replays the phase-2 seed-0 stream against cached targets (no GPU).
+
+(b) Backoff accounting: fraction of post-warmup PersonalMemory drafts served by
+the user's own store vs. the shared cold-start backoff vs. the schema draft,
+and the cross-user share of backoff-served drafts (writer != requester).
+
+Usage: OMP_NUM_THREADS=8 <python> -m harness.review_r3_rebuild_arm
+Writes results/review_r3_rebuild_backoff.json.
+"""
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .data import load_bfcl
+from .memory import (Embedder, Entry, PersonalMemory, StaticGlobal,
+                     _best_match, canonical_call_str, schema_draft)
+from .run_accept import MODEL_PATH, _parse_target
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+
+
+class RebuildArm:
+    """Frozen global store rebuilt from the full observation log every k
+    sessions; identical retrieval to StaticGlobal in between."""
+
+    def __init__(self, k: int):
+        self.k = k
+        self.name = f"rebuild_k{k}"
+        self.entries: list[Entry] = []
+        self._log: list[Entry] = []
+
+    def maybe_rebuild(self, session: int) -> None:
+        # session boundaries s = 1..11; rebuild at s = 1, 1+k, 1+2k, ...
+        if session >= 1 and (session - 1) % self.k == 0:
+            self.entries = list(self._log)
+
+    def draft(self, query, functions, user_id, embedder) -> str:
+        emb = embedder.embed(query)
+        i, _ = _best_match(emb, self.entries)
+        if i < 0:
+            return schema_draft(functions)
+        return self.entries[i].call
+
+    def observe(self, query, functions, user_id, name, args, embedder) -> None:
+        self._log.append(Entry(embedder.embed(query),
+                               canonical_call_str(name, args)))
+
+
+def main() -> None:
+    metrics.get_tokenizer(MODEL_PATH)
+    tasks = load_bfcl()
+    embedder = Embedder()
+    instances = build_users(tasks, n_users=40, tasks_per_user=15,
+                            n_sessions=12, queries_per_session=6, seed=0,
+                            perturb_prob=1.0, arrival="spread",
+                            novel_weight=3.0, warmup_frac=0.5)
+    instances.sort(key=lambda x: (x.session, x.user_id))
+    targets = json.loads((RESULTS / "phase2_targets_seed0.json").read_text())
+
+    static = StaticGlobal()
+    personal = PersonalMemory(capacity=48, eviction="lru")
+    rebuilds = [RebuildArm(k) for k in (1, 2, 4)]
+    entry_writer: dict[int, str] = {}   # id(shared Entry) -> writer user
+
+    agg = defaultdict(lambda: defaultdict(list))
+    backoff = {"own": 0, "shared": 0, "schema": 0, "shared_cross_user": 0}
+    cur_session = -1
+    for ins in instances:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur_session:
+            cur_session = ins.session
+            if cur_session == 1:
+                static.freeze()
+            for r in rebuilds:
+                r.maybe_rebuild(cur_session)
+        arms = [("static", static), ("personal", personal)] + \
+               [(r.name, r) for r in rebuilds]
+        for name, a in arms:
+            draft = a.draft(ins.query, ins.functions, ins.user_id, embedder)
+            agg[name][ins.session].append(metrics.score(draft, tgt))
+        # backoff accounting for the personal arm (mirrors PersonalMemory.draft)
+        if ins.session >= 1:
+            emb = embedder.embed(ins.query)
+            own = list(personal.stores.get(ins.user_id, {}).values())
+            i, sim = _best_match(emb, own)
+            if i >= 0 and sim >= personal.sim_threshold:
+                backoff["own"] += 1
+            else:
+                j, sj = _best_match(emb, personal.shared)
+                if j >= 0 and sj >= personal.sim_threshold:
+                    backoff["shared"] += 1
+                    w = entry_writer.get(id(personal.shared[j]))
+                    if w is not None and w != ins.user_id:
+                        backoff["shared_cross_user"] += 1
+                else:
+                    backoff["schema"] += 1
+        name, argd = _parse_target(tgt)
+        for _, a in arms:
+            a.observe(ins.query, ins.functions, ins.user_id, name, argd,
+                      embedder)
+        if ins.session == 0:
+            personal.seed_shared(ins.query, name, argd, embedder)
+            entry_writer[id(personal.shared[-1])] = ins.user_id
+
+    def overall(per_sess):
+        xs = [x for s, v in per_sess.items() if s >= 1 for x in v]
+        return {"n": len(xs),
+                "MAT": round(sum(x["accept_length"] for x in xs) / len(xs), 3)}
+
+    out = {"design": {
+        "workload": "phase2 full-scale BFCL, seed 0, cached greedy targets",
+        "rebuild": "global store rebuilt from full observation log at "
+                   "sessions 1, 1+k, 1+2k, ...; frozen in between",
+    }}
+    for name, per_sess in agg.items():
+        out[name] = overall(per_sess)
+        out[name]["by_session"] = {
+            str(s): round(sum(x["accept_length"] for x in v) / len(v), 2)
+            for s, v in sorted(per_sess.items())}
+    n_post = sum(backoff[k] for k in ("own", "shared", "schema"))
+    out["personal_backoff_accounting"] = {
+        "n_post_warmup": n_post,
+        "own_store_pct": round(100 * backoff["own"] / n_post, 1),
+        "shared_backoff_pct": round(100 * backoff["shared"] / n_post, 1),
+        "schema_draft_pct": round(100 * backoff["schema"] / n_post, 1),
+        "shared_cross_user_pct_of_all_drafts":
+            round(100 * backoff["shared_cross_user"] / n_post, 2),
+    }
+    path = RESULTS / "review_r3_rebuild_backoff.json"
+    path.write_text(json.dumps(out, indent=1))
+    print(json.dumps(out, indent=1))
+    print(f"\nWrote {path}")
+
+
+if __name__ == "__main__":
+    main()

harness/run_ablation.py

@@ -0,0 +1,193 @@
+"""Ablation: decompose PersonalMemory's gain over StaticGlobal into the
+contributions of (a) per-user personalization and (b) online-growth+eviction.
+
+2x2 design (personalized? x evicting/growing?):
+  static_global    [-pers, -grow]  frozen global datastore (ToolSpec-style)  [baseline]
+  global_evict     [-pers, +grow]  one global store, keeps ingesting + LRU-evicts
+  personal_noevict [+pers, -evict] per-user store, grows online, UNBOUNDED
+  personal_memory  [+pers, +evict] per-user store, grows online + LRU-evicts  [ours]
+
+All four share the identical target stream (arm-independent greedy target decode
+from the served model), so they differ ONLY in memory policy. Targets are cached
+to disk so re-runs need no GPU. Single seed at full scale: the primary 3-seed run
+established seed-std <= 0.20 MAT, so one seed is sufficient to attribute the gap.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .client import ToolClient
+from .data import load_bfcl
+from .memory import (Embedder, StaticGlobal, GlobalEvict, PersonalNoEvict,
+                     PersonalMemory)
+from .run_accept import generate_targets, _parse_target, MODEL_PATH
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+
+
+def _make_arms(args):
+    return [
+        StaticGlobal(),
+        GlobalEvict(capacity=args.users * args.capacity),
+        PersonalNoEvict(),
+        PersonalMemory(capacity=args.capacity, eviction="lru"),
+    ]
+
+
+def _replay(instances, targets, embedder, args):
+    arms = _make_arms(args)
+    agg = {a.name: defaultdict(list) for a in arms}
+    cur_session = -1
+    for ins in instances:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur_session:
+            cur_session = ins.session
+            for a in arms:
+                if isinstance(a, StaticGlobal) and cur_session == 1:
+                    a.freeze()
+        for a in arms:
+            draft = a.draft(ins.query, ins.functions, ins.user_id, embedder)
+            agg[a.name][ins.session].append(metrics.score(draft, tgt))
+        name, argd = _parse_target(tgt)
+        for a in arms:
+            a.observe(ins.query, ins.functions, ins.user_id, name, argd, embedder)
+            if isinstance(a, PersonalMemory) and ins.session == 0:
+                a.seed_shared(ins.query, name, argd, embedder)
+    return agg
+
+
+def _overall(agg, warmup=0):
+    out = {}
+    for arm, per_sess in agg.items():
+        scores = [x for s, xs in per_sess.items() if s > warmup for x in xs]
+        n = len(scores)
+        out[arm] = {
+            "n": n,
+            "MAT": round(sum(x["accept_length"] for x in scores) / n, 3),
+            "accepted_frac": round(sum(x["accepted_frac"] for x in scores) / n, 4),
+            "exact_rate": round(sum(1 for x in scores if x["exact"]) / n, 4),
+        }
+    return out
+
+
+def _by_session(agg, n_sessions):
+    out = {}
+    for arm, per_sess in agg.items():
+        out[arm] = {}
+        for s in range(n_sessions):
+            xs = per_sess.get(s, [])
+            if xs:
+                out[arm][str(s)] = round(
+                    sum(x["accept_length"] for x in xs) / len(xs), 3)
+    return out
+
+
+def run(args):
+    RESULTS.mkdir(exist_ok=True)
+    metrics.get_tokenizer(args.model_path or MODEL_PATH)
+    tasks = load_bfcl()
+    embedder = Embedder()
+
+    instances = build_users(
+        tasks, n_users=args.users, tasks_per_user=args.tasks_per_user,
+        n_sessions=args.sessions, queries_per_session=args.queries_per_session,
+        seed=args.seed, perturb_prob=args.perturb_prob,
+        arrival=args.arrival, novel_weight=args.novel_weight,
+        warmup_frac=args.warmup_frac)
+    instances.sort(key=lambda x: (x.session, x.user_id))
+
+    cache = RESULTS / f"{args.tag}_targets_seed{args.seed}.json"
+    if cache.exists():
+        print(f"[ablation] loading cached targets from {cache.name}", flush=True)
+        targets = json.loads(cache.read_text())
+    else:
+        client = ToolClient(url=args.url, model=args.model)
+        if not client.ping():
+            raise SystemExit(f"served model not reachable at {args.url}")
+        print(f"[ablation] {len(instances)} instances; generating targets ...",
+              flush=True)
+        targets = generate_targets(client, instances, workers=args.workers)
+        cache.write_text(json.dumps(targets, indent=1))
+        print(f"[ablation] cached {len(targets)} targets -> {cache.name}",
+              flush=True)
+
+    n_none = sum(1 for v in targets.values() if v is None)
+    agg = _replay(instances, targets, embedder, args)
+    overall = _overall(agg, warmup=0)
+    by_session = _by_session(agg, args.sessions)
+
+    static = overall["static_global"]["MAT"]
+    ours = overall["personal_memory"]["MAT"]
+    gevict = overall["global_evict"]["MAT"]
+    pnoev = overall["personal_noevict"]["MAT"]
+    # Clean single-variable contrasts (each isolates ONE knob, holding the rest):
+    #   live/growth : freeze->live on a global store (evict held on)   = gevict - static
+    #   personalization: global->per-user, holding live+evict          = ours   - gevict
+    #   eviction    : unbounded->bounded, holding live+per-user         = ours   - pnoev
+    decomp = {
+        "total_gap_ours_vs_static": round(ours - static, 3),
+        "live_growth_effect (global_evict - static)": round(gevict - static, 3),
+        "personalization_effect (personal_memory - global_evict)": round(ours - gevict, 3),
+        "eviction_effect (personal_memory - personal_noevict)": round(ours - pnoev, 3),
+        "live_growth_share_pct": round(100 * (gevict - static) / (ours - static), 1)
+            if ours != static else None,
+    }
+
+    out = {
+        "config": vars(args),
+        "seed": args.seed,
+        "n_instances": len(instances),
+        "n_unique_queries": len(targets),
+        "n_no_toolcall": n_none,
+        "overall_post_warmup": overall,
+        "by_session": by_session,
+        "decomposition": decomp,
+    }
+    (RESULTS / f"{args.tag}_ablation_results.json").write_text(
+        json.dumps(out, indent=2))
+
+    print("\n=== Ablation: post-warmup MAT (sessions >= 1) ===", flush=True)
+    for arm in ["static_global", "global_evict", "personal_noevict",
+                "personal_memory"]:
+        v = overall[arm]
+        print(f"  {arm:>16}: MAT={v['MAT']:6.2f}  acc_frac={v['accepted_frac']:.3f}"
+              f"  exact={v['exact_rate']:.3f}  n={v['n']}", flush=True)
+    print("\n=== Decomposition ===", flush=True)
+    for k, v in decomp.items():
+        print(f"  {k}: {v}", flush=True)
+    print(f"\nWrote results/{args.tag}_ablation_results.json", flush=True)
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--users", type=int, default=40)
+    p.add_argument("--tasks-per-user", type=int, default=15)
+    p.add_argument("--sessions", type=int, default=12)
+    p.add_argument("--queries-per-session", type=int, default=6)
+    p.add_argument("--capacity", type=int, default=48)
+    p.add_argument("--workers", type=int, default=16)
+    p.add_argument("--seed", type=int, default=0)
+    p.add_argument("--url", default="http://localhost:30000/v1")
+    p.add_argument("--model", default="gpt-oss-120b")
+    p.add_argument("--model-path", default="")
+    p.add_argument("--tag", default="phase2")
+    p.add_argument("--arrival", default="spread", choices=["spread","burst","late"])
+    p.add_argument("--novel-weight", type=float, default=3.0)
+    p.add_argument("--warmup-frac", type=float, default=0.5)
+    p.add_argument("--perturb-prob", type=float, default=1.0,
+                   help="probability a post-warmup query gets numeric-variant "
+                        "perturbation (1.0 = legacy behaviour; 0.0 = verbatim "
+                        "repeats only, isolating novel-task drift).")
+    run(p.parse_args())
+
+
+if __name__ == "__main__":
+    main()

harness/run_accept.py

@@ -0,0 +1,275 @@
+"""POC acceptance experiment: 3 memory arms across simulated users/sessions.
+
+Pipeline:
+  1. Build simulated users -> ordered (session-major) instance stream.
+  2. Generate the genuine greedy target tool call for every unique query from
+     the served gpt-oss-120b (concurrent; cached by exact query string).
+  3. Replay the stream through each arm. For each instance an arm first DRAFTS
+     (from its current memory), we score token-LCP accept vs the target, then
+     the arm OBSERVES the target (growing its store). static_global observes
+     only during warmup (session 0) then freezes -- ToolSpec behaviour.
+  4. Aggregate Mean Accepted Tokens (MAT) and acceptance rate by (arm, session)
+     and dump results/accept_results.json.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from concurrent.futures import ThreadPoolExecutor
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .client import ToolClient
+from .data import load_bfcl, load_sealtools, load_tau2
+from .memory import Embedder, NoMemory, PersonalMemory, StaticGlobal
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+# Tokenizer for the token-LCP accept metric: HF hub id by default;
+# override with a local snapshot path if running offline.
+MODEL_PATH = os.environ.get("SPECMEM_TOKENIZER", "openai/gpt-oss-120b")
+
+
+def generate_targets(client, instances, workers=16):
+    """Return {query: canonical_target_str} for every unique query."""
+    uniq = {}
+    for ins in instances:
+        uniq.setdefault(ins.query, ins.functions)
+    items = list(uniq.items())
+
+    def _one(qf):
+        q, funcs = qf
+        call = client.generate_call(q, funcs)
+        if call is None:
+            return q, None
+        return q, metrics.canonical_call_str(call["name"], call["arguments"])
+
+    targets = {}
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        for i, (q, tgt) in enumerate(ex.map(_one, items)):
+            targets[q] = tgt
+            if (i + 1) % 25 == 0:
+                print(f"  targets {i+1}/{len(items)}", flush=True)
+    return targets
+
+
+def _replay(instances, targets, embedder, args, per_instance):
+    """Replay one seed's stream through the 3 arms; return per-session scores."""
+    arms = [NoMemory(), StaticGlobal(),
+            PersonalMemory(capacity=args.capacity, eviction=args.eviction)]
+    agg = {a.name: defaultdict(list) for a in arms}
+    cur_session = -1
+    for ins in instances:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur_session:
+            cur_session = ins.session
+            for a in arms:                     # freeze static after warmup
+                if isinstance(a, StaticGlobal) and cur_session == 1:
+                    a.freeze()
+        for a in arms:
+            draft = a.draft(ins.query, ins.functions, ins.user_id, embedder)
+            sc = metrics.score(draft, tgt)
+            agg[a.name][ins.session].append(sc)
+            if a.name == "personal_memory" and per_instance is not None:
+                per_instance.append({
+                    "user": ins.user_id, "session": ins.session,
+                    "sig": ins.signature_id, "novel": ins.novel,
+                    "accept": sc["accept_length"], "tlen": sc["target_len"]})
+        call_name, call_args = _parse_target(tgt)
+        for a in arms:
+            if isinstance(a, StaticGlobal):
+                a.observe(ins.query, ins.functions, ins.user_id,
+                          call_name, call_args, embedder)
+            elif isinstance(a, PersonalMemory):
+                a.observe(ins.query, ins.functions, ins.user_id,
+                          call_name, call_args, embedder)
+                if ins.session == 0:
+                    a.seed_shared(ins.query, call_name, call_args, embedder)
+    return agg
+
+
+def run(args):
+    RESULTS.mkdir(exist_ok=True)
+    # Tokenizer used for the token-LCP accept metric: use the served model's
+    # own tokenizer so acceptance reflects what a spec decoder for THAT model
+    # would see. Defaults to gpt-oss for backward compatibility.
+    metrics.get_tokenizer(args.model_path or MODEL_PATH)
+
+    bench = getattr(args, "benchmark", "bfcl")
+    if bench == "tau2":
+        raise SystemExit(
+            "REJECTED DESIGN: --benchmark tau2 previously extracted decision "
+            "points from tau2-bench's SHIPPED reference trajectories, which "
+            "were generated with GPT-4.1 as the agent — an off-policy "
+            "target-substitution bug. Use harness/tau2_live.py to generate canonical "
+            "traces with the real served model + a live user simulator "
+            "(requires OPENAI_API_KEY), then score with its replay mode.")
+    tasks = {"bfcl": load_bfcl, "sealtools": load_sealtools}[bench]()
+    client = ToolClient(url=args.url, model=args.model)
+    if not client.ping():
+        raise SystemExit(f"served model not reachable at {args.url}")
+    embedder = Embedder()
+
+    seeds = list(range(args.seed, args.seed + args.n_seeds))
+    arm_names = ["no_memory", "static_global", "personal_memory"]
+    agg = {a: defaultdict(list) for a in arm_names}      # pooled over seeds
+    per_seed_overall = {a: [] for a in arm_names}        # MAT per seed
+    per_instance = []
+    n_instances_total = n_unique_total = n_none_total = 0
+
+    for si, sd in enumerate(seeds):
+        instances = build_users(
+            tasks, n_users=args.users, tasks_per_user=args.tasks_per_user,
+            n_sessions=args.sessions,
+            queries_per_session=args.queries_per_session, seed=sd)
+        instances.sort(key=lambda x: (x.session, x.user_id))
+        n_instances_total += len(instances)
+        print(f"[seed {sd}] {len(instances)} instances; generating targets ...",
+              flush=True)
+        targets = generate_targets(client, instances, workers=args.workers)
+        n_none = sum(1 for v in targets.values() if v is None)
+        n_unique_total += len(targets)
+        n_none_total += n_none
+        print(f"[seed {sd}] {len(targets)} unique queries, {n_none} no-call",
+              flush=True)
+
+        seed_agg = _replay(instances, targets, embedder, args,
+                           per_instance if si == 0 else None)
+        for a in arm_names:
+            for s, xs in seed_agg[a].items():
+                agg[a][s].extend(xs)
+            post = [x for s, xs in seed_agg[a].items() if s > 0 for x in xs]
+            if post:
+                per_seed_overall[a].append(
+                    sum(x["accept_length"] for x in post) / len(post))
+
+    summary = _summarize(agg, args)
+    overall = _overall(agg, warmup_session=0)   # post-warmup aggregate
+    for a in arm_names:                          # add cross-seed std of MAT
+        vals = per_seed_overall[a]
+        if vals and a in overall:
+            mean = sum(vals) / len(vals)
+            var = sum((v - mean) ** 2 for v in vals) / len(vals)
+            overall[a]["MAT_seed_std"] = round(var ** 0.5, 3)
+            overall[a]["n_seeds"] = len(vals)
+    out = {
+        "config": vars(args),
+        "seeds": seeds,
+        "n_instances": n_instances_total,
+        "n_unique_queries": n_unique_total,
+        "n_no_toolcall": n_none_total,
+        "summary": summary,
+        "overall_post_warmup": overall,
+    }
+    tag = args.tag + "_" if args.tag else ""
+    (RESULTS / f"{tag}accept_results.json").write_text(json.dumps(out, indent=2))
+    (RESULTS / f"{tag}personal_per_instance.json").write_text(
+        json.dumps(per_instance, indent=2))
+    _write_csv(summary, args.sessions, tag)
+    print("\n=== Mean Accepted Tokens (MAT) by session ===", flush=True)
+    _print_table(summary, args.sessions)
+    print("\n=== Overall (sessions >= 1) ===", flush=True)
+    for arm, v in overall.items():
+        print(f"  {arm:>16}: MAT={v['MAT']:.2f}  "
+              f"accepted_frac={v['accepted_frac']:.3f}  "
+              f"exact_rate={v['exact_rate']:.3f}  n={v['n']}", flush=True)
+    print("\nWrote results/accept_results.json + accept_by_session.csv", flush=True)
+
+
+def _overall(agg, warmup_session=0):
+    out = {}
+    for arm, per_sess in agg.items():
+        scores = [x for s, xs in per_sess.items() if s > warmup_session
+                  for x in xs]
+        if not scores:
+            continue
+        n = len(scores)
+        out[arm] = {
+            "n": n,
+            "MAT": round(sum(x["accept_length"] for x in scores) / n, 3),
+            "accepted_frac": round(sum(x["accepted_frac"] for x in scores) / n, 4),
+            "exact_rate": round(sum(1 for x in scores if x["exact"]) / n, 4),
+        }
+    return out
+
+
+def _write_csv(summary, n_sessions, tag=""):
+    lines = ["arm,session,n,MAT,accepted_frac,exact_rate"]
+    for arm in summary:
+        for s in range(n_sessions):
+            v = summary[arm].get(str(s))
+            if v:
+                lines.append(f"{arm},{s},{v['n']},{v['MAT']},"
+                             f"{v['accepted_frac']},{v['exact_rate']}")
+    (RESULTS / f"{tag}accept_by_session.csv").write_text("\n".join(lines) + "\n")
+
+
+def _parse_target(tgt: str):
+    d = json.loads(tgt)
+    return d["name"], d.get("arguments", {})
+
+
+def _summarize(agg, args):
+    summary = {}
+    for arm, per_sess in agg.items():
+        summary[arm] = {}
+        for s, scores in per_sess.items():
+            n = len(scores)
+            mat = sum(x["accept_length"] for x in scores) / n
+            frac = sum(x["accepted_frac"] for x in scores) / n
+            exact = sum(1 for x in scores if x["exact"]) / n
+            summary[arm][str(s)] = {"n": n, "MAT": round(mat, 3),
+                                    "accepted_frac": round(frac, 4),
+                                    "exact_rate": round(exact, 4)}
+    return summary
+
+
+def _print_table(summary, n_sessions):
+    arms = list(summary.keys())
+    header = "session | " + " | ".join(f"{a:>16}" for a in arms)
+    print(header)
+    print("-" * len(header))
+    for s in range(n_sessions):
+        cells = []
+        for a in arms:
+            v = summary[a].get(str(s))
+            cells.append(f"{v['MAT']:>16.2f}" if v else " " * 16)
+        print(f"{s:>7} | " + " | ".join(cells))
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--users", type=int, default=6)
+    p.add_argument("--tasks-per-user", type=int, default=5)
+    p.add_argument("--sessions", type=int, default=8)
+    p.add_argument("--queries-per-session", type=int, default=4)
+    p.add_argument("--capacity", type=int, default=32)
+    p.add_argument("--eviction", default="lru", choices=["lru", "lfu"])
+    p.add_argument("--workers", type=int, default=16)
+    p.add_argument("--seed", type=int, default=0)
+    p.add_argument("--n-seeds", type=int, default=1)
+    p.add_argument("--url", default="http://localhost:30000/v1",
+                   help="OpenAI-compatible endpoint of the served model.")
+    p.add_argument("--model", default="gpt-oss-120b",
+                   help="served-model-name to target.")
+    p.add_argument("--model-path", default="",
+                   help="local path/HF id for the tokenizer used by the "
+                        "accept metric. Empty -> gpt-oss tokenizer.")
+    p.add_argument("--tag", default="", help="output filename prefix "
+                   "(e.g. 'phase2' -> results/phase2_accept_results.json). "
+                   "Empty keeps the original POC filenames.")
+    p.add_argument("--benchmark", default="bfcl",
+                   choices=["bfcl", "sealtools", "tau2"],
+                   help="task pool: BFCL v4 (default), Seal-Tools in-domain "
+                        "test split, or tau2-bench frozen-trajectory decision "
+                        "points.")
+    run(p.parse_args())
+
+
+if __name__ == "__main__":
+    main()

harness/safety.py

@@ -0,0 +1,438 @@
+"""Safety experiment: speculative EXECUTION of non-idempotent tools.
+
+Speculative *decoding* is always safe -- a wrong draft is rejected token by
+token, costing nothing but compute. The danger is speculative *execution*: to
+hide tool latency an agent may fire the drafted call before the target verifies
+it. For a read-only / idempotent tool (get_weather) a wrong speculative
+execution is harmless (re-run with corrected args). For a non-idempotent tool
+(transfer_funds, drop_table, deploy_release) a wrong speculative execution is an
+irreversible real-world side effect that verification cannot undo.
+
+Phase 2 upgrades over the POC:
+
+  * A larger, harder tool registry (14 non-idempotent tools spanning money,
+    data-loss, security, communication, and infrastructure actions) instead of
+    the POC's send_email / delete_file / place_order / transfer_funds.
+  * A **severity-weighted "cost of being wrong"** rather than a raw count.
+    Each irreversible tool carries a domain-motivated severity in [1, 10]
+    (transfer_funds / drop_table = 10, spam email = 3, ...). The headline metric
+    is the *expected cost incurred by wrong irreversible speculative
+    executions*, reported both as a total weighted cost and normalized per
+    opportunity and per 1000 calls -- so a policy that fires one catastrophic
+    action is not scored the same as one that fires one trivial action.
+  * A third policy, ``conf_gate``: a plausible-but-unsafe heuristic that
+    speculatively executes a non-idempotent call only when the draft's retrieval
+    *confidence* clears a threshold. Confidence is correlated with correctness
+    but imperfect, so this heuristic *reduces* but does not *eliminate* cost --
+    which is exactly why a hard idempotency gate is needed. We report it to show
+    the middle ground is not safe.
+  * Multiple seeds; every reported number is a mean over seeds.
+
+Policies:
+  naive_exec : whenever a draft exists, speculatively EXECUTE it immediately.
+  conf_gate  : speculatively execute idempotent drafts always; execute a
+               non-idempotent draft only if its confidence >= CONF_THRESHOLD.
+  gated_exec : speculatively execute ONLY idempotent tools; non-idempotent tools
+               are drafted (for the spec-decoding speedup) but NEVER executed
+               before verification.  (ours)
+"""
+from __future__ import annotations
+
+import json
+import random
+from dataclasses import dataclass
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+
+CONF_THRESHOLD = 0.7      # conf_gate fires a non-idempotent call above this
+
+# Tool registry. `severity` is the "cost of being wrong" if a wrong call is
+# speculatively executed and cannot be undone (0 for idempotent/reversible
+# tools -- a wrong idempotent exec is simply re-run). Severities are ordinal
+# domain judgements in [1, 10].
+TOOL_REGISTRY = {
+    # ---- idempotent / read-only / trivially reversible (severity 0) ----
+    "get_weather":      {"idempotent": True,  "severity": 0},
+    "search_docs":      {"idempotent": True,  "severity": 0},
+    "read_file":        {"idempotent": True,  "severity": 0},
+    "list_dir":         {"idempotent": True,  "severity": 0},
+    "get_balance":      {"idempotent": True,  "severity": 0},
+    "lookup_stock":     {"idempotent": True,  "severity": 0},
+    "translate_text":   {"idempotent": True,  "severity": 0},
+    # ---- non-idempotent / irreversible (severity 1..10) ----
+    "send_email":       {"idempotent": False, "severity": 3},
+    "send_sms":         {"idempotent": False, "severity": 3},
+    "publish_post":     {"idempotent": False, "severity": 5},   # public, hard to retract
+    "cancel_subscription": {"idempotent": False, "severity": 4},
+    "place_order":      {"idempotent": False, "severity": 6},
+    "charge_card":      {"idempotent": False, "severity": 8},
+    "transfer_funds":   {"idempotent": False, "severity": 10},  # money moved
+    "execute_trade":    {"idempotent": False, "severity": 9},   # market order
+    "delete_file":      {"idempotent": False, "severity": 5},
+    "drop_table":       {"idempotent": False, "severity": 10},  # data loss
+    "overwrite_file":   {"idempotent": False, "severity": 6},
+    "deploy_release":   {"idempotent": False, "severity": 8},   # ships to prod
+    "grant_access":     {"idempotent": False, "severity": 8},   # security boundary
+    "power_off_host":   {"idempotent": False, "severity": 7},
+}
+
+IDEM_TOOLS = [t for t, m in TOOL_REGISTRY.items() if m["idempotent"]]
+NONIDEM_TOOLS = [t for t, m in TOOL_REGISTRY.items() if not m["idempotent"]]
+
+# A pool of distinct target objects per tool, so a stale "last object" draft is
+# wrong exactly when the user targets a different object this time.
+_OBJECTS = {
+    "send_email":    ["alice@x.com", "bob@y.com", "carol@z.com", "dan@w.com", "eve@v.com"],
+    "send_sms":      ["+1555000111", "+1555000222", "+1555000333", "+1555000444"],
+    "publish_post":  ["draft-1", "draft-2", "draft-3", "draft-4"],
+    "cancel_subscription": ["sub-basic", "sub-pro", "sub-team", "sub-ent"],
+    "place_order":   ["SKU-1", "SKU-2", "SKU-3", "SKU-4", "SKU-5"],
+    "charge_card":   ["inv-1001", "inv-1002", "inv-1003", "inv-1004"],
+    "transfer_funds":["acct-1", "acct-2", "acct-3", "acct-4", "acct-5"],
+    "execute_trade": ["AAPL", "TSLA", "NVDA", "AMZN", "GOOG"],
+    "delete_file":   ["/tmp/a.log", "/tmp/b.log", "/data/old.csv", "/data/new.csv"],
+    "drop_table":    ["users", "orders", "events", "audit"],
+    "overwrite_file":["cfg.yaml", "prod.env", "hosts", "creds.json"],
+    "deploy_release":["v1.2.0", "v1.2.1", "v1.3.0", "v2.0.0"],
+    "grant_access":  ["role-admin", "role-billing", "role-deploy", "role-read"],
+    "power_off_host":["host-a", "host-b", "host-c", "host-d"],
+    "get_weather":   ["NYC", "SF", "LA", "Austin"],
+    "search_docs":   ["kpi", "roadmap", "budget", "hiring"],
+    "read_file":     ["/etc/hosts", "/tmp/x", "/tmp/y", "/tmp/z"],
+    "list_dir":      ["/", "/home", "/var", "/data"],
+    "get_balance":   ["acct-1", "acct-2", "acct-3"],
+    "lookup_stock":  ["AAPL", "TSLA", "NVDA"],
+    "translate_text":["s1", "s2", "s3"],
+}
+
+
+@dataclass
+class Call:
+    tool: str
+    args: dict
+    draft: dict | None     # what memory would speculatively propose (or None)
+    correct: bool          # does the draft match the true target this turn?
+    conf: float            # retrieval confidence the memory attaches to its draft
+
+
+def build_stream(seed: int = 0, n: int = 400, nonidem_frac: float = 0.45):
+    """A per-user stream mixing idempotent + non-idempotent calls.
+
+    Drafts come from a 'last call to this tool by this user' memory, so a draft
+    is stale exactly when the user targets a new object -- a realistic
+    mispredict. Each draft also carries a *confidence*: high when correct, lower
+    (but overlapping) when stale, modelling an imperfect retrieval score.
+    """
+    rng = random.Random(seed)
+    last_arg: dict[str, str] = {}
+    stream = []
+    for _ in range(n):
+        tool = (rng.choice(NONIDEM_TOOLS) if rng.random() < nonidem_frac
+                else rng.choice(IDEM_TOOLS))
+        obj = rng.choice(_OBJECTS[tool])
+        if tool in last_arg:
+            correct = (last_arg[tool] == obj)
+            # confidence correlates with correctness but overlaps: correct
+            # drafts skew high, stale drafts skew lower yet can still be high.
+            conf = (rng.uniform(0.6, 1.0) if correct
+                    else rng.uniform(0.3, 0.85))
+            draft = {"tool": tool, "target": last_arg[tool]}
+        else:
+            correct, conf, draft = False, 0.0, None
+        stream.append(Call(tool=tool, args={"target": obj}, draft=draft,
+                           correct=correct, conf=conf))
+        last_arg[tool] = obj
+    return stream
+
+
+def run_policy(stream, policy: str, conf_threshold: float = CONF_THRESHOLD,
+               severity_of=None):
+    """Return stats for a speculative-execution policy over the stream.
+
+    Cost accounting:
+      * a wrong speculative exec of a non-idempotent tool incurs its severity;
+      * a wrong speculative exec of an idempotent tool incurs 0 (re-run);
+      * a correct speculative exec incurs 0 and is a latency win.
+
+    ``conf_threshold`` and ``severity_of`` are parameterized so the robustness
+    analyses (conf-threshold sweep, alternative severity weightings) can reuse
+    this exact accounting. Defaults reproduce the headline run() numbers.
+    """
+    if severity_of is None:
+        severity_of = lambda t: TOOL_REGISTRY[t]["severity"]
+    spec_exec = 0
+    safe_spec_exec = 0            # correct, or wrong-but-reversible (latency win, no harm)
+    bad_irreversible = 0         # count of wrong irreversible spec-execs
+    weighted_cost = 0.0          # severity-weighted cost of being wrong
+    worst = 0                    # max single-action severity incurred
+    nonidem_opportunities = 0    # drafted non-idempotent calls the policy *could* fire
+    for call in stream:
+        if call.draft is None:
+            continue
+        idem = TOOL_REGISTRY[call.tool]["idempotent"]
+        if not idem:
+            nonidem_opportunities += 1
+        # --- policy decides whether to speculatively execute ---
+        if policy == "gated_exec" and not idem:
+            continue                       # hard gate: never fire unsafe tools
+        if policy == "conf_gate" and not idem and call.conf < conf_threshold:
+            continue                       # confidence heuristic declines
+        spec_exec += 1
+        if call.correct:
+            safe_spec_exec += 1
+        elif idem:
+            safe_spec_exec += 1            # wrong but reversible -> re-run, no harm
+        else:
+            sev = severity_of(call.tool)
+            bad_irreversible += 1
+            weighted_cost += sev
+            worst = max(worst, sev)
+    n_calls = len(stream)
+    return {
+        "policy": policy,
+        "spec_executions": spec_exec,
+        "safe_spec_executions": safe_spec_exec,
+        "bad_irreversible_actions": bad_irreversible,
+        "weighted_cost": round(weighted_cost, 2),
+        "worst_single_severity": worst,
+        # normalized "cost of being wrong" views
+        "cost_per_1000_calls": round(1000.0 * weighted_cost / n_calls, 2),
+        "cost_per_nonidem_opportunity": (
+            round(weighted_cost / nonidem_opportunities, 3)
+            if nonidem_opportunities else 0.0),
+        "bad_action_rate_per_opportunity": (
+            round(bad_irreversible / nonidem_opportunities, 4)
+            if nonidem_opportunities else 0.0),
+    }
+
+
+def _mean_dicts(dicts):
+    """Mean over a list of same-keyed numeric dicts (keeps str fields)."""
+    out = {}
+    for k in dicts[0]:
+        vals = [d[k] for d in dicts]
+        if isinstance(vals[0], (int, float)):
+            m = sum(vals) / len(vals)
+            out[k] = round(m, 3) if isinstance(vals[0], float) else round(m, 2)
+        else:
+            out[k] = vals[0]
+    return out
+
+
+def run(seeds=(0, 1, 2, 3, 4), n=400, nonidem_frac=0.45):
+    RESULTS.mkdir(exist_ok=True)
+    policies = ["naive_exec", "conf_gate", "gated_exec"]
+    per_seed = {p: [] for p in policies}
+    nonidem_calls = []
+    for sd in seeds:
+        stream = build_stream(seed=sd, n=n, nonidem_frac=nonidem_frac)
+        nonidem_calls.append(sum(1 for c in stream
+                                 if not TOOL_REGISTRY[c.tool]["idempotent"]))
+        for p in policies:
+            per_seed[p].append(run_policy(stream, p))
+    agg = {p: _mean_dicts(per_seed[p]) for p in policies}
+
+    naive_cost = agg["naive_exec"]["weighted_cost"] or 1e-9
+    out = {
+        "design": {
+            "n_seeds": len(seeds),
+            "seeds": list(seeds),
+            "stream_len": n,
+            "nonidem_frac_target": nonidem_frac,
+            "mean_non_idempotent_calls": round(sum(nonidem_calls)
+                                               / len(nonidem_calls), 1),
+            "n_nonidem_tools": len(NONIDEM_TOOLS),
+            "n_idempotent_tools": len(IDEM_TOOLS),
+            "severity_scale": "ordinal 1..10; 0 = idempotent/reversible",
+            "conf_threshold": CONF_THRESHOLD,
+            "primary_metric": "weighted_cost (severity-weighted cost of being "
+                              "wrong); normalized as cost_per_1000_calls and "
+                              "cost_per_nonidem_opportunity",
+        },
+        "severity_table": {t: TOOL_REGISTRY[t]["severity"]
+                           for t in NONIDEM_TOOLS},
+        # backward-compatible top-level policy blocks (old plot reads these)
+        "naive_exec": agg["naive_exec"],
+        "conf_gate": agg["conf_gate"],
+        "gated_exec": agg["gated_exec"],
+        "cost_reduction_vs_naive": {
+            "conf_gate": round(1.0 - agg["conf_gate"]["weighted_cost"]
+                               / naive_cost, 4),
+            "gated_exec": round(1.0 - agg["gated_exec"]["weighted_cost"]
+                                / naive_cost, 4),
+        },
+        "cost_of_being_wrong_metric": "weighted_cost",
+    }
+    (RESULTS / "safety_results.json").write_text(json.dumps(out, indent=2))
+    print(json.dumps(out, indent=2))
+    print("\n=== cost of being wrong (mean over seeds) ===")
+    for p in policies:
+        a = agg[p]
+        print(f"  {p:>11}: weighted_cost={a['weighted_cost']:>7.2f}  "
+              f"bad_actions={a['bad_irreversible_actions']:>5.2f}  "
+              f"cost/1k={a['cost_per_1000_calls']:>6.2f}  "
+              f"safe_execs={a['safe_spec_executions']:>6.2f}")
+    return out
+
+
+# Alternative severity assignments, to show the qualitative policy ranking is
+# not an artifact of the specific ordinal weights (review point A5).
+#   unit  : every non-idempotent tool costs 1 (weighted_cost == raw bad-action count).
+#   tiered: a coarse 3-tier domain map (comms=1, data/deploy/infra=2, money/security=3).
+_TIER = {
+    "send_email": 1, "send_sms": 1, "publish_post": 1, "cancel_subscription": 1,
+    "delete_file": 2, "overwrite_file": 2, "drop_table": 2, "deploy_release": 2,
+    "power_off_host": 2, "place_order": 2,
+    "charge_card": 3, "transfer_funds": 3, "execute_trade": 3, "grant_access": 3,
+}
+_WEIGHTINGS = {
+    "severity_1to10": lambda t: TOOL_REGISTRY[t]["severity"],
+    "unit_raw_count": lambda t: 1,
+    "tiered_1to3":    lambda t: _TIER.get(t, 1),
+}
+
+
+def run_robustness(seeds=(0, 1, 2, 3, 4), n=400, nonidem_frac=0.45):
+    """Address reviewer points A4 (conf-threshold sweep) and A5 (severity
+    robustness). Pure simulation; does NOT touch safety_results.json."""
+    RESULTS.mkdir(exist_ok=True)
+    streams = [build_stream(seed=sd, n=n, nonidem_frac=nonidem_frac) for sd in seeds]
+
+    # --- A4: confidence-threshold sweep for conf_gate ---
+    sweep = []
+    for tau in [0.5, 0.6, 0.7, 0.8, 0.9, 0.95]:
+        rows = [run_policy(s, "conf_gate", conf_threshold=tau) for s in streams]
+        sweep.append({
+            "conf_threshold": tau,
+            "weighted_cost": round(sum(r["weighted_cost"] for r in rows) / len(rows), 2),
+            "bad_irreversible_actions": round(
+                sum(r["bad_irreversible_actions"] for r in rows) / len(rows), 2),
+            "worst_single_severity": max(r["worst_single_severity"] for r in rows),
+            "safe_spec_executions": round(
+                sum(r["safe_spec_executions"] for r in rows) / len(rows), 2),
+        })
+
+    # --- A5: policy ranking under alternative severity weightings ---
+    weighting_tbl = {}
+    for wname, wfn in _WEIGHTINGS.items():
+        weighting_tbl[wname] = {}
+        for policy in ["naive_exec", "conf_gate", "gated_exec"]:
+            rows = [run_policy(s, policy, severity_of=wfn) for s in streams]
+            weighting_tbl[wname][policy] = round(
+                sum(r["weighted_cost"] for r in rows) / len(rows), 2)
+
+    out = {
+        "note": "Robustness checks for the safety experiment; the headline "
+                "numbers live in safety_results.json (unchanged). "
+                "A4 = conf-threshold sweep; A5 = severity-weighting robustness.",
+        "n_seeds": len(seeds), "stream_len": n,
+        "conf_threshold_sweep": sweep,
+        "severity_weighting_robustness": weighting_tbl,
+        "takeaways": {
+            "A4": "conf_gate weighted_cost stays > 0 and keeps firing max-severity "
+                  "actions across every threshold that still preserves latency wins; "
+                  "only tau -> 1 (which forfeits nearly all speculative execution) "
+                  "approaches the gate, and the gate reaches 0 with ~214 safe execs kept.",
+            "A5": "under all three weightings the ranking is invariant: "
+                  "naive_exec > conf_gate > gated_exec = 0.",
+        },
+    }
+    (RESULTS / "phase2_safety_robustness.json").write_text(json.dumps(out, indent=2))
+    print("=== A4: confidence-threshold sweep (conf_gate) ===")
+    for r in sweep:
+        print(f"  tau={r['conf_threshold']}: cost={r['weighted_cost']:>6.2f}  "
+              f"bad={r['bad_irreversible_actions']:>5.2f}  "
+              f"worst_sev={r['worst_single_severity']}  "
+              f"safe_execs={r['safe_spec_executions']:>6.2f}")
+    print("\n=== A5: weighted_cost under alternative severity weightings ===")
+    for wname, row in weighting_tbl.items():
+        print(f"  {wname:>16}: naive={row['naive_exec']:>7.2f}  "
+              f"conf_gate={row['conf_gate']:>7.2f}  gated={row['gated_exec']:>5.2f}")
+    print("\nWrote results/phase2_safety_robustness.json")
+    return out
+
+
+def main():
+    import sys
+    if len(sys.argv) > 1 and sys.argv[1] == "conditional":
+        run_conditional()
+        return
+    if len(sys.argv) > 1 and sys.argv[1] == "robustness":
+        run_robustness()
+    else:
+        run()
+
+
+
+
+# --------------------------------------------------------------------------- #
+# Conditionally-idempotent tools (external-review item 4, round 12).
+# A binary per-tool flag cannot express tools whose safety depends on the CALL:
+#   charge_card_keyed : idempotent ONLY if an idempotency_key is supplied
+#   get_live_quote    : nominally read-only, but each call bills a metered fee
+#   send_reminder     : server deduplicates within a window (usually safe,
+#                       harmful when the dedup window has lapsed)
+# We simulate 400-call streams where each conditional call carries its
+# condition, and compare: binary gate flagging the tool SAFE (permissive),
+# binary gate flagging it UNSAFE (conservative), and a per-CALL ternary gate.
+# Headline registry/numbers are untouched; this writes safety_conditional.json.
+# --------------------------------------------------------------------------- #
+COND_TOOLS = {
+    # name: (severity when the unsafe condition holds, P(call is in safe state))
+    "charge_card_keyed": (8, 0.5),   # safe iff idempotency_key present
+    "get_live_quote":    (1, 0.0),   # every call bills a small metered fee
+    "send_reminder":     (3, 0.7),   # safe iff within the dedup window
+}
+
+
+def run_conditional(seeds=(0, 1, 2, 3, 4), n=400, cond_frac=0.30):
+    import statistics as st
+    out = {}
+    for policy in ("binary_safe", "binary_unsafe", "ternary"):
+        wc, forgone = [], []
+        for sd in seeds:
+            rng = random.Random(1000 + sd)
+            cost, lost = 0.0, 0
+            for _ in range(n):
+                if rng.random() >= cond_frac:
+                    continue                       # non-conditional traffic
+                tool = rng.choice(list(COND_TOOLS))
+                sev, p_safe = COND_TOOLS[tool]
+                safe_now = rng.random() < p_safe
+                correct = rng.random() < 0.66      # draft correctness ~ headline
+                if policy == "binary_safe":        # always speculates
+                    if not safe_now and not correct:
+                        cost += sev                # wrong + unsafe state fires
+                    if tool == "get_live_quote":
+                        cost += sev * (0 if correct else 1) * 0  # counted above
+                elif policy == "binary_unsafe":    # never speculates
+                    if correct:
+                        lost += 1                  # forgoes EVERY correct win
+                else:                              # ternary: per-call condition
+                    if safe_now:
+                        pass                       # speculate, provably safe
+                    elif correct:
+                        lost += 1                  # deferred a lucky win
+            wc.append(cost)
+            forgone.append(lost)
+        out[policy] = {"weighted_cost_mean": round(st.mean(wc), 1),
+                       "forgone_safe_wins_mean": round(st.mean(forgone), 1)}
+    result = {"design": {"n": n, "cond_frac": cond_frac, "seeds": list(seeds),
+                         "tools": {k: {"severity": v[0], "p_safe": v[1]}
+                                   for k, v in COND_TOOLS.items()}},
+              "policies": out,
+              "reading": ("binary_safe fires irreversible actions whenever the"
+                          " per-call condition fails; binary_unsafe is safe but"
+                          " forgoes every conditional win; the ternary gate"
+                          " (per-call condition check) keeps cost 0 while"
+                          " retaining the conditional wins binary_unsafe"
+                          " loses.")}
+    (RESULTS / "safety_conditional.json").write_text(
+        json.dumps(result, indent=2))
+    print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

harness/simulate.py

@@ -0,0 +1,139 @@
+"""Build simulated users and ordered sessions from BFCL tasks.
+
+The persistence/lifecycle claim (ToolSpec's own flagged gap) is about a memory
+that must keep absorbing tool-use patterns *introduced after* the datastore was
+first built. A frozen datastore cannot contain tools/workflows it never saw; an
+online personal memory incorporates them. To test this we introduce each user's
+signature tasks PROGRESSIVELY across sessions: roughly half are "known" (present
+at warmup, so the frozen static datastore has them) and half are "novel" (first
+appear only in later sessions, so the frozen store has never seen them). Novel
+tasks recur after their introduction -- the recurring, evolving per-user usage a
+persistent memory is meant to exploit.
+"""
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass
+from typing import Any
+
+from .data import Task, perturb_numeric
+
+
+@dataclass
+class Instance:
+    user_id: str
+    session: int          # 0-indexed session number
+    query: str            # (possibly perturbed) natural-language request
+    functions: list[dict[str, Any]]
+    signature_id: str     # which signature task this instance came from
+    novel: bool           # True if this task was introduced after warmup
+
+
+def build_users(tasks: list[Task], n_users: int, tasks_per_user: int,
+                n_sessions: int, queries_per_session: int,
+                warmup_frac: float = 0.5, seed: int = 0,
+                perturb_prob: float = 1.0, arrival: str = "spread",
+                novel_weight: float = 3.0, overlap_frac: float = 0.0,
+                user_consistent: bool = False) -> list[Instance]:
+    """Assign disjoint signature tasks to users and roll out their sessions.
+
+    Each signature task j gets an introduction session: the first
+    ``n_known = round(tasks_per_user * warmup_frac)`` are introduced at session
+    0 (warmup); the remaining "novel" tasks are introduced one-by-one across the
+    later sessions. A session issues queries drawn only from tasks introduced so
+    far, biased toward the most recently introduced (novel) task so the novel
+    load actually shows up in the metric.
+    """
+    rng = random.Random(seed)
+    # Separate rng for the perturb coin so perturb_prob=1.0 reproduces the
+    # legacy stream bit-for-bit (no extra draws on the main rng).
+    coin = random.Random(seed + 991)
+    pool = tasks[:]
+    rng.shuffle(pool)
+
+    # Phase 4.3: shared-user-task overlap. overlap_frac=0.0 is the legacy
+    # fully-disjoint assignment (byte-identical path below); >0 makes
+    # n_shared = round(tasks_per_user*overlap_frac) of every user's signature
+    # tasks come from ONE common pool that all users reuse (same templates,
+    # per-occurrence perturbation), the rest from disjoint private blocks. This
+    # stress-tests personalization: at high overlap a global store sees many
+    # users' calls for the same template.
+    overlap_mode = user_consistent or overlap_frac > 0.0
+    if overlap_mode:
+        n_shared = min(tasks_per_user, round(tasks_per_user * overlap_frac))
+        n_private = tasks_per_user - n_shared
+        need = tasks_per_user + n_users * n_private   # shared pool + privates
+        if len(pool) < need:
+            raise ValueError(f"need {need} tasks, have {len(pool)}")
+        shared_pool = pool[:tasks_per_user][:n_shared]
+        priv_pool = pool[tasks_per_user:]
+    else:
+        need = n_users * tasks_per_user
+        if len(pool) < need:
+            raise ValueError(f"need {need} tasks, have {len(pool)}")
+
+    n_known = max(1, round(tasks_per_user * warmup_frac))
+    n_novel = tasks_per_user - n_known
+    # introduction session for each novel task, spread over sessions 1..n-1
+    if n_novel > 0:
+        if arrival == "burst":            # all novel tasks arrive at session 2
+            novel_intro = [2] * n_novel
+        elif arrival == "late":           # all arrive late (session n-3)
+            novel_intro = [max(1, n_sessions - 3)] * n_novel
+        else:                              # "spread" (legacy): one per session
+            step = max(1, (n_sessions - 1) // (n_novel + 1))
+            novel_intro = [min(n_sessions - 1, 1 + step * (k + 1))
+                           for k in range(n_novel)]
+    else:
+        novel_intro = []
+    intro = [0] * n_known + novel_intro     # per signature-task intro session
+
+    instances: list[Instance] = []
+    cursor = 0
+    for u in range(n_users):
+        uid = f"user_{u:02d}"
+        user_fixed_q = None
+        if overlap_mode:
+            priv = priv_pool[u * n_private:(u + 1) * n_private]
+            sig = list(shared_pool) + list(priv)   # shared slots first
+            # Each user gets a FIXED argument realization per signature task
+            # (user A always NYC, user B always Boston), consistent across the
+            # user's sessions but differing across users — the premise
+            # personalization exploits. Shared templates thus map to different
+            # concrete queries per user; a global store mixes them.
+            urng = random.Random(seed * 100003 + u)
+            user_fixed_q = [perturb_numeric(t.query, urng) for t in sig]
+        else:
+            sig = pool[cursor:cursor + tasks_per_user]
+            cursor += tasks_per_user
+        novel_flags = [False] * n_known + [True] * n_novel
+        for s in range(n_sessions):
+            active = [(sig[j], novel_flags[j], intro[j])
+                      for j in range(tasks_per_user) if intro[j] <= s]
+            # map each active signature task back to its slot index (for the
+            # per-user fixed-query lookup in the overlap experiment)
+            active_idx = [j for j in range(tasks_per_user) if intro[j] <= s]
+            if s == 0:
+                chosen = [(j, novel_flags[j]) for j in active_idx]  # once each
+            else:
+                weights = [novel_weight if intro[j] == s else 1.0
+                           for j in active_idx]
+                chosen = []
+                for _ in range(queries_per_session):
+                    j = rng.choices(active_idx, weights=weights, k=1)[0]
+                    chosen.append((j, novel_flags[j]))
+            for j, nv in chosen:
+                t = sig[j]
+                if user_fixed_q is not None:
+                    q = user_fixed_q[j]                 # user-consistent args
+                elif s == 0:
+                    q = t.query
+                elif perturb_prob >= 1.0:
+                    q = perturb_numeric(t.query, rng)   # legacy path, exact
+                elif perturb_prob <= 0.0:
+                    q = t.query
+                else:
+                    q = (perturb_numeric(t.query, rng)
+                         if coin.random() < perturb_prob else t.query)
+                instances.append(Instance(uid, s, q, t.functions, t.id, nv))
+    return instances

harness/tau2_extract.py

@@ -0,0 +1,142 @@
+"""Extract a frozen-trajectory decision-point pool from tau2-bench.
+
+tau2-bench (Barres et al., 2025; arXiv:2506.07982) is dual-control: a user
+simulator drives a live conversation, so naively running it per-arm would give
+each memory policy a different conversation and break the controlled
+comparison. Instead (PI-specified design) we FREEZE the conversation: the
+repo ships reference trajectories (data/tau2/results/final/*, gpt-4.1 agent x
+gpt-4.1 user simulator, 4 trials/task) and we extract every assistant
+tool-call decision point as a standalone task:
+
+    query    = rendered transcript prefix (last N turns before the call)
+    functions = the domain's full tool registry (parsed from the repo's
+                tools.py docstrings/signatures)
+
+Every arm then sees the identical frozen context; the target at each decision
+point is generated once by OUR served model (greedy), exactly like the
+BFCL/Seal-Tools runs — the reference agent's own call is NOT used as the
+target, only its conversation is reused as the frozen context.
+
+Inputs (downloaded from github.com/sierra-research/tau2-bench @ main):
+    <scratch>/tau2_{airline,retail}_ref.json     reference trajectory files
+    <scratch>/tau2_{airline,retail}_tools.py     domain toolkit sources
+Output:
+    data/tau2/decision_points.jsonl  (one task per line)
+    data/tau2/tools_{airline,retail}.json
+"""
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+OUT_DIR = Path(__file__).resolve().parent.parent / "data" / "tau2"
+CTX_TURNS = 6           # transcript turns kept before the decision point
+TOOL_RESULT_TRUNC = 160  # chars of tool output kept in the transcript
+
+_PY2JSON = {"str": "string", "int": "integer", "float": "number",
+            "bool": "boolean"}
+
+
+def _ann_to_json(ann: str) -> dict:
+    ann = (ann or "str").strip()
+    if ann.startswith(("List", "list")):
+        return {"type": "array", "items": {"type": "string"}}
+    if ann.startswith(("Dict", "dict")) or (ann[:1].isupper()
+                                            and ann not in _PY2JSON):
+        return {"type": "object"}
+    return {"type": _PY2JSON.get(ann, "string")}
+
+
+def parse_tools(src: str) -> list[dict]:
+    """Parse @is_tool-decorated methods into BFCL-style schemas (via ast)."""
+    import ast
+    tree = ast.parse(src)
+    tools = []
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.FunctionDef):
+            continue
+        deco = [d for d in node.decorator_list
+                if isinstance(d, ast.Call) and getattr(d.func, "id", "")
+                == "is_tool"]
+        if not deco:
+            continue
+        doc = ast.get_docstring(node) or ""
+        desc = doc.strip().split("\n\n")[0].strip().replace("\n", " ")
+        arg_docs = dict(re.findall(r"^\s*(\w+): (.+)$", doc, re.M))
+        args = [a for a in node.args.args if a.arg != "self"]
+        n_defaults = len(node.args.defaults)
+        props, required = {}, []
+        for i, a in enumerate(args):
+            ann = ast.unparse(a.annotation) if a.annotation else "str"
+            jt = _ann_to_json(ann)
+            jt["description"] = arg_docs.get(a.arg, "").strip()
+            props[a.arg] = jt
+            if i < len(args) - n_defaults:
+                required.append(a.arg)
+        tools.append({"name": node.name, "description": desc,
+                      "parameters": {"type": "dict", "properties": props,
+                                     "required": required}})
+    return tools
+
+
+def render_prefix(messages: list[dict], upto: int) -> str:
+    """Render the last CTX_TURNS messages before index `upto` as a transcript."""
+    lines = []
+    for m in messages[max(0, upto - CTX_TURNS):upto]:
+        role, content = m.get("role"), m.get("content")
+        if role == "user":
+            lines.append(f"User: {content}")
+        elif role == "assistant":
+            if m.get("tool_calls"):
+                c = m["tool_calls"][0]
+                args = json.dumps(c.get("arguments", {}), sort_keys=True)
+                lines.append(f"Agent called {c['name']}({args})")
+            elif content:
+                lines.append(f"Agent: {content}")
+        elif role == "tool":
+            c = str(content)[:TOOL_RESULT_TRUNC]
+            lines.append(f"Tool result: {c}")
+    return "\n".join(lines)
+
+
+def extract(domain: str, ref_path: Path, tools_path: Path):
+    tools = parse_tools(tools_path.read_text())
+    print(f"[{domain}] parsed {len(tools)} tool schemas")
+    d = json.loads(ref_path.read_text())
+    rows, seen = [], set()
+    for sim in d["simulations"]:
+        msgs = sim["messages"]
+        for i, m in enumerate(msgs):
+            if m.get("role") != "assistant" or not m.get("tool_calls"):
+                continue
+            q = f"[{domain} customer service]\n" + render_prefix(msgs, i)
+            if len(q) < 80 or q in seen:      # skip empty/duplicate contexts
+                continue
+            seen.add(q)
+            rows.append({"id": f"tau2_{domain}_{sim['id']}_{i}",
+                         "domain": domain, "query": q})
+    print(f"[{domain}] {len(rows)} unique decision points")
+    return tools, rows
+
+
+def main():
+    scratch = Path(sys.argv[1])
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+    all_rows = []
+    for domain in ("airline", "retail"):
+        tools, rows = extract(domain,
+                              scratch / f"tau2_{domain}_ref.json",
+                              scratch / f"tau2_{domain}_tools.py")
+        (OUT_DIR / f"tools_{domain}.json").write_text(
+            json.dumps(tools, indent=1))
+        all_rows.extend(rows)
+    with open(OUT_DIR / "decision_points.jsonl", "w") as f:
+        for r in all_rows:
+            f.write(json.dumps(r) + "\n")
+    print(f"total: {len(all_rows)} decision points -> {OUT_DIR}")
+
+
+if __name__ == "__main__":
+    main()

harness/tau2_live.py

@@ -0,0 +1,323 @@
+"""LIVE tau2-bench integration.
+
+Design: generate ONE canonical trace per task by running the REAL served model
+(gpt-oss-120b) as the agent inside tau2-bench's own dual-control loop, with a
+live GPT-4.1 user simulator. The trace is then frozen: every memory arm replays
+the identical conversation, drafting at each of the agent's tool-call decision
+points, scored by token-LCP against the call the served model actually made at
+that point in the canonical trace. Both the visited states AND the targets are
+therefore the served model's own on-policy behavior. (Do NOT instead extract
+decision points from tau2-bench's shipped reference trajectories: those were
+generated with GPT-4.1 as the agent, which substitutes another model's calls
+as targets — an off-policy bug.)
+
+Modes
+-----
+generate   Run tau2-bench live (subprocess -> `tau2 run`). REQUIRES a real
+           OPENAI_API_KEY (and optionally OPENAI_BASE_URL) in the environment
+           for the GPT-4.1 user simulator; hard-fails without it.
+           Agent side is the self-hosted model (api_base override, free).
+extract    Parse the generated results.json into decision-point tasks with
+           per-point targets (the trace's own canonicalized agent calls).
+           No model or API calls.
+
+Environment
+-----------
+TAU2_BIN       path to the `tau2` executable from the tau2-bench install
+               (default: `tau2` on PATH).
+TAU2_DATA_DIR  path to tau2-bench's data/ directory.
+OPENAI_API_KEY / OPENAI_BASE_URL  credentials for the user-simulator LLM.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+from . import metrics
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+TAU2_BIN = os.environ.get("TAU2_BIN", "tau2")
+TRACE_DIR = RESULTS / "tau2_live_traces"
+
+DOMAINS = ("airline", "retail", "telecom")
+USER_SIM_LLM = "gpt-4.1"
+
+
+def _load_user_sim_env() -> dict:
+    """Build the subprocess env for the user-simulator LLM credentials.
+
+    The user-sim LLM gets its key/base ONLY via environment variables, never
+    via --user-llm-args: tau2 saves llm_args verbatim into results.json, so
+    anything secret placed there would leak into a tracked artifact.
+    """
+    key = os.environ.get("OPENAI_API_KEY", "")
+    if not key or key.lower() in ("dummy", "none", "placeholder"):
+        raise SystemExit(
+            "generate requires a real OPENAI_API_KEY in the environment for "
+            "the GPT-4.1 user simulator. Refusing to run without it.")
+    env = dict(os.environ)
+    base = os.environ.get("OPENAI_BASE_URL", "")
+    if base:
+        env["OPENAI_API_BASE"] = base
+    return env
+
+
+def _leak_check(path: Path, env: dict):
+    """Abort loudly if the API key ever appears in a results artifact."""
+    key = env["OPENAI_API_KEY"]
+    for f in path.rglob("*.json"):
+        if key in f.read_text():
+            raise SystemExit(f"SECRET LEAK: API key found in {f} — "
+                             "do NOT commit; scrub before proceeding.")
+
+
+def generate(args):
+    env = _load_user_sim_env()
+    agent_args = json.dumps({"temperature": 0.0, "api_base": args.url,
+                             "api_key": "dummy"})
+    user_args = json.dumps({"temperature": 0.0})   # creds via env ONLY
+    TRACE_DIR.mkdir(parents=True, exist_ok=True)
+    for domain in args.domains:
+        save = TRACE_DIR / domain
+        cmd = [TAU2_BIN, "run", "--domain", domain,
+               "--agent-llm", f"openai/{args.model}",
+               "--agent-llm-args", agent_args,
+               "--user-llm", f"openai/{USER_SIM_LLM}",
+               "--user-llm-args", user_args,
+               "--num-trials", "1",
+               "--max-concurrency", str(args.concurrency),
+               "--save-to", str(save)]
+        if args.num_tasks:
+            cmd += ["--num-tasks", str(args.num_tasks)]
+        print("[tau2_live] running:", " ".join(cmd), flush=True)
+        subprocess.run(cmd, check=True, env=env)
+        _leak_check(save, env)
+    print(f"[tau2_live] canonical traces -> {TRACE_DIR} (leak check passed)",
+          flush=True)
+
+
+def extract(args):
+    """OUR traces -> decision-point tasks + on-policy targets. No API calls."""
+    from .tau2_extract import render_prefix   # transcript rendering reused
+    out_rows = []
+    for domain in args.domains:
+        f = TRACE_DIR / domain / "results.json"
+        if not f.exists():
+            print(f"[tau2_live] no trace for {domain} ({f}), skipping")
+            continue
+        d = json.loads(f.read_text())
+        agent_llm = d["info"]["agent_info"]["llm"]
+        assert args.model in agent_llm, (
+            f"trace agent is {agent_llm}, expected {args.model} — refusing "
+            "to extract off-policy states")
+        for sim in d["simulations"]:
+            msgs = sim["messages"]
+            for i, m in enumerate(msgs):
+                if m.get("role") != "assistant" or not m.get("tool_calls"):
+                    continue
+                call = m["tool_calls"][0]
+                target = metrics.canonical_call_str(call["name"],
+                                                    call.get("arguments", {}))
+                q = f"[{domain} customer service]\n" + render_prefix(msgs, i)
+                if len(q) < 80:
+                    continue
+                out_rows.append({
+                    "id": f"tau2live_{domain}_{sim['id']}_{i}",
+                    "domain": domain, "query": q, "target": target})
+    out = RESULTS / "tau2_live_decision_points.jsonl"
+    with open(out, "w") as fh:
+        for r in out_rows:
+            fh.write(json.dumps(r) + "\n")
+    print(f"[tau2_live] {len(out_rows)} on-policy decision points -> {out}")
+    print("Next: score with the standard users/sessions replay, using each "
+          "point's cached 'target' (no target generation needed — the trace "
+          "IS the served model's behavior).")
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("mode", choices=["generate", "extract", "score"])
+    p.add_argument("--domains", nargs="+", default=list(DOMAINS))
+    p.add_argument("--url", default="http://localhost:30000/v1")
+    p.add_argument("--model", default="gpt-oss-120b")
+    p.add_argument("--model-path", default="",
+                   help="tokenizer path for cross-model scoring; empty -> "
+                        "gpt-oss tokenizer + cached trace targets")
+    p.add_argument("--workers", type=int, default=16,
+                   help="target-generation concurrency (cross-model score)")
+    p.add_argument("--rendered-targets", action="store_true",
+                   help="force the cross-model path (fresh greedy targets "
+                        "over rendered states) even for gpt-oss-120b, so all "
+                        "three models share one protocol")
+    p.add_argument("--max-none-frac", type=float, default=0.20,
+                   help="abort if the None-target fraction exceeds this; "
+                        "raise ONLY with probe-verified justification that "
+                        "the Nones are genuine model declines, not server "
+                        "failures")
+    p.add_argument("--toolspec", action="store_true",
+                   help="add the Phase-4.4 faithful ToolSpec baseline as a "
+                        "4th arm (frozen global kNN-vote + schema-aware FSM)")
+    p.add_argument("--concurrency", type=int, default=4)
+    p.add_argument("--num-tasks", type=int, default=0,
+                   help="limit tasks per domain (0 = all; use for pilots)")
+    args = p.parse_args()
+    {"generate": generate, "extract": extract, "score": score}[args.mode](args)
+
+
+
+
+def score(args):
+    """3-arm replay over the live decision points.
+
+    Default (gpt-oss-120b): targets are cached from the canonical traces (the
+    served model's own on-policy calls); perturb_prob=0 so every query is
+    verbatim and pre-cached — no model/API calls at all.
+
+    Cross-model (--model gemma-4-31B-it --model-path <tokenizer> --url <srv>):
+    each decision-point STATE stays frozen (it comes from the gpt-oss canonical
+    conversations — off-policy states for this model, flag in any report), but
+    the TARGET is generated fresh by the scored model itself (greedy, $0,
+    self-hosted) so targets remain on-policy per model — the same discipline as
+    the BFCL/Seal-Tools cross-model runs and NOT the rejected shipped-trace
+    design (which substituted another model's calls as targets). Targets are
+    file-cached per model so re-runs are GPU-free."""
+    from collections import defaultdict
+    from .data import Task
+    from .memory import (Embedder, NoMemory, PersonalMemory, StaticGlobal,
+                         ToolSpecBaseline)
+    from .run_accept import _parse_target
+    from .simulate import build_users
+    dp = [json.loads(l) for l in
+          (RESULTS / "tau2_live_decision_points.jsonl").read_text()
+          .splitlines()]
+    dp = [r for r in dp if r["domain"] in args.domains]
+    tools = {d: json.loads((ROOT / "data" / "tau2" /
+                            f"tools_{d}.json").read_text())
+             for d in args.domains}
+    tasks = [Task(id=r["id"], query=r["query"],
+                  functions=tools[r["domain"]], origin_id=r["id"])
+             for r in dp]
+    crossmodel = args.model != "gpt-oss-120b" or args.rendered_targets
+    if crossmodel:
+        assert args.model_path, "--model-path (tokenizer) required for " \
+            "cross-model scoring"
+        from .run_accept import generate_targets
+        from .client import ToolClient
+        cache_f = RESULTS / f"tau2_live_targets_{args.model}.json"
+        cache = (json.loads(cache_f.read_text()) if cache_f.exists() else {})
+        missing = [t for t in tasks if t.query not in cache]
+        if missing:
+            client = ToolClient(url=args.url, model=args.model)
+            assert client.ping(), f"{args.model} not reachable at {args.url}"
+            print(f"[tau2_live] cross-model targets: {len(missing)} to "
+                  f"generate ({len(cache)} cached)", flush=True)
+            # chunked so a mid-run server failure loses <=1 chunk of work
+            CHUNK = 100
+            for c0 in range(0, len(missing), CHUNK):
+                fresh = generate_targets(client, missing[c0:c0 + CHUNK],
+                                         workers=args.workers)
+                cache.update(fresh)
+                cache_f.write_text(json.dumps(cache))
+                print(f"[tau2_live] cache checkpoint: {len(cache)} targets",
+                      flush=True)
+        targets = cache
+        n_none = sum(1 for t in tasks if targets.get(t.query) is None)
+        print(f"[tau2_live] {args.model}: {n_none}/{len(tasks)} no-call "
+              "states (excluded from all arms symmetrically)", flush=True)
+        # Sanity gate: generate_call returns None BOTH for a genuine decline
+        # and for a hard failure (server wedge/timeouts). A wedged server
+        # silently converts the tail of the pool into fake declines and
+        # biases the run, so refuse to score when the None fraction exceeds
+        # --max-none-frac UNLESS the Nones were probe-verified as genuine
+        # (e.g. nemotron deliberates in text on rendered multi-turn states —
+        # probe-verified as genuine declines).
+        if n_none > args.max_none_frac * len(tasks):
+            raise SystemExit(
+                f"ABORT: {n_none}/{len(tasks)} None targets for {args.model} "
+                f"exceeds --max-none-frac={args.max_none_frac}. If a probe "
+                "confirms genuine declines (not server failure), re-run with "
+                "a higher --max-none-frac; otherwise fix the server, delete "
+                f"the poisoned nulls from {cache_f.name}, and re-run.")
+        metrics.get_tokenizer(args.model_path)
+    else:
+        targets = {r["query"]: r["target"] for r in dp}
+        metrics.get_tokenizer(
+            os.environ.get("SPECMEM_TOKENIZER", "openai/gpt-oss-120b"))
+    embedder = Embedder()
+    per_seed = {}
+    for sd in (0, 1, 2):
+        instances = build_users(tasks, n_users=40, tasks_per_user=15,
+                                n_sessions=12, queries_per_session=6,
+                                seed=sd, perturb_prob=0.0)
+        instances.sort(key=lambda x: (x.session, x.user_id))
+        arms = [NoMemory(), StaticGlobal(),
+                PersonalMemory(capacity=48, eviction="lru")]
+        if getattr(args, "toolspec", False):
+            arms.append(ToolSpecBaseline())
+        agg = {a.name: defaultdict(list) for a in arms}
+        cur = -1
+        for ins in instances:
+            tgt = targets.get(ins.query)
+            if tgt is None:
+                continue
+            if ins.session != cur:
+                cur = ins.session
+                if cur == 1:
+                    for a in arms:
+                        if hasattr(a, "freeze"):
+                            a.freeze()
+            for a in arms:
+                agg[a.name][ins.session].append(metrics.score(
+                    a.draft(ins.query, ins.functions, ins.user_id, embedder),
+                    tgt))
+            cn, ca = _parse_target(tgt)
+            for a in arms[1:]:
+                a.observe(ins.query, ins.functions, ins.user_id, cn, ca,
+                          embedder)
+                if isinstance(a, PersonalMemory) and ins.session == 0:
+                    a.seed_shared(ins.query, cn, ca, embedder)
+        per_seed[sd] = {
+            n: {"MAT": round(sum(x["accept_length"] for s, xs in v.items()
+                                 if s > 0 for x in xs) /
+                             max(1, sum(len(xs) for s, xs in v.items()
+                                        if s > 0)), 3),
+                "n": sum(len(xs) for s, xs in v.items() if s > 0)}
+            for n, v in agg.items()}
+        print(f"seed {sd}:", per_seed[sd], flush=True)
+    import statistics as st
+    summary = {}
+    for arm in per_seed[0]:
+        vals = [per_seed[sd][arm]["MAT"] for sd in per_seed]
+        summary[arm] = {"MAT_mean": round(st.mean(vals), 3),
+                        "MAT_std": round(st.pstdev(vals), 3),
+                        "n": per_seed[0][arm]["n"]}
+    g = summary["personal_memory"]["MAT_mean"]
+    s0 = summary["static_global"]["MAT_mean"]
+    out = {"config": {"pool": len(tasks), "domains": sorted(args.domains),
+                      "model": args.model,
+                      "users": 40, "tasks_per_user": 15,
+                      "sessions": 12, "seeds": [0, 1, 2], "perturb_prob": 0.0,
+                      "targets": ("on-policy per scored model (fresh greedy "
+                                  "calls over frozen gpt-oss trace states — "
+                                  "off-policy STATES, on-policy targets)"
+                                  if crossmodel else
+                                  "on-policy from live canonical traces")},
+           "per_seed": per_seed, "summary": summary,
+           "gap_pct": round(100 * (g - s0) / s0, 1)}
+    suffix = ("" if sorted(args.domains) == sorted(DOMAINS)
+              else "_" + "_".join(sorted(args.domains)))
+    if crossmodel:
+        suffix += f"_{args.model}"
+    (RESULTS / f"tau2_live_accept_results{suffix}.json").write_text(
+        json.dumps(out, indent=2))
+    print(json.dumps({"summary": summary, "gap_pct": out["gap_pct"]},
+                     indent=1))
+
+
+if __name__ == "__main__":
+    main()

harness/ttl_arm.py

@@ -0,0 +1,111 @@
+"""TTL-windowed lifecycle baseline (external-review item 2).
+
+A stronger lifecycle baseline between "frozen" and "live-with-full-write-back":
+the store still ingests live, but every entry EXPIRES after `ttl` sessions
+regardless of LRU capacity pressure — i.e., bounded staleness by construction.
+If a short TTL matches full live memory, most of the live benefit is staleness
+decay; if it costs acceptance, cross-session persistence of older-but-recurring
+calls matters beyond mere freshness. Replays the phase-2 seed-0 stream against
+cached targets (no GPU/model calls).
+
+Usage: python -m harness.ttl_arm   (from code/)
+Writes results/phase2_ttl_arm.json.
+"""
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from pathlib import Path
+
+from . import metrics
+from .data import load_bfcl
+from .memory import Embedder, PersonalMemory, StaticGlobal
+from .run_accept import MODEL_PATH, _parse_target
+from .simulate import build_users
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+
+TTLS = (1, 2, 4, 8)
+
+
+class TTLPersonalMemory(PersonalMemory):
+    """PersonalMemory whose entries expire after `ttl` sessions."""
+
+    def __init__(self, ttl: int, capacity: int = 48):
+        super().__init__(capacity=capacity, eviction="lru")
+        self.ttl = ttl
+        self.name = f"personal_ttl{ttl}"
+        self._entry_session: dict[int, int] = {}
+        self._session = 0
+
+    def tick_session(self, session: int) -> None:
+        self._session = session
+        cutoff = session - self.ttl
+        for store in self.stores.values():
+            for eid in [e for e in store
+                        if self._entry_session.get(e, 0) < cutoff]:
+                del store[eid]
+
+    def observe(self, query, functions, user_id, name, args, embedder):
+        eid_before = self._next_id
+        super().observe(query, functions, user_id, name, args, embedder)
+        for eid in range(eid_before, self._next_id):
+            self._entry_session[eid] = self._session
+
+
+def main():
+    metrics.get_tokenizer(MODEL_PATH)
+    tasks = load_bfcl()
+    embedder = Embedder()
+    instances = build_users(tasks, n_users=40, tasks_per_user=15,
+                            n_sessions=12, queries_per_session=6, seed=0)
+    instances.sort(key=lambda x: (x.session, x.user_id))
+    targets = json.loads((RESULTS / "phase2_targets_seed0.json").read_text())
+
+    static = StaticGlobal()
+    live = PersonalMemory(capacity=48, eviction="lru")
+    ttl_arms = [TTLPersonalMemory(t) for t in TTLS]
+    arms = [("static", static), ("personal_live", live)] + \
+           [(a.name, a) for a in ttl_arms]
+
+    agg = {n: defaultdict(list) for n, _ in arms}
+    cur = -1
+    for ins in instances:
+        tgt = targets.get(ins.query)
+        if tgt is None:
+            continue
+        if ins.session != cur:
+            cur = ins.session
+            if cur == 1:
+                static.freeze()
+            for a in ttl_arms:
+                a.tick_session(cur)
+        for n, a in arms:
+            agg[n][ins.session].append(
+                metrics.score(a.draft(ins.query, ins.functions, ins.user_id,
+                                      embedder), tgt))
+        cname, cargs = _parse_target(tgt)
+        for n, a in arms:
+            a.observe(ins.query, ins.functions, ins.user_id, cname, cargs,
+                      embedder)
+            if isinstance(a, PersonalMemory) and ins.session == 0:
+                a.seed_shared(ins.query, cname, cargs, embedder)
+
+    out = {}
+    for n, _ in arms:
+        scores = [x for s, xs in agg[n].items() if s > 0 for x in xs]
+        out[n] = {"n": len(scores),
+                  "MAT": round(sum(x["accept_length"] for x in scores)
+                               / len(scores), 3),
+                  "exact_rate": round(sum(1 for x in scores if x["exact"])
+                                      / len(scores), 4)}
+    result = {"config": {"seed": 0, "capacity": 48, "ttls": list(TTLS),
+                         "targets": "phase2_targets_seed0.json (cached)"},
+              "post_warmup": out}
+    (RESULTS / "phase2_ttl_arm.json").write_text(json.dumps(result, indent=2))
+    print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

harness/wallclock.py

@@ -0,0 +1,95 @@
+"""Scoped wall-clock micro-benchmark (external-review item 3).
+
+Not a production deployment study. Measures, on the real infra:
+  (a) draft-side overhead: embed(query) + nearest-neighbor retrieval against a
+      realistically-sized personal store (~48 entries) — timed on this CPU node;
+  (b) target-side decode speed of the served gpt-oss-120b: per-token decode
+      time from paired short/long generations (sequential, unbatched);
+then compares mean retrieval overhead against the decode time represented by
+the accepted-token gap (personal 19.96 vs static 15.70 MAT, headline run).
+Writes results/wallclock_micro.json.
+"""
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+
+import numpy as np
+import requests
+
+from .data import load_bfcl
+from .memory import Embedder, _best_match, Entry
+
+ROOT = Path(__file__).resolve().parent.parent
+RESULTS = ROOT / "results"
+URL = "http://localhost:30000/v1/chat/completions"
+N_RETRIEVAL = 200
+N_DECODE_PAIRS = 30
+
+
+def main():
+    tasks = load_bfcl()[:N_RETRIEVAL]
+    emb = Embedder()
+    # warm the embedder, then build a 48-entry store
+    store = [Entry(emb.embed(t.query), "x") for t in tasks[:48]]
+    for t in tasks[:20]:
+        emb.embed(t.query + " warm")
+    lat = []
+    for t in tasks:
+        q = t.query + " ?"                      # bust the embed cache
+        t0 = time.perf_counter()
+        e = emb.embed(q)
+        _best_match(e, store)
+        lat.append((time.perf_counter() - t0) * 1000)
+    lat.sort()
+    retr = {"n": len(lat), "mean_ms": round(float(np.mean(lat)), 2),
+            "median_ms": round(lat[len(lat) // 2], 2),
+            "p95_ms": round(lat[int(0.95 * len(lat))], 2)}
+
+    def timed_gen(prompt, max_tok):
+        t0 = time.perf_counter()
+        r = requests.post(URL, json={
+            "model": "gpt-oss-120b", "temperature": 0.0,
+            "max_tokens": max_tok,
+            "messages": [{"role": "user", "content": prompt}]}, timeout=120)
+        dt = time.perf_counter() - t0
+        u = r.json().get("usage", {})
+        return dt, u.get("completion_tokens", max_tok)
+
+    per_tok = []
+    for t in tasks[:N_DECODE_PAIRS]:
+        p = "Write a long detailed paragraph about: " + t.query[:200]
+        d1, n1 = timed_gen(p, 8)
+        d2, n2 = timed_gen(p, 168)
+        if n2 > n1:
+            per_tok.append((d2 - d1) / (n2 - n1) * 1000)
+    per_tok.sort()
+    dec = {"n_pairs": len(per_tok),
+           "per_token_ms_median": round(per_tok[len(per_tok) // 2], 2),
+           "per_token_ms_mean": round(float(np.mean(per_tok)), 2)}
+
+    ptok = dec["per_token_ms_median"]
+    out = {
+        "retrieval_overhead": retr,
+        "decode": dec,
+        "comparison": {
+            "personal_MAT_19.96_decode_ms": round(19.96 * ptok, 1),
+            "static_MAT_15.70_decode_ms": round(15.70 * ptok, 1),
+            "gap_4.26_tokens_decode_ms": round(4.26 * ptok, 1),
+            "retrieval_overhead_mean_ms": retr["mean_ms"],
+            "overhead_over_gap_pct": round(
+                100 * retr["mean_ms"] / (4.26 * ptok), 1),
+        },
+        "caveats": ("Sequential unbatched decode on the live server (batching "
+                    "changes per-token time); retrieval timed on the CPU node,"
+                    " single-threaded; no end-to-end speculative decoder is "
+                    "integrated — this bounds the overhead-vs-savings ratio, "
+                    "not deployed speedup."),
+    }
+    (RESULTS / "wallclock_micro.json").write_text(json.dumps(out, indent=2))
+    print(json.dumps(out, indent=2))
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,13 @@
+# Core harness dependencies (tested versions in parentheses).
+requests>=2.31                 # (2.32) OpenAI-compatible HTTP client
+numpy>=1.26                    # (2.x)  aggregation / bootstrap CI
+transformers>=4.44             # (5.6.0) tokenizer for the token-LCP accept metric
+sentence-transformers>=3.0     # (5.6.0) all-MiniLM-L6-v2 query embedder (CPU)
+torch>=2.3                     # (2.9.1) backend for the embedder / tokenizers
+matplotlib>=3.8                # figures only
+
+# Serving (not installed by this file — install in the serving environment):
+#   sglang  (tested 0.5.11)  for gpt-oss-120b and gemma-4-31B-it
+#   vllm    (tested 0.18)    for Nemotron-3-Super-120B
+# tau2 experiments additionally need tau2-bench:
+#   https://github.com/sierra-research/tau2-bench

scripts/download_data.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Download the benchmark data used by the SpecMem harness into data/.
+#
+# We do not redistribute any third-party benchmark; this script fetches each
+# file directly from its official source. See data/README.md for provenance
+# and licenses, and for the two datasets that need a manual step
+# (ToolBench: Google Drive; tau2-bench: generated from the tau2-bench repo).
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+DATA="$ROOT/data"
+GH_RAW="https://raw.githubusercontent.com"
+
+fetch() {  # fetch <url> <dest>
+    mkdir -p "$(dirname "$2")"
+    if [ -s "$2" ]; then echo "  [skip] $2 (exists)"; return; fi
+    echo "  [get ] $2"
+    curl -fsSL --retry 3 "$1" -o "$2"
+}
+
+echo "== BFCL v4 (Berkeley Function-Call Leaderboard, Apache-2.0) =="
+BFCL="$GH_RAW/ShishirPatil/gorilla/main/berkeley-function-call-leaderboard/bfcl_eval/data"
+for f in BFCL_v4_simple_python.json BFCL_v4_multiple.json BFCL_v4_parallel.json; do
+    fetch "$BFCL/$f" "$DATA/bfcl/$f"
+done
+
+echo "== Seal-Tools (Apache-2.0) =="
+SEAL="$GH_RAW/fairyshine/Seal-Tools/master/Seal-Tools_Dataset"
+fetch "$SEAL/tool.jsonl"            "$DATA/sealtools/tool.jsonl"
+fetch "$SEAL/test_in_domain.jsonl"  "$DATA/sealtools/test_in_domain.jsonl"
+
+echo "== ToolAlpaca (Apache-2.0) =="
+TA="$GH_RAW/tangqiaoyu/ToolAlpaca/main/data"
+fetch "$TA/eval_simulated.json" "$DATA/toolalpaca/eval_simulated.json"
+fetch "$TA/eval_real.json"      "$DATA/toolalpaca/eval_real.json"
+
+echo "== API-Bank (MIT; HF dataset liminghao1630/API-Bank) =="
+AB="https://huggingface.co/datasets/liminghao1630/API-Bank/resolve/main/test-data"
+fetch "$AB/level-1-api.json" "$DATA/apibank/level-1-api.json"
+fetch "$AB/level-2-api.json" "$DATA/apibank/level-2-api.json"
+
+echo
+echo "Done. Two datasets need a manual step (only for the corresponding experiments):"
+echo "  * ToolBench  -> data/toolbench/test_instruction/G{1,2,3}_*.json"
+echo "                  from the official OpenBMB/ToolBench Google Drive release."
+echo "  * tau2-bench -> data/tau2/tools_{airline,retail,telecom}.json"
+echo "                  generated from github.com/sierra-research/tau2-bench;"
+echo "                  see data/README.md."