| """Scoped wall-clock micro-benchmark (external-review item 3). |
| |
| Not a production deployment study. Measures, on the real infra: |
| (a) draft-side overhead: embed(query) + nearest-neighbor retrieval against a |
| realistically-sized personal store (~48 entries) — timed on this CPU node; |
| (b) target-side decode speed of the served gpt-oss-120b: per-token decode |
| time from paired short/long generations (sequential, unbatched); |
| then compares mean retrieval overhead against the decode time represented by |
| the accepted-token gap (personal 19.96 vs static 15.70 MAT, headline run). |
| Writes results/wallclock_micro.json. |
| """ |
| from __future__ import annotations |
|
|
| import json |
| import time |
| from pathlib import Path |
|
|
| import numpy as np |
| import requests |
|
|
| from .data import load_bfcl |
| from .memory import Embedder, _best_match, Entry |
|
|
| ROOT = Path(__file__).resolve().parent.parent |
| RESULTS = ROOT / "results" |
| URL = "http://localhost:30000/v1/chat/completions" |
| N_RETRIEVAL = 200 |
| N_DECODE_PAIRS = 30 |
|
|
|
|
| def main(): |
| tasks = load_bfcl()[:N_RETRIEVAL] |
| emb = Embedder() |
| |
| store = [Entry(emb.embed(t.query), "x") for t in tasks[:48]] |
| for t in tasks[:20]: |
| emb.embed(t.query + " warm") |
| lat = [] |
| for t in tasks: |
| q = t.query + " ?" |
| t0 = time.perf_counter() |
| e = emb.embed(q) |
| _best_match(e, store) |
| lat.append((time.perf_counter() - t0) * 1000) |
| lat.sort() |
| retr = {"n": len(lat), "mean_ms": round(float(np.mean(lat)), 2), |
| "median_ms": round(lat[len(lat) // 2], 2), |
| "p95_ms": round(lat[int(0.95 * len(lat))], 2)} |
|
|
| def timed_gen(prompt, max_tok): |
| t0 = time.perf_counter() |
| r = requests.post(URL, json={ |
| "model": "gpt-oss-120b", "temperature": 0.0, |
| "max_tokens": max_tok, |
| "messages": [{"role": "user", "content": prompt}]}, timeout=120) |
| dt = time.perf_counter() - t0 |
| u = r.json().get("usage", {}) |
| return dt, u.get("completion_tokens", max_tok) |
|
|
| per_tok = [] |
| for t in tasks[:N_DECODE_PAIRS]: |
| p = "Write a long detailed paragraph about: " + t.query[:200] |
| d1, n1 = timed_gen(p, 8) |
| d2, n2 = timed_gen(p, 168) |
| if n2 > n1: |
| per_tok.append((d2 - d1) / (n2 - n1) * 1000) |
| per_tok.sort() |
| dec = {"n_pairs": len(per_tok), |
| "per_token_ms_median": round(per_tok[len(per_tok) // 2], 2), |
| "per_token_ms_mean": round(float(np.mean(per_tok)), 2)} |
|
|
| ptok = dec["per_token_ms_median"] |
| out = { |
| "retrieval_overhead": retr, |
| "decode": dec, |
| "comparison": { |
| "personal_MAT_19.96_decode_ms": round(19.96 * ptok, 1), |
| "static_MAT_15.70_decode_ms": round(15.70 * ptok, 1), |
| "gap_4.26_tokens_decode_ms": round(4.26 * ptok, 1), |
| "retrieval_overhead_mean_ms": retr["mean_ms"], |
| "overhead_over_gap_pct": round( |
| 100 * retr["mean_ms"] / (4.26 * ptok), 1), |
| }, |
| "caveats": ("Sequential unbatched decode on the live server (batching " |
| "changes per-token time); retrieval timed on the CPU node," |
| " single-threaded; no end-to-end speculative decoder is " |
| "integrated — this bounds the overhead-vs-savings ratio, " |
| "not deployed speedup."), |
| } |
| (RESULTS / "wallclock_micro.json").write_text(json.dumps(out, indent=2)) |
| print(json.dumps(out, indent=2)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
