เจาะลึก Cactus Whistle: โมเดลเสียง 16.9 MB เล็กพริกขี้หนูสู้ Whisper ได้ไหม?

import os
import io
import re
import sys
import json
import math
import time
import random
import hashlib
import shutil
import zipfile
import platform
import traceback
import subprocess
import collections
import urllib.error
import urllib.request
 
RESULTS = {}
VERDICTS = []
 
def banner(title):
    print("\n" + "=" * 78)
    print(title)
    print("=" * 78)
 
def section(name):
    def wrap(fn):
        def run(*a, **kw):
            banner(name)
            t0 = time.time()
            try:
                out = fn(*a, **kw)
                RESULTS[name] = out if isinstance(out, str) else "ok"
                return out
            except Exception as e:
                RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
                print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
                traceback.print_exc(limit=3)
                return None
            finally:
                print(f"\n[{time.time() - t0:.0f} s]")
                json.dump({"results": RESULTS, "verdicts": VERDICTS}, open("results.json", "w"), indent=1)
        return run
    return wrap
 
banner("1. Install Whistle, pin its weights and engine by Hub revision, transcribe one clip")
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "cactus-needle==3.2.0", "openai-whisper==20250625",
                "jiwer==4.0.0", "soundfile==0.14.0", "soxr==1.1.0"], check=True)
os.environ["NEEDLE_TELEMETRY"] = "0"            # the package sends anonymous usage counts unless told not to
os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
 
import numpy as np
import soundfile as sf
import soxr
from huggingface_hub import hf_hub_download
 
# cactus-needle fetches three files on first use, from Hugging Face repos whose files are replaced in place: the 16.9 MB
# whistle.cact weights, the ~1 MB engine library (version 3.3.0 in this package; the needle3 repo received new binaries
# on 2026-10-10) and the 35 MB needle3.cact used in step 12. Pinning the Hub revision of each, and printing the hashes,
# is what makes this run reproducible. The engine version is read from the package, not typed in.
WHISTLE_REV = "ca5287601bef25af26dcf2e1b2bdc0843a7c19e5"   # Cactus-Compute/whistle, 2026-10-10
NEEDLE3_REV = "93e88ede867e4311c39151285e74df513e4b835b"   # Cactus-Compute/needle3 (engine + Needle 3 weights), 2026-10-10
from needle.agent import fetch as needle_fetch
TAG = needle_fetch._platform_tag()
ENGINE_VERSION = needle_fetch.engine_version(3)
weights = hf_hub_download("Cactus-Compute/whistle", "whistle.cact", revision=WHISTLE_REV)
wheel = hf_hub_download("Cactus-Compute/needle3", needle_fetch.engine_wheel(ENGINE_VERSION, TAG, 3), revision=NEEDLE3_REV)
with zipfile.ZipFile(wheel) as z:
    member = next(n for n in z.namelist() if n.startswith("needle/libneedle"))
    os.makedirs("engine", exist_ok=True)
    engine = os.path.abspath(os.path.join("engine", os.path.basename(member)))
    open(engine, "wb").write(z.read(member))
needle3_weights = hf_hub_download("Cactus-Compute/needle3", "needle3.cact", revision=NEEDLE3_REV)
os.makedirs(needle_fetch.cache_dir(3), exist_ok=True)           # Needle(tools=...) reads its weights from this cache path
shutil.copyfile(needle3_weights, os.path.join(needle_fetch.cache_dir(3), needle_fetch.base_weights(3)))
os.environ["NEEDLE_WHISTLE_WEIGHTS"] = weights   # both variables are honoured by the package
os.environ["NEEDLE3_LIB_PATH"] = engine
 
import needle
import importlib.metadata as md
sha = lambda p: hashlib.sha256(open(p, "rb").read()).hexdigest()[:12]
CPU = platform.processor() or platform.machine()
if sys.platform == "linux" and os.path.exists("/proc/cpuinfo"):
    CPU = next((l.split(":", 1)[1].strip() for l in open("/proc/cpuinfo") if l.startswith("model name")), CPU)
elif sys.platform == "darwin":
    CPU = subprocess.run(["sysctl", "-n", "machdep.cpu.brand_string"], capture_output=True, text=True).stdout.strip() or CPU
print(f"cactus-needle {needle.__version__} (engine {ENGINE_VERSION}, wheel tag {TAG})   python {platform.python_version()}   "
      f"soundfile {md.version('soundfile')}  soxr {md.version('soxr')}  numpy {np.__version__}")
print(f"hardware: {CPU}, {os.cpu_count()} cpus visible")
print(f"whistle.cact  {os.path.getsize(weights)/1e6:.1f} MB  sha256 {sha(weights)}   @ {WHISTLE_REV[:12]}")
print(f"engine        {os.path.getsize(engine)/1e6:.1f} MB  sha256 {sha(engine)}   @ {NEEDLE3_REV[:12]}")
print(f"needle3.cact  {os.path.getsize(needle3_weights)/1e6:.1f} MB  sha256 {sha(needle3_weights)}   @ {NEEDLE3_REV[:12]}")
 
SR = 16000
W = needle.Whistle()      # one model per process, not thread-safe; every later step reuses this object
 
def fetch(url, tries=7):
    """GET with retries. The datasets-server rate-limits anonymous callers (HTTP 429): honour Retry-After when it is
    sent, otherwise back off 5, 10, 20, 40 ... seconds, and never give up on the first refusal."""
    for k in range(tries):
        try:
            with urllib.request.urlopen(url, timeout=60) as r:
                return r.read()
        except urllib.error.HTTPError as e:
            if k == tries - 1:
                raise
            wait = int(e.headers.get("Retry-After", 0) or 0) if e.code == 429 else 0
            time.sleep(max(wait, 5 * 2 ** k) if e.code in (429, 500, 502, 503, 504) else 2)
        except Exception:
            if k == tries - 1:
                raise
            time.sleep(3 * (k + 1))
 
def page(dataset, config, split, offset, length=100):
    """One page of a Hub dataset through the datasets-server rows API: metadata plus signed audio URLs."""
    url = (f"https://datasets-server.huggingface.co/rows?dataset={dataset}&config={config}"
           f"&split={split}&offset={offset}&length={length}")
    time.sleep(0.25)                                  # stay under the anonymous rate limit when paging a whole split
    return json.loads(fetch(url))
 
def load_audio(row):
    """Download one row's clip; return 16 kHz mono float32 samples and the sha256 of the original file."""
    data = fetch(row["audio"][0]["src"])
    audio, sr = sf.read(io.BytesIO(data), dtype="float32")
    if audio.ndim > 1:
        audio = audio.mean(axis=1)
    if sr != SR:
        audio = soxr.resample(audio, sr, SR, quality="HQ")
    return audio, hashlib.sha256(data).hexdigest()[:12]
 
first = page("openslr/librispeech_asr", "clean", "test", 0, 1)["rows"][0]["row"]
clip, _ = load_audio(first)
t0 = time.perf_counter()
out = W.transcribe(clip, word_timestamps=True)
wall = (time.perf_counter() - t0) * 1000
print(f"\nclip {first['id']}  {len(clip)/SR:.1f} s   reference: {first['text']}")
print(f"-> {out['text']!r}")
print(f"   language {out['language']}   engine time to first token {out['ttft_ms']:.1f} ms   decode {out['decode_tps']:.0f} tok/s"
      f"   wall {wall:.0f} ms")
print("   first words:", [(w["word"], w["start"], w["end"], w["probability"]) for w in out["words"][:3]])
RESULTS["1. Install Whistle, pin its weights and engine by Hub revision, transcribe one clip"] = \
    f"{os.path.getsize(weights)/1e6:.1f} MB weights, {len(clip)/SR:.1f} s clip in {wall:.0f} ms"

import jiwer
import whisper
import torch
from whisper.normalizers import EnglishTextNormalizer, BasicTextNormalizer
 
NORM_EN = EnglishTextNormalizer()      # the normalisers Whistle's own benchmarks use; applied to both sides, always
NORM_BASIC = BasicTextNormalizer()
SEED = 2026
N_WHISTLE = 300                        # clips per LibriSpeech split that Whistle transcribes (it is cheap)
N_PAIRED = {"other": 100, "clean": 60}  # the first N of them also go through Whisper (it is not); test-other is primary
MAX_CLIP_S = 30.0                      # Whistle's single-pass window; longer clips are declared out of scope here
WHISPER_SECONDS = collections.Counter()
 
def all_rows(dataset, config, split):
    out, offset = [], 0
    while True:
        rows = page(dataset, config, split, offset)["rows"]
        out += [r["row"] for r in rows]
        offset += 100
        if len(rows) < 100:
            return out
 
def dataset_sha(dataset):
    return json.loads(fetch(f"https://huggingface.co/api/datasets/{dataset}"))["sha"][:12]
 
def wer_counts(ref, hyp):
    """Errors (S+D+I) and reference words for one normalised pair; an empty hypothesis is all deletions."""
    n = len(ref.split())
    if not hyp.strip():
        return n, n
    o = jiwer.process_words(ref, hyp)
    return o.substitutions + o.deletions + o.insertions, n
 
def sdi(ref, hyp):
    if not hyp.strip():
        return 0, len(ref.split()), 0
    o = jiwer.process_words(ref, hyp)
    return o.substitutions, o.deletions, o.insertions
 
def corpus_wer(errs, words):
    return 100.0 * sum(errs) / max(sum(words), 1)
 
def _by_cluster(clusters, *arrays):
    tot = collections.defaultdict(lambda: [0.0] * len(arrays))
    for i, c in enumerate(clusters):
        for j, a in enumerate(arrays):
            tot[c][j] += a[i]
    return tot
 
def wer_ci(clusters, errs, words, B=2000, seed=0):
    """95% percentile interval of a corpus WER, resampling clusters (speakers) with replacement."""
    tot = _by_cluster(clusters, errs, words)
    keys, rng = sorted(tot), random.Random(seed)
    boots = []
    for _ in range(B):
        d = rng.choices(keys, k=len(keys))
        boots.append(100 * sum(tot[c][0] for c in d) / max(sum(tot[c][1] for c in d), 1))
    return np.percentile(boots, [2.5, 97.5])
 
def paired_contrast(clusters, errs_a, words, errs_b, label_a, label_b, B=2000, seed=0):
    """WER(a) - WER(b) with a paired speaker-cluster bootstrap: the same resampled speakers for both systems."""
    tot = _by_cluster(clusters, errs_a, errs_b, words)
    keys, rng = sorted(tot), random.Random(seed)
    boots = []
    for _ in range(B):
        d = rng.choices(keys, k=len(keys))
        w = max(sum(tot[c][2] for c in d), 1)
        boots.append(100 * (sum(tot[c][0] for c in d) - sum(tot[c][1] for c in d)) / w)
    boots = np.array(boots)
    lo, hi = np.percentile(boots, [2.5, 97.5])
    p = 2 * min((boots <= 0).mean(), (boots >= 0).mean())
    return {"a": label_a, "b": label_b, "diff": corpus_wer(errs_a, words) - corpus_wer(errs_b, words),
            "lo": lo, "hi": hi, "p": max(p, 1 / B), "clusters": len(keys)}
 
def holm(ps, alpha=0.05):
    order, keep = np.argsort(ps), [False] * len(ps)
    for rank, i in enumerate(order):
        if ps[i] <= alpha / (len(ps) - rank):
            keep[i] = True
        else:
            break
    return keep
 
def verdict(c, significant):
    """Verdict rule: diff = WER(other system) - WER(Whistle). CI entirely > 0 and Holm-significant -> Whistle ahead;
    entirely < 0 and significant -> Whistle behind; otherwise no significant difference at this sample size."""
    if c["lo"] > 0 and significant:
        return "Whistle ahead"
    if c["hi"] < 0 and significant:
        return "Whistle behind"
    return "no significant difference at this n"
 
def fmt_contrast(c):
    return f"{c['diff']:+.2f} pts (95% CI {c['lo']:+.2f}..{c['hi']:+.2f}, {c['clusters']} speakers)"
 
@section("2. Fixed, seeded samples of LibriSpeech test-clean and test-other, and the rules for the verdicts")
def sample_english():
    global EN
    print("Rules for the verdicts below, written into this notebook before its final run (exploratory probes on the")
    print("same splits came first; the prose says which):")
    print(f"  * simple random sample of {N_WHISTLE} rows per split (seed {SEED}); the first {N_PAIRED['other']} (test-other) and "
          f"{N_PAIRED['clean']} (test-clean) are the paired Whisper subsets")
    print(f"  * clips longer than {MAX_CLIP_S:.0f} s are excluded and counted (Whistle's single-pass window), as are empty references")
    print("  * the same normaliser on both sides of each comparison (Whisper's EnglishTextNormalizer for LibriSpeech, its")
    print("    BasicTextNormalizer for MLS); corpus WER = summed errors / summed reference words")
    print("  * interval: 95% speaker-cluster paired bootstrap, 2,000 draws, the same resampled speakers for every system")
    print("  * verdict per contrast: unadjusted CI of WER(other) - WER(Whistle) entirely > 0 and Holm-adjusted p < 0.05 ->")
    print("    Whistle ahead; entirely < 0 and significant -> behind; otherwise no significant difference at this n")
    print("    family A (published claims): test-other and test-clean vs base greedy and vs base beam 5, MLS pooled vs base greedy;")
    print("    family B: the keyword contrasts. Holm at alpha = 0.05 within each family. Whisper tiny rows are descriptive.\n")
    global MANIFEST
    EN, manifest = {}, {"dataset_sha": dataset_sha("openslr/librispeech_asr"), "seed": SEED, "splits": {}}
    MANIFEST = manifest
    for split in ("other", "clean"):
        t0 = time.time()
        rows = all_rows("openslr/librispeech_asr", split, "test")
        idx = random.Random(SEED).sample(range(len(rows)), N_WHISTLE)
        kept, too_long, empty = [], 0, 0
        for i in idx:
            r = rows[i]
            audio, h = load_audio(r)
            ref = NORM_EN(r["text"])
            if len(audio) / SR > MAX_CLIP_S:
                too_long += 1
                continue
            if not ref:
                empty += 1
                continue
            kept.append({"id": r["id"], "spk": str(r["speaker_id"]), "text": r["text"], "ref": ref,
                         "audio": audio, "dur": len(audio) / SR, "sha": h})
        EN[split] = kept
        manifest["splits"][split] = {"rows": len(rows), "sampled": N_WHISTLE, "too_long": too_long, "empty_ref": empty,
                                     "ids": [c["id"] for c in kept], "sha": [c["sha"] for c in kept]}
        durs = [c["dur"] for c in kept]
        pk = kept[:N_PAIRED[split]]
        print(f"test-{split}: {len(rows)} rows -> {len(kept)} clips kept ({too_long} over {MAX_CLIP_S:.0f} s, {empty} empty), "
              f"{len(set(c['spk'] for c in kept))} speakers, {sum(durs)/60:.1f} min, clip {np.mean(durs):.1f} s mean / {max(durs):.1f} s max"
              f"   [{time.time()-t0:.0f} s]")
        print(f"   paired subset: {len(pk)} clips, {len(set(c['spk'] for c in pk))} speakers, {sum(c['dur'] for c in pk):.0f} s of audio, "
              f"{sum(len(c['ref'].split()) for c in pk)} reference words")
    json.dump(manifest, open("manifest.json", "w"))
    print(f"\nmanifest.json: LibriSpeech revision {manifest['dataset_sha']}, clip ids and file hashes (the MLS and MInDS-14 clips are added to it in")
    print("steps 9 and 12), so a rerun can prove it used the same audio")
    return f"{len(EN['other'])} + {len(EN['clean'])} clips, manifest written"
 
sample_english()
``` เพื่อให้การประเมินผลเป็นกลาง เราได้กำหนดกฎการตัดสินไว้ล่วงหน้าก่อนการรันจริง โดยเลือกสุ่มตัวอย่าง 300 คลิปต่อชุดข้อมูล และคัดคลิปที่ยาวเกินความสามารถการประมวลผลรอบเดียวของ Whistle ออก จนเหลือ 299 คลิป
 
สำหรับการเปรียบเทียบแบบจับคู่ เราใช้ 100 คลิปแรกจาก test-other และ 60 คลิปจาก test-clean รันผ่าน Whisper โดยความแตกต่างจะถูกถือว่ามีนัยสำคัญก็ต่อเมื่อช่วงความเชื่อมั่น 95% ไม่ครอบคลุมค่าศูนย์ และค่า p ที่ปรับตามวิธี Holm มีค่าน้อยกว่า 0.05 เท่านั้น
 
```python
WHISPER = {}
 
def whisper_model(size):
    if size not in WHISPER:
        WHISPER[size] = whisper.load_model(size, device="cpu")
    return WHISPER[size]
 
def run_whisper(size, audio, language="en", **kw):
    """openai-whisper with its defaults (greedy with temperature fallback) unless kw says otherwise. Seeded per call,
    because the fallback samples. Returns text, whether any segment needed the fallback, and the wall time."""
    torch.manual_seed(0)
    WHISPER_SECONDS[size] += len(audio) / SR
    t0 = time.perf_counter()
    r = whisper_model(size).transcribe(audio, language=language, fp16=False, **kw)
    fell_back = any(s.get("temperature", 0) > 0 for s in r["segments"])
    return r["text"], fell_back, time.perf_counter() - t0
 
SYSTEMS = {"other": [("whisper tiny, greedy", "tiny", {}), ("whisper base, greedy", "base", {}), ("whisper base, beam 5", "base", {"beam_size": 5})],
           "clean": [("whisper tiny, greedy", "tiny", {}), ("whisper base, greedy", "base", {}), ("whisper base, beam 5", "base", {"beam_size": 5})]}
FAMILY_A = {"test-clean: Whistle vs whisper base, greedy", "test-clean: Whistle vs whisper base, beam 5",
            "test-other: Whistle vs whisper base, greedy", "test-other: Whistle vs whisper base, beam 5",
            "MLS pooled: Whistle vs whisper base, greedy"}
PUBLISHED = {"other": "Whistle 10.49 / Whisper base 11.0 (paper table 9, beam) / 12.4 (table 8, greedy)",
             "clean": "Whistle 4.31 / Whisper base 4.9 (paper table 9, beam) / 5.0 (table 8, greedy)"}
CONTRASTS = []
 
@section("3. Word error rate against Whisper tiny and base, paired by clip, with a verdict per contrast")
def english_wer():
    global HYP
    HYP = {}
    for split in ("other", "clean"):
        clips = EN[split]
        t0 = time.perf_counter()
        hw = [W.transcribe(c["audio"], language="en")["text"] for c in clips]
        tw = time.perf_counter() - t0
        HYP[(split, "Whistle")] = hw
        e_w, n_w = zip(*[wer_counts(c["ref"], NORM_EN(h)) for c, h in zip(clips, hw)])
        lo, hi = wer_ci([c["spk"] for c in clips], e_w, n_w)
        print(f"test-{split}: Whistle on all {len(clips)} clips: WER {corpus_wer(e_w, n_w):.2f}% (95% speaker-cluster CI {lo:.2f}..{hi:.2f}) "
              f"in {tw:.1f} s ({1000*tw/len(clips):.0f} ms per clip)")
        print(f"   published full-split figures, different sample: {PUBLISHED[split]}")
        pk = clips[:N_PAIRED[split]]
        spk = [c['spk'] for c in pk]
        e_wp, n_p = list(e_w[:len(pk)]), list(n_w[:len(pk)])
        s = np.sum([sdi(c["ref"], NORM_EN(h)) for c, h in zip(pk, hw[:len(pk)])], axis=0)
        print(f"   paired subset, {len(pk)} clips / {len(set(spk))} speakers / {sum(n_p)} words:")
        print(f"   {'Whistle, 5-beam':22s} {corpus_wer(e_wp, n_p):6.2f}%  S/D/I {s[0]:3d}/{s[1]:3d}/{s[2]:3d}")
        for label, size, kw in SYSTEMS[split]:
            res = [run_whisper(size, c["audio"], **kw) for c in pk]
            hyps = [r[0] for r in res]
            HYP[(split, label)] = hyps
            e_o = [wer_counts(c["ref"], NORM_EN(h))[0] for c, h in zip(pk, hyps)]
            s = np.sum([sdi(c["ref"], NORM_EN(h)) for c, h in zip(pk, hyps)], axis=0)
            con = paired_contrast(spk, e_o, n_p, e_wp, label, "Whistle")
            con["name"] = f"test-{split}: Whistle vs {label}"
            CONTRASTS.append(con)
            better, worse = sum(a < b for a, b in zip(e_wp, e_o)), sum(a > b for a, b in zip(e_wp, e_o))
            print(f"   {label:22s} {corpus_wer(e_o, n_p):6.2f}%  S/D/I {s[0]:3d}/{s[1]:3d}/{s[2]:3d}  fallback on {sum(r[1] for r in res):2d} clips  "
                  f"{np.median([r[2] for r in res])*1000:5.0f} ms/clip  | {fmt_contrast(con)}  clips Whistle better/worse {better}/{worse}")
    fam = [c for c in CONTRASTS if c["name"] in FAMILY_A]
    print(f"\nContrasts so far (family A: {len(fam)} of 5; the Holm correction and the verdicts come once the MLS contrast is in):")
    for c in CONTRASTS:
        tag = "family A" if c["name"] in FAMILY_A else "descriptive"
        print(f"   {c['name']:48s} {fmt_contrast(c)}  p~{c['p']:.3f}  [{tag}]")
    print("\n   Whistle decodes with a 5-beam length-normalised search. openai-whisper's default is greedy with temperature")
    print("   fallback; the Whisper paper's LibriSpeech numbers come from beam search, so the beam-5 rows are the like-for-like ones.")
    print("   Every gap here is a system gap (model plus decoder), not a model-only one.")
    return " | ".join(f"test-{s}: Whistle {corpus_wer(*zip(*[wer_counts(c['ref'], NORM_EN(h)) for c, h in zip(EN[s][:N_PAIRED[s]], HYP[(s, 'Whistle')][:N_PAIRED[s]])])):.2f}% "
                      f"vs base greedy {corpus_wer(*zip(*[wer_counts(c['ref'], NORM_EN(h)) for c, h in zip(EN[s][:N_PAIRED[s]], HYP[(s, 'whisper base, greedy')])])):.2f}%" for s in ("other", "clean"))
 
english_wer()

def audio_seconds(clips, seconds):
    """Exactly `seconds` of speech: clips concatenated in sample order, then trimmed."""
    out, total = [], 0
    for c in clips:
        out.append(c["audio"])
        total += c["dur"]
        if total >= seconds:
            break
    return np.concatenate(out)[:int(seconds * SR)]
 
PROBE_SCRIPT = r"""
import sys, json, time, resource, numpy as np
kind, paths = sys.argv[1], sys.argv[2:]
scale = 1 << 20 if sys.platform == "darwin" else 1 << 10        # ru_maxrss is bytes on macOS, kilobytes on Linux
rss = lambda: resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / scale
out = {"kind": kind}
if kind.startswith("whistle"):
    import needle
    out["after_import"] = rss()
    W = needle.Whistle()
    run = lambda a: W.transcribe(a, language="en")
else:
    import whisper, torch
    out["after_import"] = rss()
    size, kw = kind.split("-")[1], ({"beam_size": 5} if kind.endswith("beam5") else {})
    m = whisper.load_model(size, device="cpu")
    def run(a):
        torch.manual_seed(0)
        return m.transcribe(a, language="en", fp16=False, **kw)
clips = {p: np.load(p) for p in paths}
run(clips[paths[0]])                                               # warm-up
reps = 5 if kind.startswith("whistle") else 3
for p, a in clips.items():
    ts = []
    for _ in range(reps):
        t0 = time.perf_counter(); r = run(a); ts.append(time.perf_counter() - t0)
    out[p] = {"wall_ms": 1000 * float(np.median(ts)), "spread_ms": 1000 * float(np.ptp(ts))}
    if kind.startswith("whistle"):
        out[p].update(ttft_ms=r["ttft_ms"], tps=r["decode_tps"])
out["peak"] = rss()
print(json.dumps(out))
"""
 
@section("4. Speed and footprint: identical arrays, one fresh process per engine, total wall time and peak memory")
def speed():
    pool = EN["other"]
    print(f"hardware: {CPU}, {os.cpu_count()} cpus visible. Each engine runs in its own fresh process so torch's threads cannot slow")
    print("Whistle down (or the reverse); Whistle's engine picks its own thread count. Primary measure: total wall time on the same")
    print("float32 array, median of 5 (Whistle) or 3 (Whisper) after a warm-up. ttft_ms is the engine's own clock, for reference.\n")
    for secs in (5, 10, 30):
        np.save(f"clip{secs}.npy", audio_seconds(pool, secs))
    open("probe.py", "w").write(PROBE_SCRIPT)
    res = {}
    for kind in ("whistle", "whisper-tiny", "whisper-base", "whisper-base-beam5"):
        out = subprocess.run([sys.executable, "probe.py", kind, "clip5.npy", "clip10.npy", "clip30.npy"], capture_output=True, text=True, env=os.environ)
        if out.returncode != 0:
            raise RuntimeError(f"{kind} probe failed: {out.stderr[-300:]}")
        res[kind] = json.loads(out.stdout.strip().splitlines()[-1])
    print(f"{'clip':>6s} | {'Whistle wall':>12s} {'spread':>7s} {'ttft':>6s} {'tok/s':>6s} | {'tiny greedy':>11s} | {'base greedy':>11s} | {'base beam 5':>11s}")
    for secs in (5, 10, 30):
        p = f"clip{secs}.npy"
        w = res["whistle"][p]
        print(f"{secs:5d} s | {w['wall_ms']:9.0f} ms {w['spread_ms']:5.0f} ms {w['ttft_ms']:6.1f} {w['tps']:6.0f} | "
              f"{res['whisper-tiny'][p]['wall_ms']:8.0f} ms | {res['whisper-base'][p]['wall_ms']:8.0f} ms | {res['whisper-base-beam5'][p]['wall_ms']:8.0f} ms")
    w5, w10, w30 = (res["whistle"][p]["wall_ms"] for p in ("clip5.npy", "clip10.npy", "clip30.npy"))
    b5, b10, b30 = (res["whisper-base"][p]["wall_ms"] for p in ("clip5.npy", "clip10.npy", "clip30.npy"))
    bb5, bb10, bb30 = (res["whisper-base-beam5"][p]["wall_ms"] for p in ("clip5.npy", "clip10.npy", "clip30.npy"))
    print(f"\nreal-time factor on the 10 s clip: Whistle {w10/10000:.4f}, base greedy {b10/10000:.3f}, base beam 5 {bb10/10000:.3f}")
    print(f"base greedy / Whistle wall time: {b5/w5:.1f}x at 5 s, {b10/w10:.1f}x at 10 s, {b30/w30:.1f}x at 30 s; beam 5 / Whistle: "
          f"{bb5/w5:.1f}x, {bb10/w10:.1f}x, {bb30/w30:.1f}x")
    print(f"from 5 to 10 s the wall time grows {w10/w5:.2f}x for Whistle, {b10/b5:.2f}x for base greedy, {bb10/bb5:.2f}x for base beam 5;")
    print(f"from 10 to 30 s: {w30/w10:.2f}x, {b30/b10:.2f}x, {bb30/bb10:.2f}x. Whisper's encoder always sees a padded 30 s window; its decoder")
    print("still grows with the words it emits, so the padding fixes only part of its cost. The encoder is not timed separately here.")
    print("\npeak RSS of each process (model loaded, three clips transcribed) and the RSS right after its imports:")
    for kind in ("whistle", "whisper-tiny", "whisper-base"):
        r = res[kind]
        print(f"   {kind:13s} peak {r['peak']:6.0f} MB   after imports {r['after_import']:4.0f} MB   -> {r['peak']-r['after_import']:4.0f} MB for the model and the run")
    sizes = {"whistle.cact + engine": (os.path.getsize(weights) + os.path.getsize(engine)) / 1e6}
    for size in ("tiny", "base"):
        p = os.path.expanduser(f"~/.cache/whisper/{size}.pt")
        sizes[f"whisper {size}.pt"] = os.path.getsize(p) / 1e6 if os.path.exists(p) else float("nan")
    print("files on disk: " + ", ".join(f"{k} {v:.1f} MB" for k, v in sizes.items()) + "  (file sizes, not deployed footprint)")
    off = subprocess.run([sys.executable, "-c", "import needle, numpy as np; W = needle.Whistle(); print(W.transcribe(np.zeros(16000, np.float32))['text'] == '')"],
                         capture_output=True, text=True, env={**os.environ, "HF_HUB_OFFLINE": "1"})
    print(f"fresh process with HF_HUB_OFFLINE=1 and the pinned paths: {'loads and runs' if 'True' in off.stdout else 'FAILED: ' + off.stderr[-200:]}")
    return f"10 s clip: Whistle {w10:.0f} ms vs base greedy {b10:.0f} ms ({b10/w10:.1f}x, {b5/w5:.1f}x at 5 s, {b30/w30:.1f}x at 30 s); peak RSS {res['whistle']['peak']:.0f} vs {res['whisper-base']['peak']:.0f} MB"
 
speed()

def quiet_windows(a, limit=30.0, search=5.0, hop=0.2):
    """Cut before `limit` seconds at the quietest `hop`-second patch inside the last `search` seconds of each window."""
    out, i, n, s, h = [], 0, int(limit * SR), int(search * SR), int(hop * SR)
    while len(a) - i > n:
        seg = a[i + n - s:i + n]
        energy = [np.mean(seg[j:j + h] ** 2) for j in range(0, len(seg) - h, h // 4)]
        cut = i + n - s + int(np.argmin(energy)) * (h // 4) + h // 2
        out.append(a[i:cut])
        i = cut
    out.append(a[i:])
    return out
 
def speaker_sessions(min_s=40, max_s=100, k=8):
    """Same-speaker concatenations (0.4 s gaps) from the sampled clips of both splits, longest speakers first."""
    by = collections.defaultdict(list)
    for split in ("other", "clean"):
        for c in EN[split]:
            by[c["spk"]].append(c)
    out, gap = [], np.zeros(int(0.4 * SR), np.float32)
    for spk, cs in sorted(by.items(), key=lambda kv: -sum(c["dur" ] for c in kv[1])):
        parts, used, total = [], [], 0
        for c in cs:
            if total + c["dur"] + 0.4 > max_s:
                break
            parts += [c["audio"], gap]
            used.append(c)
            total += c["dur"] + 0.4
        if total >= min_s:
            out.append({"spk": spk, "clips": used, "audio": np.concatenate(parts), "ref": " ".join(c["ref"] for c in used), "dur": total})
        if len(out) == k:
            break
    return out
 
OTHER_ENGINE_SCRIPT = r"""
import sys, os, numpy as np
os.environ["NEEDLE3_LIB_PATH"] = sys.argv[1]
import needle
W = needle.Whistle()
a = np.load(sys.argv[2])
for secs in (30.0, 30.01, 45.0):
    try:
        r = W.transcribe(a[:int(secs * 16000)], language="en", word_timestamps=True)
        print(f"   {secs:6.2f} s -> {len(r['words'])} words, last word ends at {r['words'][-1]['end']:.2f} s")
    except RuntimeError as e:
        print(f"   {secs:6.2f} s -> RuntimeError: {e}")
"""
 
@section("5. The 30-second window: what transcribe() does with longer audio, and two ways around it")
def long_audio():
    global SESSIONS
    SESSIONS = speaker_sessions()
    s = SESSIONS[0]
    gap = np.zeros(int(0.4 * SR), np.float32)
    print(f"speaker {s['spk']}: {len(s['clips'])} utterances joined with 0.4 s gaps, {s['dur']:.0f} s, {len(s['ref'].split())} reference words")
    print("the docs say transcribe() takes 'audio of any length, one pass up to 30 seconds and windowed beyond it'.")
    print(f"the pinned engine {ENGINE_VERSION}, on whole utterances cut from the session so the reference is exact:\n")
    print(f"{'audio':>7s} | {'ref words':>9s} {'hyp words':>9s} {'WER':>7s} | gaps longer than 3 s in the word timeline")
    for target in (20, 29, 35, 45, 60, 90):
        parts, refs, total = [], [], 0
        for c in s["clips"]:
            if total + c["dur"] + 0.4 > target:
                break
            parts += [c["audio"], gap]
            refs.append(c["ref"])
            total += c["dur"] + 0.4
        if total < min(target, s["dur"]) - 8:
            continue
        a, ref = np.concatenate(parts), " ".join(refs)
        r = W.transcribe(a, language="en", word_timestamps=True)
        hyp = NORM_EN(r["text"])
        gaps = [f"{w1['end']:.0f}-{w2['start']:.0f} s" for w1, w2 in zip(r["words"], r["words"][1:]) if w2["start"] - w1["end"] > 3]
        print(f"{total:5.0f} s | {len(ref.split()):9d} {len(hyp.split()):9d} {100*jiwer.wer(ref, hyp or '<empty>'):6.1f}% | {', '.join(gaps) or 'none'}")
    full = W.transcribe(s["audio"], language="en", word_timestamps=True)
    hyp = NORM_EN(full["text"])
    print(f"\nthe whole {s['dur']:.0f} s in one call: {len(hyp.split())} of {len(s['ref'].split())} words, WER {100*jiwer.wer(s['ref'], hyp or '<empty>'):.1f}%, "
          f"nothing raised, engine ttft {full['ttft_ms']:.0f} ms")
    # the same repo revision also holds a 3.2.0 engine wheel; a child process shows what that build does with the same audio
    other_wheel = hf_hub_download("Cactus-Compute/needle3", needle_fetch.engine_wheel("3.2.0", TAG, 3), revision=NEEDLE3_REV)
    with zipfile.ZipFile(other_wheel) as z:
        member = next(n for n in z.namelist() if n.startswith("needle/libneedle"))
        other = os.path.abspath(os.path.join("engine", "v3.2.0-" + os.path.basename(member)))
        open(other, "wb").write(z.read(member))
    np.save("session0.npy", s["audio"])
    open("other_engine.py", "w").write(OTHER_ENGINE_SCRIPT)
    out = subprocess.run([sys.executable, "other_engine.py", other, "session0.npy"], capture_output=True, text=True, env=os.environ)
    print(f"\nthe 3.2.0 engine wheel from the same repo revision (sha256 {sha(other)}), in a child process, on the same audio:")
    print(out.stdout.rstrip() or out.stderr[-300:])
    print("   two builds, two behaviours: one windows and loses the middle, the other refuses. Either way the split is yours to make.\n")
    print("workarounds on the whole clip, scored against the exact reference (one clip; the joins are 0.4 s of silence):")
    results = {}
    for name, wins in (("fixed 30 s windows", [s["audio"][i:i + 30 * SR] for i in range(0, len(s["audio"]), 30 * SR)]),
                       ("fixed 25 s windows", [s["audio"][i:i + 25 * SR] for i in range(0, len(s["audio"]), 25 * SR)]),
                       ("quiet-point cuts <= 30 s", quiet_windows(s["audio"], 30)),
                       ("quiet-point cuts <= 20 s", quiet_windows(s["audio"], 20))):
        t0 = time.perf_counter()
        hyp = NORM_EN(" ".join(W.transcribe(w, language="en")["text"] for w in wins))
        results[name] = 100 * jiwer.wer(s["ref"], hyp or "<empty>")
        print(f"   {name:26s} {len(wins):2d} calls   WER {results[name]:5.2f}%   {1000*(time.perf_counter()-t0):4.0f} ms   "
              f"windows of {', '.join(f'{len(w)/SR:.1f}' for w in wins)} s")
    t0 = time.perf_counter()
    steps = list(W.stream(iter([s["audio"][i:i + SR] for i in range(0, len(s["audio"]), SR)]), language="en"))
    hyp = NORM_EN(" ".join(x["text"] for x in steps if x["text"]))
    results["stream(), 1 s chunks"] = 100 * jiwer.wer(s["ref"], hyp or "<empty>")
    print(f"   {'stream(), 1 s chunks':26s} {len(steps):2d} passes  WER {results['stream(), 1 s chunks']:5.2f}%   {1000*(time.perf_counter()-t0):4.0f} ms")
    best = min(results, key=results.get)
    return (f"one call on {s['dur']:.0f} s keeps {len(NORM_EN(full['text']).split())}/{len(s['ref'].split())} words; {best}: {results[best]:.2f}% WER "
            f"(fixed 30 s windows {results['fixed 30 s windows']:.2f}%)")
 
long_audio()

@section("6. Streaming: cost per pass, how long a word waits to be committed, and what chunk size changes")
def streaming():
    durs = [s["dur"] for s in SESSIONS]
    print(f"{len(SESSIONS)} same-speaker sessions of {', '.join(f'{x:.0f}' for x in durs)} s, "
          f"{sum(len(s['ref'].split()) for s in SESSIONS)} reference words; the joins between utterances are artificial")
    off = {c["id"]: h for split in ("other", "clean") for c, h in zip(EN[split], HYP[(split, "Whistle")])}
    off_err = [sum(wer_counts(c["ref"], NORM_EN(off[c["id"]]))[0] for c in s["clips"]) for s in SESSIONS]
    words = [len(s["ref"].split()) for s in SESSIONS]
    print(f"the same utterances one at a time through transcribe() (the step 3 outputs): WER {100*sum(off_err)/sum(words):.2f}%\n")
    print(f"{'chunk':>6s} {'sessions':>8s} | {'WER':>6s} | {'pass ms mean / max':>18s} | {'passes > 1 s':>12s} | {'commit latency median / p90':>27s} | {'words only in tail':>18s} | {'pass vs time r':>14s}")
    summary, stream_err = {}, None
    for chunk_s, sessions in ((1.0, SESSIONS), (0.5, SESSIONS[:2]), (1.0, SESSIONS[:2]), (2.0, SESSIONS[:2])):
        n = int(chunk_s * SR)
        errs, tail, allw = [], 0, 0
        pass_ms, lat, rs = [], [], []
        for s in sessions:
            steps = list(W.stream(iter([s["audio"][i:i + n] for i in range(0, len(s["audio"]), n)]), language="en"))
            errs.append(wer_counts(s["ref"], NORM_EN(" ".join(x["text"] for x in steps if x["text"])))[0])
            body = steps[:-1]
            pass_ms += [x["pass_ms"] for x in body]
            lat += [x["received"] - wd["end"] for x in steps for wd in x["words"]]
            tail += len(steps[-1]["words"])
            allw += sum(len(x["words"]) for x in steps)
            if len(body) > 2:
                rs.append(np.corrcoef([x["received"] for x in body], [x["pass_ms"] for x in body])[0, 1])
        w = words[:len(sessions)]
        summary[(chunk_s, len(sessions))] = (100 * sum(errs) / sum(w), np.median(lat), np.mean(pass_ms))
        if chunk_s == 1.0 and len(sessions) == len(SESSIONS):
            stream_err = errs
        print(f"{chunk_s:5.1f} s {len(sessions):8d} | {100*sum(errs)/sum(w):5.2f}% | {np.mean(pass_ms):9.0f} / {max(pass_ms):4.0f} | {sum(p > 1000 for p in pass_ms):12d} | "
              f"{np.median(lat):12.2f} / {np.percentile(lat, 90):4.2f} s | {100*tail/allw:17.1f}% | {np.mean(rs):+.2f}")
    c = paired_contrast(list(range(len(SESSIONS))), stream_err, words, off_err, "stream, 1 s chunks", "one utterance at a time")
    print(f"\nstream (1 s chunks, 8 sessions) minus utterance-by-utterance: {c['diff']:+.2f} pts (95% session-cluster CI {c['lo']:+.2f}..{c['hi']:+.2f}, "
          f"{c['clusters']} sessions) -> {'no significant difference' if c['lo'] <= 0 <= c['hi'] else 'a real gap'} at this n")
    print("   pass_ms is the engine time per chunk; below the chunk length means real time. Commit latency is the stream time")
    print("   at which a word was committed minus the word's end: each chunk re-transcribes the recent audio and commits what two")
    print("   passes agree on, so the wait scales with the chunk. r is the correlation of pass time with stream time (buffer growth).")
    print("   The three chunk sizes are compared on the same two sessions; the eight-session row is the headline for 1 s chunks.")
    a8, a1, a05, a2 = summary[(1.0, len(SESSIONS))], summary[(1.0, 2)], summary[(0.5, 2)], summary[(2.0, 2)]
    return (f"1 s chunks on 8 sessions: WER {a8[0]:.2f}% vs {100*sum(off_err)/sum(words):.2f}% utterance by utterance, median commit {a8[1]:.2f} s; "
            f"on 2 sessions the median commit is {a05[1]:.2f} / {a1[1]:.2f} / {a2[1]:.2f} s for 0.5 / 1 / 2 s chunks")
 
streaming()

def auroc(scores, labels):
    """Probability that a random positive scores above a random negative (Mann-Whitney)."""
    pos = [s for s, l in zip(scores, labels) if l]
    neg = [s for s, l in zip(scores, labels) if not l]
    if not pos or not neg:
        return float("nan")
    order = sorted((s, i) for i, s in enumerate(scores))
    ranks, i = {}, 0
    while i < len(order):
        j = i
        while j + 1 < len(order) and order[j + 1][0] == order[i][0]:
            j += 1
        for k in range(i, j + 1):
            ranks[order[k][1]] = (i + j) / 2 + 1
        i = j + 1
    pos_idx = [i for i, l in enumerate(labels) if l]
    return (sum(ranks[i] for i in pos_idx) - len(pos) * (len(pos) + 1) / 2) / (len(pos) * len(neg))
 
@section("7. Word timestamps and word probabilities: consistency, a known shift, agreement with Whisper, and what p < 0.7 means")
def timestamps():
    import difflib
    pk = EN["other"][:40]
    key = lambda w: re.sub(r"[^a-z']", "", w.lower())
    diffs, same, mono, shifts = [], 0, 0, []
    probs, correct, durs = [], [], []
    replay_hits = replay_n = 0
    for i, c in enumerate(pk):
        ws = W.transcribe(c["audio"], language="en", word_timestamps=True)["words"]
        mono += all(a["start"] <= a["end"] <= b["start"]. + 1e-6 for a, b in zip(ws, ws[1:]))
        if i < 20:   # a known shift: 2.000 s of zeros in front should move every word by 2.000 s
            ws2 = W.transcribe(np.concatenate([np.zeros(2 * SR, np.float32), c["audio"]]), language="en", word_timestamps=True)["words"]
            if ws2 and ws:
                shifts.append(ws2[0]["start"] - ws[0]["start"])
        if i < 20 and len(ws) >= 6:   # cut the audio where word k ends and transcribe only the cut
            k = len(ws) // 2
            got = NORM_EN(W.transcribe(c["audio"][:int(ws[k]["end"] * SR)], language="en")["text"]).split()
            exp = NORM_EN(" ".join(w["word"] for w in ws[:k + 1])).split()
            replay_n += 1
            replay_hits += got == exp
        torch.manual_seed(0)
        WHISPER_SECONDS["base"] += c["dur"]
        ww = [w for s in whisper_model("base").transcribe(c["audio"], language="en", fp16=False, word_timestamps=True)["segments"] for w in s["words"]]
        if [key(w["word"]) for w in ws] == [key(w["word"]) for w in ww]:
            same += 1
            diffs += [x["start"] - y["start"]. for x, y in zip(ws, ww)] + [x["end"] - y["end"] for x, y in zip(ws, ww)]
        H, tp, td = [], [], []
        for w in ws:
            for t in NORM_EN(w["word"]).split():   # "hmm" normalises to nothing, "didn't" to two tokens, as in the reference
                H.append(t)
                tp.append(w["probability"])
                td.append(w["end"] - w["start"])
        ok = [False] * len(H)
        for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(a=c["ref"].split(), b=H).get_opcodes():
            if tag == "equal":
                for j in range(j1, j2):
                    ok[j] = True
        probs += tp
        correct += ok
        durs += td
    diffs, shifts = np.abs(diffs), np.array(shifts)
    print(f"{len(pk)} test-other clips: word times monotonic on {mono}/{len(pk)}")
    print(f"2.000 s of silence prepended on 20 clips: first word moved by {np.median(shifts):.3f} s (median), within 50 ms of 2.000 on "
          f"{100*np.mean(np.abs(shifts-2) <= 0.05):.0f}% of clips")
    print(f"cut-and-replay: cutting the audio at a word's 'end' and transcribing the cut reproduced the exact word prefix on {replay_hits}/{replay_n} clips")
    print(f"agreement with Whisper base's DTW word times, on the {same}/{len(pk)} clips where both emit the same words ({len(diffs)//2} words; a selected, easier subset):")
    print(f"   |boundary difference| median {np.median(diffs):.3f} s, within 100 ms {100*np.mean(diffs <= 0.1):.0f}%, within 200 ms "
          f"{100*np.mean(diffs <= 0.2):.0f}%   (agreement between สอง systems, not accuracy: there is no gold alignment)")
    probs, correct, durs = np.array(probs), np.array(correct), np.array(durs)
    print(f"\nword probability over {len(probs)} normalised tokens from {len(pk)} clips: median {np.median(probs):.3f}, 10th percentile {np.percentile(probs, 10):.3f}")
    for lo, hi in ((0, 0.5), (0.5, 0.7), (0.7, 0.9), (0.9, 1.01)):
        m = (probs >= lo) & (probs < hi)
        if m.sum():
            print(f"   p in [{lo:.1f}, {min(hi, 1):.1f}): {m.sum():4d} words, {100*(~correct[m]).mean():5.1f}% wrong")
    print(f"   AUROC for 'this word is wrong': 1 - probability {auroc(-probs, ~correct):.3f}; word duration as a baseline (shorter = riskier) {auroc(-durs, ~correct):.3f}")
    low, high = probs < 0.7, probs >= 0.7
    return (f"shift {np.median(shifts):.2f} s; within 200 ms of Whisper on {100*np.mean(diffs <= 0.2):.0f}% of {len(diffs)//2} matched words; "
            f"p<0.7 words wrong {100*(~correct[low]).mean():.0f}% vs {100*(~correct[high]).mean():.1f}%")
 
timestamps()

@section("8. Keyword biasing: an oracle ceiling, a matched wrong-keyword control and a distractor list")
def keywords():
    import difflib
    clips = EN["other"]
    base = HYP[("other", "Whistle")]
    freq = collections.Counter(w for c in clips for w in c["ref"].split())
 
def error_words(ref, hyp):
        out = []
        for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(a=ref.split(), b=NORM_EN(hyp).split()).get_opcodes():
            if tag in ("replace", "delete"):
                out += ref.split()[i1:i2]
        return sorted(set(out))
    oracle = [error_words(c["ref"], h) for c, h in zip(clips, base)]
    idx = [i for i, k in enumerate(oracle) if k]
    print(f"{len(idx)}/{len(clips)} test-other clips have at least one Whistle error ({sum(len(oracle[i]) for i in idx)} distinct wrong words)")
    print("   oracle keywords  = the reference words Whistle got wrong on that clip (a CEILING: it reads the answer)")
    print("   matched wrong    = the next error clip's oracle list, minus every word the target clip actually says")
    print("   distractor (8)   = the 8 most frequent reference words of the whole sample that the target clip does not say")
    print("   distractor (2)   = the same list cut to 2 words, so its size matches the oracle and matched arms\n")
 
def matched_wrong(i):
        donor = oracle[idx[(idx.index(i) + 1) % len(idx)]]
        return [w for w in donor if w not in clips[i]["ref"].split()] or None
    arms = {}
    distractor = lambda i, k: [w for w, _ in freq.most_common(60) if w not in clips[i]["ref"].split()][:k]
    for name, keylist in (("no keywords", lambda i: None), ("oracle (ceiling)", lambda i: oracle[i]), ("matched wrong", matched_wrong),
                          ("distractor (8)", lambda i: distractor(i, 8)), ("distractor (2)", lambda i: distractor(i, 2))):
        errs, words, ins, recovered, total_kw, bad_kw, bad_kw_base, n_kw = [], [], 0, 0, 0, 0, 0, 0
        for i in idx:
            c = clips[i]
            kws = keylist(i)
            n_kw += len(kws or [])
            h = base[i] if name == "no keywords" else W.transcribe(c["audio"], language="en", keywords=kws)["text"]
            s, d, ins_i = sdi(c["ref"], NORM_EN(h))
            errs.append(s + d + ins_i)
            words.append(len(c["ref"].split()))
            ins += ins_i
            still_wrong = set(error_words(c["ref"], h))
            recovered += sum(w not in still_wrong for w in oracle[i])
            total_kw += len(oracle[i])
            if kws and name != "oracle (ceiling)":
                bad_kw += sum(w in NORM_EN(h).split() for w in kws)
                bad_kw_base += sum(w in NORM_EN(base[i]).split() for w in kws)
        arms[name] = (errs, words)
        extra = f"   wrong keywords in the output {bad_kw:3d} (already {bad_kw_base} without biasing)" if name not in ("no keywords", "oracle (ceiling)") else ""
        print(f"   {name:18s} {n_kw/len(idx):4.1f} keywords/clip   WER {corpus_wer(errs, words):6.2f}%   insertions {ins:3d}   oracle words now right {recovered:3d}/{total_kw}{extra}")
    spk = [clips[i]["spk"] for i in idx]
    w = arms["no keywords"][1]
    fam = []
    for a, b in (("no keywords", "oracle (ceiling)"), ("matched wrong", "no keywords"), ("distractor (8)", "no keywords"),
                 ("distractor (2)", "no keywords"), ("matched wrong", "oracle (ceiling)")):
        c = paired_contrast(spk, arms[a][0], w, arms[b][0], a, b)
        fam.append(c)
    sig = holm([c["p"] for c in fam])
    print(f"\n   family B, {len(fam)} contrasts, Holm at 0.05:")
    for c, s in zip(fam, sig):
        print(f"   WER({c['a']}) - WER({c['b']}) = {fmt_contrast(c)}  p~{c['p']:.3f}  {'significant' if s else 'not significant'}")
    print("   the decisive contrast is matched wrong minus oracle: how much of the gain needs the right words, not just any words")
    return f"oracle ceiling {corpus_wer(*arms['oracle (ceiling)']):.2f}% vs {corpus_wer(*arms['no keywords']):.2f}% baseline; wrong keywords {corpus_wer(*arms['matched wrong']):.2f}%"
 
keywords()

MLS = {"de": "german", "fr": "french", "es": "spanish", "it": "italian", "nl": "dutch", "pl": "polish"}
WHISTLE_LANGS = ("en", "de", "fr", "es", "it", "nl", "pl")
N_PER_LANG = 12
PUBLISHED_MLS_BASE = {"nl": 28.4, "fr": 26.6, "de": 17.7, "it": 31.1, "pl": 22.8, "es": 12.8}   # Whisper paper, table 10 (full splits)
 
def clopper_pearson_lower(k, n, alpha=0.05):
    """Exact one-sided 95% lower bound for a binomial proportion, by bisection on the binomial tail."""
    lo, hi = 0.0, 1.0
    for _ in range(60):
        p = (lo + hi) / 2
        tail = sum(math.comb(n, j) * p ** j * (1 - p) ** (n - j) for j in range(k, n + 1))
        lo, hi = (p, hi) if tail < alpha else (lo, p)
    return lo
 
@section("9. Six more languages on Multilingual LibriSpeech, pooled, with a symmetric language-ID test")
def multilingual():
    rng = random.Random(SEED)
    print(f"sampling rule, fixed in advance: {N_PER_LANG} clips per language from 3 random pages of 100 rows of each test split (seed {SEED});")
    print("MLS pages are speaker-sorted, so the pages are the first stage of the sample. Normaliser: Whisper's BasicTextNormalizer on both sides.\n")
    pooled = {"Whistle (forced)": [], "Whistle (detected)": [], "whisper base, greedy": []}
    words, clusters, lid_w, lid_b, rows_out = [], [], 0, 0, []
    wb = whisper_model("base")
    for code, cfg in MLS.items():
        total = page("facebook/multilingual_librispeech", cfg, "test", 0, 1)["num_rows_total"]
        offsets = rng.sample(range(0, total, 100), 3)
        rows = [r["row"] for off in offsets for r in page("facebook/multilingual_librispeech", cfg, "test", off)["rows"]]
        sample = rng.sample(rows, N_PER_LANG)
        e, n_words, spk = collections.defaultdict(list), [], []
        for r in sample:
            audio, h = load_audio(r)
            MANIFEST.setdefault("mls", {}).setdefault(code, []).append([r["id"], h])
            ref = NORM_BASIC(r["transcript"])
            forced = W.transcribe(audio, language=code)["text"]
            det = W.transcribe(audio)
            lid_w += det["language"] == code
            hb, _, _ = run_whisper("base", audio, language=code)
            mel = whisper.log_mel_spectrogram(whisper.pad_or_trim(audio), n_mels=wb.dims.n_mels)
            _, probs = wb.detect_language(mel)
            lid_b += max(WHISTLE_LANGS, key=lambda l: probs.get(l, 0)) == code
            for name, h in (("Whistle (forced)", forced), ("Whistle (detected)", det["text"]), ("whisper base, greedy", hb)):
                e[name].append(wer_counts(ref, NORM_BASIC(h))[0])
            n_words.append(len(ref.split()))
            spk.append(f"{code}-{r['speaker_id']}")
        for name in pooled:
            pooled[name] += e[name]
        words += n_words
        clusters += spk
        rows_out.append((code, len(sample), len(set(spk)), sum(n_words), corpus_wer(e["Whistle (forced)"], n_words), corpus_wer(e["whisper base, greedy"], n_words)))
    print(f"{'lang':>4s} {'clips':>5s} {'spk':>3s} {'words':>5s} | {'Whistle':>8s} {'base':>8s} | Whisper base published on the full split (paper table 10)")
    for code, n, s, nw, ww, wbase in rows_out:
        print(f"{code:>4s} {n:5d} {s:3d} {nw:5d} | {ww:7.1f}% {wbase:7.1f}% | {PUBLISHED_MLS_BASE[code]}")
    print("   per-language rows are descriptive: 12 clips from a handful of speakers cannot rank the systems within a language\n")
    c = paired_contrast(clusters, pooled["whisper base, greedy"], words, pooled["Whistle (forced)"], "whisper base, greedy", "Whistle")
    c["name"] = "MLS pooled: Whistle vs whisper base, greedy"
    CONTRASTS.append(c)
    print(f"pooled over {len(words)} clips / {c['clusters']} speakers / {sum(words)} words: Whistle (forced) {corpus_wer(pooled['Whistle (forced)'], words):.2f}%, "
          f"whisper base greedy {corpus_wer(pooled['whisper base, greedy'], words):.2f}%; {fmt_contrast(c)}  [family A]")
    print(f"Whistle with the language detected instead of forced: {corpus_wer(pooled['Whistle (detected)'], words):.2f}%")
    n = len(words)
    print(f"language identification over the same seven labels: Whistle {lid_w}/{n} (one-sided 95% lower bound {100*clopper_pearson_lower(lid_w, n):.0f}%, "
          f"treating clips as independent), whisper base {lid_b}/{n} (lower bound {100*clopper_pearson_lower(lid_b, n):.0f}%)")
    json.dump(MANIFEST, open("manifest.json", "w"))
    fam = [x for x in CONTRASTS if x["name"] in FAMILY_A]
    sig = dict(zip([x["name"] for x in fam], holm([x["p" ] for x in fam])))
    print(f"\nfamily A complete ({len(fam)} contrasts), Holm at 0.05 (the printed CIs are unadjusted; the verdict uses the adjusted p):")
    for x in fam:
        v = verdict(x, sig[x["name"]])
        VERDICTS.append((x["name"], fmt_contrast(x), v))
        print(f"   {x['name']:48s} {fmt_contrast(x)}  p~{x['p']:.3f}  -> {v}")
    return f"pooled WER Whistle {corpus_wer(pooled['Whistle (forced)'], words):.1f}% vs base {corpus_wer(pooled['whisper base, greedy'], words):.1f}%; language ID {lid_w}/{n} vs {lid_b}/{n}"
 
multilingual()