voice input: ask takes a WAV, the model transcribes it

An ask with audio sends a transcribing state, has the loaded model
transcribe the clip (thinking off, no tools, the prompt naming Italian
or English), then answers the transcript as a typed question, so the
user event, history and search hold the words. A stop while
transcribing ends the turn like any reply.

Refused: a relative path, a file that is not a WAV, over 60 s (the
ubatch bound, untried for audio), a model without audio input, and an
empty transcript. The length comes from the file size, since a
recording stopped by a signal leaves the header's length unset.

Live: 10/10 clips heard right, the right tool each time.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

1 files changed, 109 insertions, 4 deletions

diff --git a/assistant.py b/assistant.py
index 725f21d..f246101 100644
--- a/assistant.py
+++ b/assistant.py
@@ -39,6 +39,7 @@ import threading
 import time
 import tomllib
 import traceback
+import wave
 import urllib.parse
 import urllib.request
 
@@ -238,6 +239,11 @@ MAX_FILES = 8
 MAX_IMAGE = 20 << 20
 MAX_PAGE_IMAGES = 4                       # scanned PDF pages sent as images
 PAGE_PX = 1600                            # longest side of a page render
+# ~25 tokens a second: 60 s stays under the 2048 ubatch, which an image
+# crossing took the model down (AGENTS.md). Audio past it is untried.
+MAX_AUDIO_S = 60
+HEAR = ("Transcribe this audio exactly, in the language spoken (Italian or English). "
+        "Output only the transcript.")
 
 
 class Refused(Exception):
@@ -560,6 +566,8 @@ def handle(raw):
            else "bad files" if kind == "ask" and not (
                isinstance(cmd.get("files", []), list)
                and all(isinstance(f, str) for f in cmd.get("files", [])))
+           else "bad audio" if kind == "ask" and "audio" in cmd and not (
+               isinstance(cmd["audio"], str) and not cmd["text"].strip())
            else "bad query" if kind == "search" and not isinstance(cmd.get("q"), str)
            else "bad write" if kind == "write" and not (
                isinstance(cmd.get("files"), list) and cmd["files"]
@@ -590,10 +598,10 @@ def command(kind, cmd):
         threading.Thread(target=write, daemon=True,
                          args=(turn, cmd["files"], cmd["prompt"], cmd["dir"], cmd["name"])).start()
     elif kind == "ask":
-        text, files = cmd["text"].strip(), cmd.get("files", [])
-        if text or files:
+        text, files, audio = cmd["text"].strip(), cmd.get("files", []), cmd.get("audio")
+        if text or files or audio:
             interrupt()
-            threading.Thread(target=reply, args=(turn, text, files), daemon=True).start()
+            threading.Thread(target=reply, args=(turn, text, files, audio), daemon=True).start()
     elif kind == "stop":
         interrupt()
     elif kind == "new":
@@ -873,8 +881,53 @@ def write(mine, files, prompt, d, name):
             os.remove(tmp)
 
 
-def reply(mine, text, files=()):
+def transcribe(mine, path):
+    # The model hears the clip. Asked to answer it directly, it replied in
+    # Estonian or Spanish 5 times in 6; the words first, then a typed ask
+    # (AGENTS.md, "Audio input"). History and search need the words anyway.
+    if not os.path.isabs(path):
+        raise Refused("audio path must be absolute")
     try:
+        with wave.open(path) as w:
+            rate = w.getframerate() * w.getnchannels() * w.getsampwidth()
+        # From the size: a recording stopped by a signal has a header whose
+        # length field was never filled in.
+        secs = (os.path.getsize(path) - 44) / rate
+    except (OSError, EOFError, wave.Error, ZeroDivisionError):
+        raise Refused(f"not a readable WAV: {os.path.basename(path)}") from None
+    if secs > MAX_AUDIO_S:
+        raise Refused("recording too long")
+    model, inputs = model_inputs()
+    if inputs is not None and "audio" not in inputs:
+        raise Refused(f"{model} cannot hear audio")
+    with open(path, "rb") as f:
+        b64 = base64.b64encode(f.read()).decode()
+    with lock:
+        if turn != mine:
+            return None
+        # A turn in flight from here: a stop ends it with done, as a reply.
+        cur["pending"] = {"user": "", "reply": "", "think": "", "tools": [], "files": []}
+        emit({"type": "state", "state": "transcribing"})
+    msgs = [{"role": "user", "content": [
+        {"type": "text", "text": HEAR},
+        {"type": "input_audio", "input_audio": {"data": b64, "format": "wav"}}]}]
+    text = "".join(p for k, p in ask(msgs, {}, None, mine, think=False, meter=False)
+                   if k == "token").strip()
+    with lock:
+        if turn != mine:
+            return None
+    if not text:
+        raise Refused("heard nothing")
+    log(f"heard: {text}")
+    return text
+
+
+def reply(mine, text, files=(), audio=None):
+    try:
+        if audio:
+            text = transcribe(mine, audio)
+            if text is None:
+                return
         answer(mine, text, files)
     except Exception as e:
         # A refused file is the user's to fix, not a bug: one line, no trace.
@@ -1238,6 +1291,7 @@ def test():
 
         def do_POST(self):
             script["body"] = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+            script.get("bodies", []).append(script["body"])
             if script.get("hold"):
                 # A long prompt pass: nothing is sent, so only the client
                 # hanging up ends it. Records how soon that happened.
@@ -1429,6 +1483,57 @@ def test():
         assert "chat supports" in until("error")[-1]["message"]
         send(type="ask", text="x", files="not a list")
         assert until("error")[-1]["message"] == "bad files"
+        # Audio: the model transcribes, then the words are asked as text.
+        import wave
+
+        def tiny_wav(p, secs, rate=100):
+            with wave.open(p, "wb") as w:
+                w.setnchannels(1)
+                w.setsampwidth(2)
+                w.setframerate(rate)
+                w.writeframes(b"\0\0" * int(secs * rate))
+        wav, long = os.path.join(fd, "q.wav"), os.path.join(fd, "long.wav")
+        tiny_wav(wav, 3)
+        tiny_wav(long, 61)
+        script["inputs"] = ["text", "image", "audio"]
+        script["bodies"] = []
+        script["replies"] = [[{"content": " Che ore sono? "}], [{"content": "Noon."}]]
+        send(type="ask", text="", audio=wav)
+        seen = until("user")
+        assert {"type": "state", "state": "transcribing"} in seen
+        assert seen[-1]["text"] == "Che ore sono?"
+        until("done")
+        heard, asked = script["bodies"]
+        with open(wav, "rb") as fh:
+            b64 = base64.b64encode(fh.read()).decode()
+        assert heard["messages"][0]["content"][1] == {
+            "type": "input_audio", "input_audio": {"data": b64, "format": "wav"}}
+        assert "tools" not in heard and heard["chat_template_kwargs"]["enable_thinking"] is False
+        assert asked["messages"][-1]["content"].endswith("] Che ore sono?")
+        assert cur["turns"][-1]["user"] == "Che ore sono?"
+        script["replies"] = [[{"content": " "}]]
+        send(type="ask", text="", audio=wav)
+        assert until("error")[-1]["message"] == "heard nothing"
+        assert until("state")[-1]["state"] == "idle"
+        for p, msg in ((long, "recording too long"), ("q.wav", "audio path must be absolute"),
+                       (pdf, "not a readable WAV")):
+            send(type="ask", text="", audio=p)
+            e = until("error")[-1]
+            assert msg in e["message"] and e["turn"] is True, (p, e)
+        send(type="ask", text="hi", audio=wav)
+        assert until("error")[-1]["message"] == "bad audio"
+        # A stop while transcribing ends the turn like any reply.
+        script.update(hold=True, hung_up=None)
+        script["replies"] = [[]]
+        send(type="ask", text="", audio=wav)
+        until("state")
+        send(type="stop")
+        seen = until("state")
+        assert seen[-1]["state"] == "idle" and seen[-2]["stopped"] is True, seen
+        script["hold"] = False
+        script["inputs"] = ["text", "image"]
+        send(type="ask", text="", audio=wav)
+        assert until("error")[-1]["message"] == "m cannot hear audio"
         # write: the reply goes to a file, not the conversation.
         before = len(list_convs(DB))
         with open(os.path.join(fd, "doc.md"), "w") as fh:     # f is the socket's