diff options
| -rw-r--r-- | AGENTS.md | 9 | ||||
| -rw-r--r-- | assistant.py | 113 | ||||
| -rw-r--r-- | docs/superpowers/specs/2026-10-16-text-daemon-design.md | 2 | ||||
| -rw-r--r-- | docs/superpowers/specs/2026-10-19-voice-input-design.md | 19 |
4 files changed, 130 insertions, 13 deletions
@@ -148,7 +148,14 @@ no `data:` prefix), no whisper. The 10 clips in 20/20. ~70 more prompt tokens per clip. **Decision: transcribe first, then ask the transcript as text.** History and search need the words anyway. -- Not tried: a clip long enough to cross the 2048 ubatch (~80 s). +- Not tried: a clip long enough to cross the 2048 ubatch (~80 s), so + the daemon refuses clips over 60 s. +- **Through the daemon** (`ask` with `audio`, a second instance on a + scratch socket): 10/10 heard right, `user` event 0.36-0.87 s after the + ask, and the right tool every time (weather, updates, mail, calendar, + now; none for the time in Italian and "Ciao"). "What's the weather + tomorrow?" was answered in Italian: the reply after the weather tool, + not the transcript, which was English. ### Attachments, live check (2026-10-16) diff --git a/assistant.py b/assistant.py index 725f21d..f246101 100644 --- a/assistant.py +++ b/assistant.py @@ -39,6 +39,7 @@ import threading import time import tomllib import traceback +import wave import urllib.parse import urllib.request @@ -238,6 +239,11 @@ MAX_FILES = 8 MAX_IMAGE = 20 << 20 MAX_PAGE_IMAGES = 4 # scanned PDF pages sent as images PAGE_PX = 1600 # longest side of a page render +# ~25 tokens a second: 60 s stays under the 2048 ubatch, which an image +# crossing took the model down (AGENTS.md). Audio past it is untried. +MAX_AUDIO_S = 60 +HEAR = ("Transcribe this audio exactly, in the language spoken (Italian or English). " + "Output only the transcript.") class Refused(Exception): @@ -560,6 +566,8 @@ def handle(raw): else "bad files" if kind == "ask" and not ( isinstance(cmd.get("files", []), list) and all(isinstance(f, str) for f in cmd.get("files", []))) + else "bad audio" if kind == "ask" and "audio" in cmd and not ( + isinstance(cmd["audio"], str) and not cmd["text"].strip()) else "bad query" if kind == "search" and not isinstance(cmd.get("q"), str) else "bad write" if kind == "write" and not ( isinstance(cmd.get("files"), list) and cmd["files"] @@ -590,10 +598,10 @@ def command(kind, cmd): threading.Thread(target=write, daemon=True, args=(turn, cmd["files"], cmd["prompt"], cmd["dir"], cmd["name"])).start() elif kind == "ask": - text, files = cmd["text"].strip(), cmd.get("files", []) - if text or files: + text, files, audio = cmd["text"].strip(), cmd.get("files", []), cmd.get("audio") + if text or files or audio: interrupt() - threading.Thread(target=reply, args=(turn, text, files), daemon=True).start() + threading.Thread(target=reply, args=(turn, text, files, audio), daemon=True).start() elif kind == "stop": interrupt() elif kind == "new": @@ -873,8 +881,53 @@ def write(mine, files, prompt, d, name): os.remove(tmp) -def reply(mine, text, files=()): +def transcribe(mine, path): + # The model hears the clip. Asked to answer it directly, it replied in + # Estonian or Spanish 5 times in 6; the words first, then a typed ask + # (AGENTS.md, "Audio input"). History and search need the words anyway. + if not os.path.isabs(path): + raise Refused("audio path must be absolute") try: + with wave.open(path) as w: + rate = w.getframerate() * w.getnchannels() * w.getsampwidth() + # From the size: a recording stopped by a signal has a header whose + # length field was never filled in. + secs = (os.path.getsize(path) - 44) / rate + except (OSError, EOFError, wave.Error, ZeroDivisionError): + raise Refused(f"not a readable WAV: {os.path.basename(path)}") from None + if secs > MAX_AUDIO_S: + raise Refused("recording too long") + model, inputs = model_inputs() + if inputs is not None and "audio" not in inputs: + raise Refused(f"{model} cannot hear audio") + with open(path, "rb") as f: + b64 = base64.b64encode(f.read()).decode() + with lock: + if turn != mine: + return None + # A turn in flight from here: a stop ends it with done, as a reply. + cur["pending"] = {"user": "", "reply": "", "think": "", "tools": [], "files": []} + emit({"type": "state", "state": "transcribing"}) + msgs = [{"role": "user", "content": [ + {"type": "text", "text": HEAR}, + {"type": "input_audio", "input_audio": {"data": b64, "format": "wav"}}]}] + text = "".join(p for k, p in ask(msgs, {}, None, mine, think=False, meter=False) + if k == "token").strip() + with lock: + if turn != mine: + return None + if not text: + raise Refused("heard nothing") + log(f"heard: {text}") + return text + + +def reply(mine, text, files=(), audio=None): + try: + if audio: + text = transcribe(mine, audio) + if text is None: + return answer(mine, text, files) except Exception as e: # A refused file is the user's to fix, not a bug: one line, no trace. @@ -1238,6 +1291,7 @@ def test(): def do_POST(self): script["body"] = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + script.get("bodies", []).append(script["body"]) if script.get("hold"): # A long prompt pass: nothing is sent, so only the client # hanging up ends it. Records how soon that happened. @@ -1429,6 +1483,57 @@ def test(): assert "chat supports" in until("error")[-1]["message"] send(type="ask", text="x", files="not a list") assert until("error")[-1]["message"] == "bad files" + # Audio: the model transcribes, then the words are asked as text. + import wave + + def tiny_wav(p, secs, rate=100): + with wave.open(p, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(rate) + w.writeframes(b"\0\0" * int(secs * rate)) + wav, long = os.path.join(fd, "q.wav"), os.path.join(fd, "long.wav") + tiny_wav(wav, 3) + tiny_wav(long, 61) + script["inputs"] = ["text", "image", "audio"] + script["bodies"] = [] + script["replies"] = [[{"content": " Che ore sono? "}], [{"content": "Noon."}]] + send(type="ask", text="", audio=wav) + seen = until("user") + assert {"type": "state", "state": "transcribing"} in seen + assert seen[-1]["text"] == "Che ore sono?" + until("done") + heard, asked = script["bodies"] + with open(wav, "rb") as fh: + b64 = base64.b64encode(fh.read()).decode() + assert heard["messages"][0]["content"][1] == { + "type": "input_audio", "input_audio": {"data": b64, "format": "wav"}} + assert "tools" not in heard and heard["chat_template_kwargs"]["enable_thinking"] is False + assert asked["messages"][-1]["content"].endswith("] Che ore sono?") + assert cur["turns"][-1]["user"] == "Che ore sono?" + script["replies"] = [[{"content": " "}]] + send(type="ask", text="", audio=wav) + assert until("error")[-1]["message"] == "heard nothing" + assert until("state")[-1]["state"] == "idle" + for p, msg in ((long, "recording too long"), ("q.wav", "audio path must be absolute"), + (pdf, "not a readable WAV")): + send(type="ask", text="", audio=p) + e = until("error")[-1] + assert msg in e["message"] and e["turn"] is True, (p, e) + send(type="ask", text="hi", audio=wav) + assert until("error")[-1]["message"] == "bad audio" + # A stop while transcribing ends the turn like any reply. + script.update(hold=True, hung_up=None) + script["replies"] = [[]] + send(type="ask", text="", audio=wav) + until("state") + send(type="stop") + seen = until("state") + assert seen[-1]["state"] == "idle" and seen[-2]["stopped"] is True, seen + script["hold"] = False + script["inputs"] = ["text", "image"] + send(type="ask", text="", audio=wav) + assert until("error")[-1]["message"] == "m cannot hear audio" # write: the reply goes to a file, not the conversation. before = len(list_convs(DB)) with open(os.path.join(fd, "doc.md"), "w") as fh: # f is the socket's diff --git a/docs/superpowers/specs/2026-10-16-text-daemon-design.md b/docs/superpowers/specs/2026-10-16-text-daemon-design.md index 223af38..6ae4dd6 100644 --- a/docs/superpowers/specs/2026-10-16-text-daemon-design.md +++ b/docs/superpowers/specs/2026-10-16-text-daemon-design.md @@ -96,7 +96,7 @@ Outbound: | Event | Fields | | --- | --- | | `hello` | `state`, `model`, `ctx_used`, `ctx_max`, `id`, `turns` (the current conversation), `pending` (`{user, reply, think, tools}` of a reply in flight, or null) | -| `state` | `state`: `idle`, `thinking`, `tool` (with `tool`) | +| `state` | `state`: `idle`, `transcribing` (an `audio` ask, see the voice input spec), `thinking`, `tool` (with `tool`) | | `user` | `text`: the question, echoed so every client draws it | | `think` | `delta` | | `token` | `delta` | diff --git a/docs/superpowers/specs/2026-10-19-voice-input-design.md b/docs/superpowers/specs/2026-10-19-voice-input-design.md index 860967a..551ed60 100644 --- a/docs/superpowers/specs/2026-10-19-voice-input-design.md +++ b/docs/superpowers/specs/2026-10-19-voice-input-design.md @@ -1,6 +1,6 @@ # Voice input: design -Date: 2026-10-19. Status: proposed. +Date: 2026-10-19. Status: implemented (2026-10-19), live-checked. ## Goal @@ -40,9 +40,13 @@ it the reply runs as a typed `ask` with the transcript as `text`: the `user` event, history, search and the date stamp all see the words, not the audio. -Errors answer with `error` (with `turn`) and nothing more is sent: +`audio` that is not a string, or comes with text, answers `bad audio` +(no `turn`, like `bad files`). Otherwise errors answer with `error` +(with `turn`) and nothing more is sent: - not an absolute path, missing, not a readable WAV (stdlib `wave`); + the length comes from the file size, since a recording stopped by a + signal leaves the header's length unset; - longer than 60 s: "recording too long". 60 s is about 1500 audio tokens; past the 2048 ubatch an image crashed the model, and audio has not been tried there; @@ -52,17 +56,18 @@ Errors answer with `error` (with `turn`) and nothing more is sent: ## Transcription -One non-streamed-to-the-client request, thinking off, temperature 0.6, -no tools, no history, no system prompt: +One non-streamed-to-the-client request, thinking off, the configured +temperature (0.6 when tested), no tools, no history, no system prompt: - a text part, "Transcribe this audio exactly, in the language spoken (Italian or English). Output only the transcript."; - an `input_audio` part, `{"data": <base64>, "format": "wav"}`, no `data:` prefix (llamachat's finding). -It goes through `ask()` with the reply's turn, so a stop while -transcribing shuts the request like any other. The transcript is -stripped and goes to `answer()` unchanged. +It goes through `ask()` with the reply's turn. The turn is pending from +the `transcribing` state on, so a stop while transcribing shuts the +request and ends with `done` (stopped) and `idle`, like any reply. The +transcript is stripped and goes to `answer()` unchanged. ## Testing |