From 7c8e8d641854d5583965b3fa8ae5ebb02ce7085a Mon Sep 17 00:00:00 2001 From: "Danilo M." Date: Sat, 26 Sep 2026 13:56:01 +0200 Subject: Use beam search in whisper, tell the model about misheard words Whisper wrote "Che ori sono?" twice in one day and the model once answered with the weather. On 10 recorded short questions beam search (beam_size=5) gets 9 right against 8 for greedy, for 0.06 s more; an initial prompt also got 9 but lost one question entirely, so it is not used. The default system prompt now says the user's words come from speech recognition and should be read by how they sound: on "Che ori sono?" stray tool calls went from 1 in 10 to none, other tools unaffected. Co-Authored-By: Claude Opus 5.5 --- assistant.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'assistant.py') diff --git a/assistant.py b/assistant.py index bf6049a..991dfa0 100644 --- a/assistant.py +++ b/assistant.py @@ -52,6 +52,8 @@ DEFAULTS = { "briefly in plain sentences: no markdown, no lists, no emoji. " "Reply in the language the user speaks, Italian or English. " "Write numbers, times and dates in digits, never spelled out. " + "The user's words come from speech recognition and may contain " + "misheard words: read them by how they sound. " "Facts about the user's computer, devices, accounts, mail, calendar " "or money come only from a tool: call one, and if none can tell, " "say you do not know. Never guess them."), @@ -261,7 +263,9 @@ def envelope(audio, sr): def transcribe(lang="auto"): # Always send language: whisper-server's own default is en, not auto. + # Beam search: 9/10 test clips right vs 8/10 greedy, +0.06 s. out = subprocess.run(["curl", "-s", "-F", f"file=@{WAV}", "-F", f"language={lang}", + "-F", "beam_size=5", "-F", "response_format=verbose_json", WHISPER + "/inference"], capture_output=True, text=True).stdout d = json.loads(out) -- cgit v1.2.3