From 0f41149cc23b58a19f986dedb7a8905567afca7a Mon Sep 17 00:00:00 2001 From: "Danilo M." Date: Sat, 26 Sep 2026 09:12:07 +0200 Subject: Log the prompt token count of every LLM round Asks llama-server for usage in the stream, so the log shows how full the model's context is; history, tool definitions and tool results all count. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 17 +++++++++++++++++ assistant.py | 8 +++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 395fed6..77313ac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -393,6 +393,23 @@ phase 2. - Whisper wrote "build system" as two words; the model mapped it to the VM name `buildsystem` without help. +### Context size (2026-09-26) + +Gemma's context is 32768 tokens. Measured against the running model: +system prompt + question 106 prompt tokens, + the 10 tool definitions +946, + a full 20-message history of typical exchanges 1576. Worst case +per question is ~9k (20 long history messages ~3k, up to 3 tool rounds +of 2000-character results ~4k), so the 20-message window and the +2000-character tool cap keep it well inside; each extra tool costs ~85. + +- The assistant logs `context: N prompt tokens` for every LLM round + (`stream_options.include_usage`). Live: a weather question 969, then + 1115 with the tool result. +- The server log (`/var/log/llama-server/server.log`) also prints + `stop processing: n_tokens = N, truncated = 0` per request, but it is + shared with every router client, so it cannot be attributed to the + assistant. `truncated = 1` there would mean an overflow. + ### Barge-in (2026-09-26) A button press cuts the current reply. A global `turn` counter goes up on diff --git a/assistant.py b/assistant.py index 4ea8113..4dc11db 100644 --- a/assistant.py +++ b/assistant.py @@ -253,6 +253,7 @@ def llm_model(): def ask(messages, calls, tools=None): # Yields reply text as it streams; tool-call fragments go into calls. body = {"model": llm_model(), "messages": messages, "stream": True, + "stream_options": {"include_usage": True}, "chat_template_kwargs": {"enable_thinking": False}} if tools: body["tools"] = tools @@ -262,7 +263,12 @@ def ask(messages, calls, tools=None): for line in r: if not line.startswith(b"data: ") or line.strip() == b"data: [DONE]": continue - choices = json.loads(line[6:])["choices"] + chunk = json.loads(line[6:]) + if chunk.get("usage"): + # How full the model's context is: history, tool results and + # tool definitions all count. + log(f"context: {chunk['usage']['prompt_tokens']} prompt tokens") + choices = chunk["choices"] if not choices: continue delta = choices[0]["delta"] -- cgit v1.2.3