Read a nameless file's content to tell whether it is text

A file whose name gives no type, such as a shebang script or a Makefile,
was refused as not text. When mimetypes knows nothing, the first 4 KB
decide: no NUL and valid UTF-8 (a character cut by the window allowed)
is text. A name that does say something, even application/octet-stream,
is still taken at its word.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

1 files changed, 34 insertions, 2 deletions

diff --git a/assistant.py b/assistant.py
index 1341ddb..ebc8ba9 100644
--- a/assistant.py
+++ b/assistant.py
@@ -243,16 +243,35 @@ class Refused(Exception):
 
 
 def kind_of(path):
-    mime = mimetypes.guess_type(path)[0] or ""
+    mime = mimetypes.guess_type(path)[0]
     if mime in IMAGE_TYPES:
         return "image"
     if mime == "application/pdf":
         return "pdf"
-    if mime.startswith("text/") or os.path.splitext(path)[1].lower() in CODE_SUFFIXES:
+    if (mime or "").startswith("text/") or os.path.splitext(path)[1].lower() in CODE_SUFFIXES:
+        return "text"
+    if mime is None and looks_like_text(path):    # no telling name: a script, a Makefile
         return "text"
     return "other"
 
 
+def looks_like_text(path):
+    # The first 4 KB: no NUL and valid UTF-8 (a character cut at the end of
+    # the window is allowed).
+    try:
+        with open(path, "rb") as f:
+            head = f.read(4096)
+    except OSError:
+        return False
+    if b"\0" in head:
+        return False
+    try:
+        head.decode("utf-8")
+    except UnicodeDecodeError as e:
+        return len(head) == 4096 and e.reason == "unexpected end of data"
+    return True
+
+
 def sha256(path):
     h = hashlib.sha256()
     with open(path, "rb") as f:
@@ -992,6 +1011,19 @@ def test():
         f.write(b"\x00\x01")
     assert [kind_of(p) for p in (code, png, pdf, other, "/x/README.md", "/x/b.SlackBuild")] == \
         ["text", "image", "pdf", "other", "text", "text"]
+    # No suffix: the content decides. A script or plain UTF-8 is text; a NUL
+    # byte, or bytes that are not UTF-8, are not.
+    for name, data, want in (("run", b"#!/bin/bash\necho hi\n", "text"),
+                             ("notes", "caff\u00e8 alle 9\n".encode(), "text"),
+                             ("blob", b"\x7fELF\x00\x01", "other"),
+                             ("latin", "caff\u00e8".encode("latin-1"), "other")):
+        with open(os.path.join(fd, name), "wb") as f:
+            f.write(data)
+        assert kind_of(os.path.join(fd, name)) == want, name
+    assert kind_of(os.path.join(fd, "missing")) == "other"
+    with open(os.path.join(fd, "long"), "wb") as f:      # a character cut by the window
+        f.write(b"a" * 4095 + "\u00e8 tail".encode())
+    assert kind_of(os.path.join(fd, "long")) == "text"
     blocks, refs, chips = attach([code, pdf, png], 100000)
     assert blocks[0].startswith("--- file: a.py ---\nprint('hi')") and "(truncated)" not in blocks[0]
     assert "[page 1]\nhello pdf" in blocks[1] and "[page 2]" in blocks[1]