feat: log permanently failed downloads to failures.json

State advances past a download that exhausts MAX_RETRIES so one broken
file cannot wedge the backup, which until now left a silent gap in the
archive. Failures are appended to failures.json in the archive dir as
JSONL, one {id, file, error} per line; appending is a single write, so a
crash costs the last record rather than the file.

download_with_retry now returns the last exception or None on success,
replacing the True/False return, so the caller has an error to log.

Nothing reads or prunes the file: it is a record, not a retry queue.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

1 files changed, 32 insertions, 6 deletions

diff --git a/tg_backup.py b/tg_backup.py
index 6c77878..382a756 100644
--- a/tg_backup.py
+++ b/tg_backup.py
@@ -75,6 +75,15 @@ def save_state(archive_dir, state):
         json.dump(state, f, indent=4)
     os.replace(tmp_file, state_file)
 
+def record_failure(archive_dir, message_id, filename, error):
+    # State advances past a permanently failed download on purpose, so without
+    # this the message ID is lost and the backup silently has a hole in it.
+    # Append-only log, one JSON object per line: a partial last line costs one
+    # record, not the whole file.
+    entry = {'id': message_id, 'file': filename, 'error': str(error)}
+    with open(archive_dir / "failures.json", 'a') as f:
+        f.write(json.dumps(entry) + "\n")
+
 def safe_filename(message_id, original_name, ext):
     # original_name comes from the sender. Strip path components so a name like
     # "../../.ssh/authorized_keys" resolves to a plain file inside archive_dir.
@@ -87,12 +96,14 @@ async def download_with_retry(client, message, filepath):
     # Download to a .part file so an interrupted run never leaves a truncated
     # file that the exists() check would mistake for a complete download.
     partpath = filepath.with_name(filepath.name + ".part")
+    last_error = f"no attempt made (MAX_RETRIES={MAX_RETRIES})"
     for attempt in range(MAX_RETRIES):
         try:
             await client.download_media(message, file=str(partpath))
             os.replace(partpath, filepath)
-            return True
+            return None
         except FloodWaitError as e:
+            last_error = e
             print(f"Rate limited on download. Waiting {e.seconds}s... (Attempt {attempt+1}/{MAX_RETRIES})")
             await asyncio.sleep(e.seconds)
         except (KeyboardInterrupt, asyncio.CancelledError):
@@ -101,6 +112,7 @@ async def download_with_retry(client, message, filepath):
         except Exception as e:
             # Network drops, expired file references, disk errors. Backing off
             # and retrying beats killing an overnight run on one bad socket.
+            last_error = e
             delay = 2 ** attempt
             print(f"Download error on {filepath.name}: {e}. "
                   f"Retrying in {delay}s... (Attempt {attempt+1}/{MAX_RETRIES})")
@@ -108,7 +120,8 @@ async def download_with_retry(client, message, filepath):
             await asyncio.sleep(delay)
     partpath.unlink(missing_ok=True)
     print(f"Failed to download {filepath.name} after {MAX_RETRIES} retries.")
-    return False
+    # Returns the last error, or None on success: the caller logs it to failures.json.
+    return last_error
 
 def migrate_session(archive_dir):
     # Sessions used to live per-archive-dir, which forced a fresh login for every
@@ -206,11 +219,14 @@ async def run_backup(target, archive_dir):
                     
                     if not filepath.exists():
                         print(f"Downloading {filename}...")
-                        success = await download_with_retry(client, message, filepath)
-                        if not success:
-                            # If download permanently fails, we still advance state 
+                        error = await download_with_retry(client, message, filepath)
+                        if error is not None:
+                            # If download permanently fails, we still advance state
                             # so we don't get stuck on a broken file forever.
-                            print("Skipping file due to max retries.")
+                            # Log it so the gap is auditable instead of silent.
+                            record_failure(archive_dir, message.id, filename, error)
+                            print(f"Skipping file due to max retries "
+                                  f"(logged to {archive_dir / 'failures.json'}).")
                     else:
                         print(f"Skipping {filename} (exists).")
                         
@@ -292,6 +308,16 @@ def _self_check():
     assert load_state(archive).get('target') == '-1001234567890'
     assert load_state(Path(tempfile.mkdtemp())).get('target') is None
 
+    # Failures append, one parseable record per line, and survive an exception
+    # object being passed straight in.
+    record_failure(archive, 11, '11_a.jpg', OSError('disk full'))
+    record_failure(archive, 12, '12_b.mp4', 'timeout')
+    lines = (archive / 'failures.json').read_text().splitlines()
+    assert len(lines) == 2, f"expected 2 failure records, got {len(lines)}"
+    first, second = json.loads(lines[0]), json.loads(lines[1])
+    assert first == {'id': 11, 'file': '11_a.jpg', 'error': 'disk full'}
+    assert second['id'] == 12 and second['error'] == 'timeout'
+
     print("self-check OK")
 
 if __name__ == "__main__":