diff options
| -rw-r--r-- | CLAUDE.md | 8 | ||||
| -rw-r--r-- | README.md | 23 | ||||
| -rw-r--r-- | tg_backup.py | 38 |
3 files changed, 57 insertions, 12 deletions
@@ -28,7 +28,7 @@ Three things are load-bearing and easy to break: 1. **`reverse=True` on `iter_messages`** (`tg_backup.py:75`). Processing must be oldest-to-newest. `state['last_id']` is a high-water mark used as `min_id` on resume; iterating newest-first would make it skip unfetched history permanently. 2. **State saved after every message**, not at the end, via temp-file + `os.replace`. Cheap insurance against interruption; do not "optimize" into a batched write without keeping resume correctness, and keep the write atomic. A truncated `state.json` makes the next run unstartable. -3. **Two-level retry handling.** The inner `download_with_retry` retries a single file up to `MAX_RETRIES`: FloodWait sleeps the server-requested interval, other exceptions (network drops, expired file references, disk errors) get exponential backoff so one flaky socket can't kill an overnight run. The outer `while True` in `run_backup` catches FloodWait on history fetching, saves state, sleeps, and restarts the iterator from the saved id. A permanently failed download still advances state on purpose, so one broken file can't wedge the backup forever. +3. **Two-level retry handling.** The inner `download_with_retry` retries a single file up to `MAX_RETRIES`: FloodWait sleeps the server-requested interval, other exceptions (network drops, expired file references, disk errors) get exponential backoff so one flaky socket can't kill an overnight run. The outer `while True` in `run_backup` catches FloodWait on history fetching, saves state, sleeps, and restarts the iterator from the saved id. A permanently failed download still advances state on purpose, so one broken file can't wedge the backup forever; the gap is recorded in `failures.json` (see below) rather than lost. `download_with_retry` returns the last exception, or `None` on success, so the caller has something to log. 4. **Media downloads to `{name}.part`, renamed on success.** The dedup check is `filepath.exists()`, so a truncated file at the final path would be skipped forever as if complete. The `.part` file is unlinked on failure and on interrupt, leaving the message to be retried on the next run. Resume/dedup is also file-based: media is named `{message.id}_{sanitized_name}` (see `safe_filename`, which strips sender-controlled path components) and an existing path is skipped, so state loss degrades to a slow rescan rather than re-downloading everything. @@ -39,7 +39,11 @@ Resume/dedup is also file-based: media is named `{message.id}_{sanitized_name}` ## Layout -`--archive-dir` holds media files and `state.json`, and is created if missing (`parents=True`). It is per-chat: one dir per backed-up chat. +`--archive-dir` holds media files, `state.json`, and `failures.json`, and is created if missing (`parents=True`). It is per-chat: one dir per backed-up chat. + +`state.json` also stores the `target` the dir was last backed up with, so subsequent runs need only `--archive-dir`. An explicit `--target` still wins; it is only required for a dir with no saved target (a first run, or one created before this was added). Nothing guards against pointing an existing dir at a *different* chat: `last_id` from the old chat would be applied as `min_id` to the new one and skip its history. + +`failures.json` is append-only JSONL, one `{"id", "file", "error"}` object per line, written only when a download exhausts `MAX_RETRIES`. JSONL rather than a JSON array so appending is a single write with no read-modify-write: a crash costs the last line, not the file. Read it with `jq -s`. Nothing reads or prunes it, and a message failing on two runs is logged twice. Auth lives in `~/.config/telegram_backup/` (mode 0700): `config.json` with the api_id/api_hash, and `session.session`, a single Telethon session **shared by every archive dir**, so one login covers all chats. `migrate_session` relocates a pre-existing `<archive_dir>/session.session` to the shared path on first run rather than forcing a re-login; it is a no-op once the shared session exists. It uses `shutil.move`, not `os.replace`, because the two paths can be on different filesystems. @@ -60,11 +60,19 @@ Numeric IDs are more durable than usernames, which their owners can change. Chat `--target` also accepts `me` (Saved Messages), t.me links, and invite links for groups you have already joined. +Update it later with just the directory: + +```bash +tg_backup.py --archive-dir ~/backups/examplegroup +``` + +The target is recorded in the archive directory on the first run, so repeat runs do not need it. Passing `--target` again still overrides what was saved. Point a directory at a different chat and the resume position carries over, which will skip that chat's history; use a fresh directory instead. + ### Options | Option | Default | Meaning | |---|---|---| -| `--target` | required | Chat to back up: `@username`, numeric ID, t.me link, or `me` | +| `--target` | first run only | Chat to back up: `@username`, numeric ID, t.me link, or `me`. Reused from the archive directory afterwards | | `--archive-dir` | `./tg_archive` | Where media and resume state are written; created if missing | | `--list-chats` | | Print your dialogs with IDs and usernames, then exit | | `--self-check` | | Run internal assertions and exit. No network, no login | @@ -82,11 +90,17 @@ Without `=`, argparse reads the leading `-` as the start of another flag. ## How it works -Messages are processed oldest to newest, and the last message ID seen is written to `state.json` in the archive directory after every message. An interrupted run resumes from there instead of rescanning the whole chat. +Messages are processed oldest to newest, and the last message ID seen is written to `state.json` in the archive directory after every message, alongside the chat it belongs to. An interrupted run resumes from there instead of rescanning the whole chat. Media is saved as `{message_id}_{original_name}`. Downloads land in a temporary `.part` file and are renamed only on success, so a killed run never leaves a truncated file that a later run would mistake for a complete one. Files that already exist are skipped, so even a lost `state.json` degrades to a slow rescan rather than re-downloading everything. -Rate limits are handled at two levels: Telegram's requested wait is honoured for both individual downloads and history fetches, and other errors (dropped connections, expired file references) are retried with exponential backoff. A download that fails permanently is logged and skipped rather than stalling the run forever. +Rate limits are handled at two levels: Telegram's requested wait is honoured for both individual downloads and history fetches, and other errors (dropped connections, expired file references) are retried with exponential backoff. A download that fails permanently is skipped rather than stalling the run forever, and its message ID is appended to `failures.json` so the gap is auditable: + +```bash +jq -s 'map(.id)' ~/backups/examplegroup/failures.json +``` + +One JSON object per line, written only after all retries are exhausted. Nothing retries or prunes these; the file is a record, not a queue. Only media is saved. Message text and captions are not. @@ -96,7 +110,8 @@ Only media is saved. Message text and captions are not. |---|---| | `~/.config/telegram_backup/config.json` | API credentials | | `~/.config/telegram_backup/session.session` | Login session, shared across all archive directories | -| `<archive-dir>/state.json` | Resume position for that chat | +| `<archive-dir>/state.json` | Resume position and saved target for that chat | +| `<archive-dir>/failures.json` | Message IDs whose media never downloaded | | `<archive-dir>/*` | Downloaded media | The session file is account credentials: anyone holding it can read your Telegram. It is created mode `0600` inside a `0700` directory. Keep it out of version control and off shared storage. diff --git a/tg_backup.py b/tg_backup.py index 6c77878..382a756 100644 --- a/tg_backup.py +++ b/tg_backup.py @@ -75,6 +75,15 @@ def save_state(archive_dir, state): json.dump(state, f, indent=4) os.replace(tmp_file, state_file) +def record_failure(archive_dir, message_id, filename, error): + # State advances past a permanently failed download on purpose, so without + # this the message ID is lost and the backup silently has a hole in it. + # Append-only log, one JSON object per line: a partial last line costs one + # record, not the whole file. + entry = {'id': message_id, 'file': filename, 'error': str(error)} + with open(archive_dir / "failures.json", 'a') as f: + f.write(json.dumps(entry) + "\n") + def safe_filename(message_id, original_name, ext): # original_name comes from the sender. Strip path components so a name like # "../../.ssh/authorized_keys" resolves to a plain file inside archive_dir. @@ -87,12 +96,14 @@ async def download_with_retry(client, message, filepath): # Download to a .part file so an interrupted run never leaves a truncated # file that the exists() check would mistake for a complete download. partpath = filepath.with_name(filepath.name + ".part") + last_error = f"no attempt made (MAX_RETRIES={MAX_RETRIES})" for attempt in range(MAX_RETRIES): try: await client.download_media(message, file=str(partpath)) os.replace(partpath, filepath) - return True + return None except FloodWaitError as e: + last_error = e print(f"Rate limited on download. Waiting {e.seconds}s... (Attempt {attempt+1}/{MAX_RETRIES})") await asyncio.sleep(e.seconds) except (KeyboardInterrupt, asyncio.CancelledError): @@ -101,6 +112,7 @@ async def download_with_retry(client, message, filepath): except Exception as e: # Network drops, expired file references, disk errors. Backing off # and retrying beats killing an overnight run on one bad socket. + last_error = e delay = 2 ** attempt print(f"Download error on {filepath.name}: {e}. " f"Retrying in {delay}s... (Attempt {attempt+1}/{MAX_RETRIES})") @@ -108,7 +120,8 @@ async def download_with_retry(client, message, filepath): await asyncio.sleep(delay) partpath.unlink(missing_ok=True) print(f"Failed to download {filepath.name} after {MAX_RETRIES} retries.") - return False + # Returns the last error, or None on success: the caller logs it to failures.json. + return last_error def migrate_session(archive_dir): # Sessions used to live per-archive-dir, which forced a fresh login for every @@ -206,11 +219,14 @@ async def run_backup(target, archive_dir): if not filepath.exists(): print(f"Downloading {filename}...") - success = await download_with_retry(client, message, filepath) - if not success: - # If download permanently fails, we still advance state + error = await download_with_retry(client, message, filepath) + if error is not None: + # If download permanently fails, we still advance state # so we don't get stuck on a broken file forever. - print("Skipping file due to max retries.") + # Log it so the gap is auditable instead of silent. + record_failure(archive_dir, message.id, filename, error) + print(f"Skipping file due to max retries " + f"(logged to {archive_dir / 'failures.json'}).") else: print(f"Skipping {filename} (exists).") @@ -292,6 +308,16 @@ def _self_check(): assert load_state(archive).get('target') == '-1001234567890' assert load_state(Path(tempfile.mkdtemp())).get('target') is None + # Failures append, one parseable record per line, and survive an exception + # object being passed straight in. + record_failure(archive, 11, '11_a.jpg', OSError('disk full')) + record_failure(archive, 12, '12_b.mp4', 'timeout') + lines = (archive / 'failures.json').read_text().splitlines() + assert len(lines) == 2, f"expected 2 failure records, got {len(lines)}" + first, second = json.loads(lines[0]), json.loads(lines[1]) + assert first == {'id': 11, 'file': '11_a.jpg', 'error': 'disk full'} + assert second['id'] == 12 and second['error'] == 'timeout' + print("self-check OK") if __name__ == "__main__": |