From 3126606d5e761e151fee0caab34d2cca2b9b3ee7 Mon Sep 17 00:00:00 2001 From: "Danilo M." Date: Tue, 8 Sep 2026 13:29:48 +0200 Subject: feat: redact recipient identifiers inside URLs Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KphFXTc2QajxXsHWyvGJ4R --- abusectl/redact.py | 105 ++++++++++++++++++++++++++++++++++++++++++++++++++ tests/test_redact.py | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 abusectl/redact.py create mode 100644 tests/test_redact.py diff --git a/abusectl/redact.py b/abusectl/redact.py new file mode 100644 index 0000000..3ca744f --- /dev/null +++ b/abusectl/redact.py @@ -0,0 +1,105 @@ +# Copyright (C) 2026 Danilo M. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Strip recipient identifiers out of phishing URLs before they are reported. + +A phishing URL commonly carries the recipient's identity in its query string: +``?e=
``, ``?u=``, ``?id=``. Publishing that +in a report leaks the victim's address to third parties (AbuseIPDB, URLhaus, +VirusTotal, abuse desks) and deanonymises the reporter, since abuse desks +forward reports to the abused customer and URLhaus is a public feed. + +So parameter NAMES are kept, because they fingerprint the phishing kit, and +parameter VALUES are redacted, because the token is unique per recipient by +design. Keeping it would make correlation worse, not better: two messages +from the same campaign would look like different URLs. + +The one exception is not a hole in the rule but its own logic applied +correctly: a redirector carries its destination in a parameter, e.g. +``http://t.example.invalid/c?url=http%3A%2F%2Fevil.invalid%2Fp``. That +destination is an indicator in its own right, not a recipient identifier, so +a value that decodes to an http(s) URL is recovered separately by +``url_valued_parameters``. It is never substituted back into ``url()``'s +output: the redirector's own query string still redacts every value, +including the one the target was recovered from, because recovery reports an +extra indicator, it does not license leaving the recipient's token in place. +""" + +import re +import string +from urllib.parse import ( + parse_qsl, + unquote, + urlencode, + urlsplit, + urlunsplit, +) + +REDACTED = "REDACTED" + +# 16 is above ordinary path words ("subscribe" is 9) and below any plausible +# encoding of an email address, so it flags identifiers without flagging +# vocabulary. +_MIN_SUSPECT_LENGTH = 16 + +_SUSPECT_CHARS = set(string.ascii_letters + string.digits + "+/=_-") + + +def url(raw: str) -> str: + """Return raw with every query parameter value replaced by REDACTED. + + Parameter names, scheme, host and path are left untouched. A URL with no + query string is returned unchanged. + """ + parts = urlsplit(raw) + if not parts.query: + return raw + pairs = parse_qsl(parts.query, keep_blank_values=True) + redacted_query = urlencode([(name, REDACTED) for name, _ in pairs]) + return urlunsplit(parts._replace(query=redacted_query)) + + +def suspect_path_segments(raw: str) -> list[str]: + """Return path segments that look like an encoded identifier. + + A segment made only of base64/hex-shaped characters and at least + ``_MIN_SUSPECT_LENGTH`` long is flagged for human review, never redacted: + unlike a query value, a path segment may be the very thing being + reported. + """ + parts = urlsplit(raw) + segments = [seg for seg in parts.path.split("/") if seg] + return [ + seg + for seg in segments + if len(seg) >= _MIN_SUSPECT_LENGTH and set(seg) <= _SUSPECT_CHARS + ] + + +def url_valued_parameters(raw: str) -> list[str]: + """Return query parameter values that are themselves http(s) URLs. + + Values are URL-decoded before the scheme check, which is how a + redirector's destination parameter is recovered as an indicator in its + own right. + """ + parts = urlsplit(raw) + if not parts.query: + return [] + found = [] + for _, value in parse_qsl(parts.query, keep_blank_values=True): + decoded = unquote(value) + if re.match(r"(?i)^https?://", decoded): + found.append(decoded) + return found diff --git a/tests/test_redact.py b/tests/test_redact.py new file mode 100644 index 0000000..91ab291 --- /dev/null +++ b/tests/test_redact.py @@ -0,0 +1,106 @@ +# Copyright (C) 2026 Danilo M. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Tests for abusectl.redact: query-value redaction and redirect recovery.""" + +import unittest + +from abusectl import redact + + +class TestRedactUrl(unittest.TestCase): + def test_query_values_are_redacted_and_names_kept(self): + # The names fingerprint the kit; the values identify the recipient. + self.assertEqual( + redact.url("http://login.example.invalid/verify?id=abc&src=mail"), + "http://login.example.invalid/verify?id=REDACTED&src=REDACTED", + ) + + def test_a_url_with_no_query_is_unchanged(self): + self.assertEqual( + redact.url("http://login.example.invalid/verify"), + "http://login.example.invalid/verify", + ) + + def test_scheme_host_and_path_survive(self): + self.assertEqual( + redact.url("https://a.example.invalid/one/two/three?x=1"), + "https://a.example.invalid/one/two/three?x=REDACTED", + ) + + def test_a_valueless_parameter_keeps_its_shape(self): + self.assertEqual( + redact.url("http://a.example.invalid/p?flag"), + "http://a.example.invalid/p?flag=REDACTED", + ) + + def test_repeated_parameter_names_are_all_redacted(self): + self.assertEqual( + redact.url("http://a.example.invalid/p?t=1&t=2"), + "http://a.example.invalid/p?t=REDACTED&t=REDACTED", + ) + + +class TestSuspectPathSegments(unittest.TestCase): + def test_a_base64_looking_segment_is_flagged(self): + # Flagged for review, NOT redacted: a path may be meaningful. + found = redact.suspect_path_segments( + "http://a.example.invalid/verify/dGVzdEBleGFtcGxlLm9yZw/" + ) + self.assertEqual(found, ["dGVzdEBleGFtcGxlLm9yZw"]) + + def test_a_long_hex_segment_is_flagged(self): + found = redact.suspect_path_segments( + "http://a.example.invalid/c/5f4dcc3b5aa765d61d8327deb882cf99" + ) + self.assertEqual(found, ["5f4dcc3b5aa765d61d8327deb882cf99"]) + + def test_ordinary_path_words_are_not_flagged(self): + found = redact.suspect_path_segments( + "http://a.example.invalid/account/verify/now" + ) + self.assertEqual(found, []) + + def test_a_short_segment_is_not_flagged(self): + # "news" is base64-shaped and four characters. Too short to carry an + # address, and flagging it would train the user to ignore the flag. + found = redact.suspect_path_segments("http://a.example.invalid/news") + self.assertEqual(found, []) + + +class TestUrlValuedParameters(unittest.TestCase): + def test_a_redirect_target_is_recovered(self): + found = redact.url_valued_parameters( + "http://t.example.invalid/c?url=http%3A%2F%2Fevil.example.invalid%2Fp" + ) + self.assertEqual(found, ["http://evil.example.invalid/p"]) + + def test_a_tracking_token_is_not_mistaken_for_one(self): + found = redact.url_valued_parameters( + "http://t.example.invalid/c?u=dGVzdEBleGFtcGxlLm9yZw" + ) + self.assertEqual(found, []) + + def test_the_original_is_still_fully_redacted(self): + # Recovery does not loosen the rule: the redirector itself keeps every + # value blanked, including the one the target was recovered from. + raw = "http://t.example.invalid/c?url=http%3A%2F%2Fe.example.invalid%2Fp&u=tok" + self.assertEqual( + redact.url(raw), + "http://t.example.invalid/c?url=REDACTED&u=REDACTED", + ) + + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3