diff --git a/raven/security/redact.py b/raven/security/redact.py index f2563aac09..7c61846215 100644 --- a/raven/security/redact.py +++ b/raven/security/redact.py @@ -59,13 +59,14 @@ ), # Command-line flags that take a credential. ("cli-flag", re.compile(r"(--(?:token|password|api-key|secret|with-token)[= ])([^\s\"']{6,})", re.IGNORECASE)), - # A URL with an inline password: scheme://user:secret@host. - ("url-userinfo", re.compile(r"(://[^\s/:@]+:)([^\s/@]{3,})(?=@)")), + # A URL with an inline password: scheme://user:secret@host. The user may be + # empty, as in redis://:secret@host. + ("url-userinfo", re.compile(r"(://[^\s/:@]*:)([^\s/@]{3,})(?=@)")), # Vendor-shaped keys, recognisable without a label because their prefix is # the label. The one class worth matching bare: a leaked key is a leaked key # whether or not somebody wrote "api_key" next to it. ("openai", re.compile(r"()(sk-(?:proj-|ant-|or-)?[A-Za-z0-9_-]{16,})")), - ("github", re.compile(r"()(gh[pousr]_[A-Za-z0-9]{16,})")), + ("github", re.compile(r"()(gh[pousr]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{22,})")), ("gitlab", re.compile(r"()(glpat-[A-Za-z0-9_-]{16,})")), ("slack", re.compile(r"()(xox[abposr]-[A-Za-z0-9-]{10,})")), ("google", re.compile(r"()(AIza[0-9A-Za-z_-]{30,})")), @@ -101,12 +102,13 @@ _TRUNCATED_SECRETS: tuple[tuple[re.Pattern[str], re.Pattern[str]], ...] = ( ( re.compile( - r"()(? str: def _apply_patterns(text: str, counts: dict[str, int]) -> str: for name, pattern in PATTERNS: - text, hits = pattern.subn(f"[REDACTED:pattern.{name}]", text) + placeholder = f"[REDACTED:pattern.{name}]" + + def _sub(match: re.Match[str], placeholder: str = placeholder) -> str: + # A pattern captures a leading group only when it keeps that part, + # as the URL rule keeps everything before the password. + return (match.group(1) if match.groups() else "") + placeholder + + text, hits = pattern.subn(_sub, text) if hits: counts[name] = counts.get(name, 0) + hits return text diff --git a/tests/test_acp_redact.py b/tests/test_acp_redact.py index 4b96bc0e27..d66759eddd 100644 --- a/tests/test_acp_redact.py +++ b/tests/test_acp_redact.py @@ -25,6 +25,14 @@ import raven.security.redact as redact_module from raven.security.redact import MAX_SCAN_CHARS, REPLACEMENT, pattern_names, redact, redact_value +# A real fine-grained PAT is ``github_pat_`` plus 82 characters: 22, an +# underscore, then 59. Fabricated, and split here so the shape the pattern has +# to match is visible. The 29-character fixture this replaced pinned only the +# floor above 29; a value class that lost the underscore still passed it. +GITHUB_PAT_FIRST = "11ABCDEFG0abcdefghijkl" +GITHUB_PAT_REST = "B0b3Dd5Ff7Hh9Jj1Kk3Ll5Mm7Nn9Pp1Qq3Rr5Ss7Tt9Uu1Vv3Ww5Xx7Yy9Z" +GITHUB_PAT = f"github_pat_{GITHUB_PAT_FIRST}_{GITHUB_PAT_REST}" + class TestWhatItCatches: @pytest.mark.parametrize( @@ -33,10 +41,12 @@ class TestWhatItCatches: ('curl -H "Authorization: Bearer sk-ant-api03-AAAABBBBCCCCDDDD" https://x', "sk-ant-api03"), ("export OPENAI_API_KEY=sk-proj-abcdefghijklmnop", "sk-proj-abcdefghijklmnop"), ("gh auth login --with-token ghp_ABCDEFGHIJKLMNOPQRSTUV", "ghp_ABCDEFGHIJKLMNOPQRSTUV"), + (f"git clone https://{GITHUB_PAT}@github.com/a/b", GITHUB_PAT), ("glab auth login --token glpat-abcdefghijklmnopqrst", "glpat-abcdefghijklmnopqrst"), ('api_key: "AIzaSyDdI0hCZtE6vySjMm-WEfRq3CPzqKqqsHI"', "AIzaSy"), ("aws sts get-caller-identity # AKIAIOSFODNN7EXAMPLE", "AKIAIOSFODNN7EXAMPLE"), ("psql postgres://admin:hunter2secret@db:5432/app", "hunter2secret"), + ("redis-cli -u redis://:hunter2secret@cache:6379/0 ping", "hunter2secret"), ("SLACK_BOT=xoxb-1234567890-abcdefghij", "xoxb-1234567890"), ("Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NX0.dBjftJeZ4CVPmB92K27uhbUJU1p1r", "eyJhbGciOi"), ('{"password": "correct-horse-battery"}', "correct-horse-battery"), @@ -53,6 +63,21 @@ def test_the_secret_is_gone_and_the_shape_remains(self, text, secret): prefix = text[: text.index(secret)] assert prefix in result, "redacting the label as well leaves a row nobody can act on" + def test_a_fine_grained_pat_leaves_nothing_of_its_value(self): + """A real-shaped token, because a 13-character prefix proves only that + the prefix went. The value class still has to cross the underscore at + position 22 and the 59 characters behind it, and each part is asserted + on its own so a partial redaction cannot pass as a whole one.""" + text = f"git clone https://{GITHUB_PAT}@github.com/a/b" + + result = redact(text) + + assert len(GITHUB_PAT) == 93 + assert GITHUB_PAT not in result + assert GITHUB_PAT[len("github_pat_") :] not in result + assert GITHUB_PAT[33:] not in result + assert result == "git clone https://[redacted]@github.com/a/b" + def test_a_private_key_body_goes_but_the_armour_stays(self): # Assembled rather than written out, and not for taste: a literal armour # line makes this file look like a leaked key to the repo's own @@ -128,6 +153,7 @@ class TestWhatItLeavesAlone: [ "set -e && npm test", "git clone https://github.com/user/repo", + "redis-cli -u redis://cache:6379/0 ping", "token: 12345", "the password is wrong", "authorization: read", @@ -156,12 +182,28 @@ def test_empty_text_is_returned_as_it_came(self): class TestTheLimits: @pytest.mark.parametrize( "text", - ['"Authorization": "Bearer opaqu', "password=abcde", "--token abcde", "ghp_ABCD", "eyJabcdefghijk"], + [ + '"Authorization": "Bearer opaqu', + "password=abcde", + "--token abcde", + "ghp_ABCD", + "github_pat_11AB", + "redis://:abc", + "eyJabcdefghijk", + ], ) def test_a_known_cut_removes_short_credential_tails(self, text): assert REPLACEMENT in redact(text, truncated=True) assert redact(text) == text + def test_the_fine_grained_pat_floor_is_twenty_two_characters(self): + """``github_pat_`` plus 21 characters is not a token yet; 22 is. Pinned + as a pair so a smaller quantifier and a larger one both show up.""" + short = "github_pat_" + "A" * 21 + + assert redact(short) == short + assert redact("github_pat_" + "A" * 22) == REPLACEMENT + def test_a_known_cut_keeps_ordinary_unlabelled_text(self): text = "ordinary diagnostic " + "x" * 1000 assert redact(text, truncated=True) == text @@ -238,6 +280,7 @@ def whole_text(text: str) -> str: # One long value per pattern: a run longer than the label room, which a # value class narrower than the pattern's own would stop short of. tails = ["eyJ", "x.sk-", "pwd=", '"password": "', "Authorization: Bearer ", "--token ", "https://user:"] + tails += ["github_pat_", "redis://:"] for tail in tails: for lead in ("", "x " * 300, "pwd=" * 100 + ","): text = lead + tail + "aZ9" * 140 @@ -247,6 +290,7 @@ def whole_text(text: str) -> str: rng = random.Random(978) heads = ["password", "PWD", "api_key", "auth", "token", "Authorization", "Proxy-Authorization"] heads += ["--token", "--with-token", "https://user", "sk-", "ghp_", "eyJ", "AKIA", "xoxb-"] + heads += ["github_pat_", "redis://"] noise = ["sk-", ".sk-", "pwd=", "--token=", "://", ":", "@", "/", "!", ",", "'", '"', " ", "\n", "x", "Z9"] for _ in range(3000): text = "".join(rng.choice(noise) for _ in range(rng.randint(0, 12))) diff --git a/tests/test_trajectory_redact.py b/tests/test_trajectory_redact.py index f8c5983098..5c1f47b33e 100644 --- a/tests/test_trajectory_redact.py +++ b/tests/test_trajectory_redact.py @@ -140,6 +140,27 @@ def test_pattern_fallback(tmp_path): assert f"[REDACTED:pattern.{name}]" in text +def test_url_passwords_are_redacted_in_the_pattern_layer(tmp_path): + """A bundle copies the model's tool calls verbatim, so a connection string + is a credential channel here too. The user part may be empty, as redis + accepts, and stays as it was; a URL with no password is untouched.""" + lines = [ + "redis-cli -u redis://:hunter2secret@cache:6379/0 ping", + "psql postgres://admin:hunter2secret@db:5432/app", + "redis-cli -u redis://cache:6379/0 ping", + ] + bundle = _make_bundle(tmp_path, spans_text="\n".join(lines), artifact_text="clean") + + report = tredact.redact_bundle(bundle, tmp_path / "red", secrets=[]) + + text = (report.redacted_dir / "spans.jsonl").read_text(encoding="utf-8") + assert "hunter2secret" not in text + assert "redis://:[REDACTED:pattern.url-password]@cache:6379/0" in text + assert "postgres://admin:[REDACTED:pattern.url-password]@db:5432/app" in text + assert "redis://cache:6379/0" in text + assert report.patterns["url-password"] == 2 + + # ── layer 3: residual scan ────────────────────────────────────────────