Merge PR #1874: redact private keys, cookies and platform tokens

agent/src/tools/redaction.py

@@ -97,7 +97,15 @@ _SENSITIVE_ARG_KEYS = {
     "token",
 } | _PII_EXACT_KEYS
 
-_SENSITIVE_ARG_MARKERS = ("api_key", "authorization", "password", "secret", "token")
+_SENSITIVE_ARG_MARKERS = (
+    "api_key",
+    "authorization",
+    "cookie",
+    "password",
+    "private_key",
+    "secret",
+    "token",
+)
 
 #: Keys that stay sensitive in the ARGUMENTS / audit sink but are released in
 #: the tool-RESULT sink.
@@ -482,6 +490,34 @@ _TEXT_URL_USERINFO_PATTERN = re.compile(
     r"@"
 )
 
+#: PEM / PGP private-key blocks (``-----BEGIN RSA PRIVATE KEY----- ... -----END``).
+#: The base64 body has no key label and no recognised prefix, so nothing else
+#: here sees it. The whole block is replaced; a block whose END marker was
+#: truncated (clipped tool output) is redacted to the end of the text, which
+#: fails closed. The body is a lazy run to a fixed terminator or ``\Z``, so a
+#: single scan consumes it and matching stays linear. Public keys and
+#: certificates (``BEGIN PUBLIC KEY`` / ``BEGIN CERTIFICATE``) do not match.
+_TEXT_PRIVATE_KEY_PATTERN = re.compile(
+    r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----"
+    r"[\s\S]*?"
+    r"(?:-----END [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----|\Z)"
+)
+
+#: ``Cookie:`` / ``Set-Cookie:`` header lines. A session cookie is a bearer
+#: credential, but its value is a ``name=value; name=value`` list that neither
+#: the key pattern (``cookie`` is not a credential key name) nor the token
+#: shapes can see, so ``curl -H "Cookie: sid=..."`` and a dumped response header
+#: both leaked it. The value runs to the end of the line or the closing quote of
+#: a quoted shell argument. An already-redacted value is skipped, which keeps the
+#: substitution idempotent.
+_TEXT_COOKIE_HEADER_PATTERN = re.compile(
+    r"(?P<head>(?<![\w-])(?:set-)?cookie\s*:)"
+    r"(?P<sp>[ \t]*)(?![ \t])"
+    r"(?!\[redacted\])"
+    r"[^\r\n\"']+",
+    re.IGNORECASE,
+)
+
 
 def _sub_credential_pair(match: re.Match[str]) -> str:
     """Rebuild one ``key=value`` match with the value redacted.
@@ -514,8 +550,14 @@ _TEXT_TOKEN_PATTERN = re.compile(
     r"sk-[A-Za-z0-9_-]{20,}"                 # OpenAI / Anthropic style
     r"|gh[pousr]_[A-Za-z0-9]{30,}"           # GitHub PAT family
     r"|xox[baprs]-[A-Za-z0-9-]{10,}"         # Slack
-    r"|AKIA[0-9A-Z]{16}"                     # AWS access key id
+    r"|(?:AKIA|ASIA)[0-9A-Z]{16}"            # AWS access key id (long/short-lived)
+    r"|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}"  # Stripe secret / restricted key
+    r"|AIza[A-Za-z0-9_-]{35}"                # Google API key
     r"|pypi-[A-Za-z0-9_-]{16,}"              # PyPI upload token
+    r"|github_pat_[A-Za-z0-9_]{22,}"         # GitHub fine-grained PAT
+    r"|glpat-[A-Za-z0-9_-]{20,}"             # GitLab PAT
+    r"|npm_[A-Za-z0-9]{36}"                  # npm access token
+    r"|hf_[A-Za-z0-9]{30,}"                  # Hugging Face token
     r"|eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{5,}"  # JWT
     r")(?![A-Za-z0-9_-])"
 )
@@ -546,6 +588,8 @@ def redact_text(text: object) -> str:
     s = text if isinstance(text, str) else str(text)
     if not s:
         return s
+    s = _TEXT_PRIVATE_KEY_PATTERN.sub(_REDACTED, s)
+    s = _TEXT_COOKIE_HEADER_PATTERN.sub(r"\g<head>\g<sp>" + _REDACTED, s)
     s = _TEXT_BEARER_PATTERN.sub(r"\g<scheme> " + _REDACTED, s)
     s = _TEXT_AUTH_SCHEME_PATTERN.sub(r"\g<head>\g<scheme> " + _REDACTED, s)
     s = _TEXT_URL_USERINFO_PATTERN.sub(_REDACTED + "@", s)

agent/tests/test_redaction_cloud_api_keys.py

@@ -0,0 +1,28 @@
+"""Bare Stripe, Google and AWS temporary keys are scrubbed from free-form text."""
+
+from __future__ import annotations
+
+import pytest
+
+from src.tools.redaction import redact_text
+
+_KEYS = [
+    "sk_live_" + "a1B2c3D4e5F6g7H8i9J0k1L2",
+    "sk_test_" + "a1B2c3D4e5F6g7H8i9J0k1L2",
+    "rk_live_" + "a1B2c3D4e5F6g7H8i9J0k1L2",
+    "AIza" + "Sy" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q",
+    "ASIA" + "IOSFODNN7EXAMPLE",
+]
+
+
+@pytest.mark.parametrize("key", _KEYS)
+def test_bare_key_is_redacted(key: str) -> None:
+    out = redact_text(f"request failed with {key} in the header")
+    assert key not in out
+    assert out == "request failed with [redacted] in the header"
+    assert redact_text(out) == out
+
+
+def test_lookalikes_are_left_alone() -> None:
+    text = "pk_live_publishableKey123456 sk_live_short AIzaShort ASIA123 task_test_" + "x" * 20
+    assert redact_text(text) == text

agent/tests/test_redaction_cookie_headers.py

@@ -0,0 +1,32 @@
+"""Cookie and Set-Cookie header values are scrubbed from free-form text."""
+
+from __future__ import annotations
+
+import pytest
+
+from src.tools.redaction import redact_text
+
+
+@pytest.mark.parametrize("header", ["Cookie", "cookie", "Set-Cookie", "SET-COOKIE"])
+def test_cookie_header_value_is_redacted(header: str) -> None:
+    out = redact_text(f"{header}: sid=abc123; theme=dark; Path=/; HttpOnly")
+    assert out == f"{header}: [redacted]"
+    assert redact_text(out) == out
+
+
+def test_only_the_header_line_is_redacted() -> None:
+    out = redact_text("HTTP/1.1 200 OK\nSet-Cookie: sid=abc123; Secure\nContent-Type: text/html")
+    assert out == "HTTP/1.1 200 OK\nSet-Cookie: [redacted]\nContent-Type: text/html"
+
+
+@pytest.mark.parametrize("quote", ['"', "'"])
+def test_quoted_curl_header_keeps_the_rest_of_the_command(quote: str) -> None:
+    cmd = f"curl -H {quote}Cookie: sid=abc123; csrf=xyz{quote} https://example.com/api"
+    out = redact_text(cmd)
+    assert "abc123" not in out and "xyz" not in out
+    assert out == f"curl -H {quote}Cookie: [redacted]{quote} https://example.com/api"
+
+
+def test_prose_and_lookalikes_are_left_alone() -> None:
+    text = "We use a cookie banner. my-cookie: x is not a header; cookies: 3"
+    assert redact_text(text) == text

agent/tests/test_redaction_cookie_private_key_fields.py

@@ -0,0 +1,37 @@
+"""``cookie`` and ``private_key`` payload fields are treated as credentials."""
+
+from __future__ import annotations
+
+import pytest
+
+from src.tools.redaction import ARGUMENTS_SINK, RESULT_SINK, is_sensitive_arg, redact_payload
+
+
+@pytest.mark.parametrize(
+    "name",
+    ["cookie", "Cookie", "cookies", "set-cookie", "Set-Cookie", "session_cookie",
+     "sessionCookie", "private_key", "privateKey", "private-key", "Private Key",
+     "ssh_private_key", "rsa_private_key_pem"],
+)
+@pytest.mark.parametrize("sink", [ARGUMENTS_SINK, RESULT_SINK])
+def test_cookie_and_private_key_field_names_are_sensitive(name: str, sink: str) -> None:
+    assert is_sensitive_arg(name, sink=sink)
+
+
+@pytest.mark.parametrize("name", ["public_key", "key_id", "cooking_time", "credential_status", "account_ref"])
+def test_unrelated_field_names_stay_readable(name: str) -> None:
+    assert not is_sensitive_arg(name)
+
+
+def test_payload_walk_redacts_cookie_and_private_key_values() -> None:
+    payload = {
+        "headers_seen": {"Set-Cookie": "sid=abc123", "Cookie": "sid=abc123"},
+        "wallet": {"private_key": "0xdeadbeef", "public_key": "0xpub"},
+        "items": [{"cookies": {"sid": "abc123"}}],
+        "symbol": "AAPL",
+    }
+    out = redact_payload(payload)
+    assert out["headers_seen"] == {"Set-Cookie": "[redacted]", "Cookie": "[redacted]"}
+    assert out["wallet"] == {"private_key": "[redacted]", "public_key": "0xpub"}
+    assert out["items"] == [{"cookies": "[redacted]"}]
+    assert out["symbol"] == "AAPL"

agent/tests/test_redaction_platform_tokens.py

@@ -0,0 +1,33 @@
+"""Bare developer-platform tokens are scrubbed from free-form text."""
+
+from __future__ import annotations
+
+import pytest
+
+from src.tools.redaction import redact_text
+
+_TOKENS = [
+    "github_pat_11ABCDEFG0" + "a1B2c3D4e5F6g7H8i9J0_" + "k" * 40,
+    "glpat-" + "a1B2c3D4e5F6g7H8i9J0",
+    "npm_" + "a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8",
+    "hf_" + "a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6",
+]
+
+
+@pytest.mark.parametrize("token", _TOKENS)
+def test_bare_platform_token_is_redacted(token: str) -> None:
+    out = redact_text(f"cloning with {token} failed")
+    assert token not in out
+    assert out == "cloning with [redacted] failed"
+    assert redact_text(out) == out
+
+
+@pytest.mark.parametrize("token", _TOKENS)
+def test_platform_token_inside_url_userinfo_is_redacted(token: str) -> None:
+    out = redact_text(f"fatal: https://{token}@host/org/repo.git")
+    assert token not in out
+
+
+def test_lookalike_identifiers_are_left_alone() -> None:
+    text = "npm_config_registry hf_home github_pat_ glpat-short npm_short hf_x"
+    assert redact_text(text) == text

agent/tests/test_redaction_private_key_blocks.py

@@ -0,0 +1,50 @@
+"""PEM and PGP private-key blocks are scrubbed from free-form text."""
+
+from __future__ import annotations
+
+import pytest
+
+from src.tools.redaction import redact_text
+
+_BODY = "MIIEowIBAAKCAQEAx1Zq9f+/abc123XYZ=\nQmFzZTY0Qm9keUxpbmUyLy8vPT0=\n"
+
+
+@pytest.mark.parametrize(
+    "label",
+    ["RSA PRIVATE KEY", "PRIVATE KEY", "EC PRIVATE KEY", "OPENSSH PRIVATE KEY",
+     "ENCRYPTED PRIVATE KEY", "PGP PRIVATE KEY BLOCK"],
+)
+def test_private_key_block_is_redacted(label: str) -> None:
+    block = f"-----BEGIN {label}-----\n{_BODY}-----END {label}-----"
+    out = redact_text(f"key file:\n{block}\nend of file")
+    assert "MIIEow" not in out and "QmFzZTY0" not in out
+    assert out == "key file:\n[redacted]\nend of file"
+    assert redact_text(out) == out
+
+
+def test_encrypted_pem_headers_are_part_of_the_block() -> None:
+    block = (
+        "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\n"
+        "DEK-Info: AES-128-CBC,0123456789ABCDEF\n\n" + _BODY
+        + "-----END RSA PRIVATE KEY-----"
+    )
+    assert redact_text(block) == "[redacted]"
+
+
+def test_truncated_block_is_redacted_to_the_end() -> None:
+    out = redact_text(f"head\n-----BEGIN PRIVATE KEY-----\n{_BODY}")
+    assert out == "head\n[redacted]"
+
+
+def test_two_blocks_are_redacted_independently() -> None:
+    one = f"-----BEGIN PRIVATE KEY-----\n{_BODY}-----END PRIVATE KEY-----"
+    out = redact_text(f"{one} and {one}")
+    assert out == "[redacted] and [redacted]"
+
+
+def test_public_material_is_left_alone() -> None:
+    text = (
+        "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQ==\n-----END PUBLIC KEY-----\n"
+        "-----BEGIN CERTIFICATE-----\nMIIDdzCCAl+gAw==\n-----END CERTIFICATE-----"
+    )
+    assert redact_text(text) == text