diff --git a/.sampo/changesets/mask-authorization-credentials.md b/.sampo/changesets/mask-authorization-credentials.md new file mode 100644 index 000000000..92000c627 --- /dev/null +++ b/.sampo/changesets/mask-authorization-credentials.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +Mask `Bearer` and `Basic` credentials in exception code variables, including values held apart from their header name, such as in header lists and ASGI scopes. Also mask signed URLs that carry a `sig` query parameter. diff --git a/posthog/exception_utils.py b/posthog/exception_utils.py index 546c977a5..c0bb345d1 100644 --- a/posthog/exception_utils.py +++ b/posthog/exception_utils.py @@ -5,6 +5,8 @@ # 💖open source (under MIT License) # We want to keep payloads as similar to Sentry as possible for easy interoperability +import base64 +import binascii import dataclasses import functools import json @@ -67,6 +69,9 @@ r"(?i)conn_str", r"(?i)connstr", r"(?i)dsn", + # The signature parameter of a signed URL, such as an Azure SAS URL. It is matched + # as a query parameter, because the bare word `sig` occurs in common names. + r"(?i)[?&]sig=", ] DEFAULT_CODE_VARIABLES_IGNORE_PATTERNS = [r"^__.*"] @@ -124,6 +129,63 @@ def _redact_url_credentials(value): ) +# Matches the credential of an HTTP `Authorization` value, e.g. `Bearer ` or +# `Basic `. A header pair list or an ASGI scope holds this value apart +# from its header name, so the name patterns never see it. The separator also accepts a +# colon or an opening quote, as in `Bearer: `, but it must not be empty, so that +# names such as `basicConfig` stay untouched. +_AUTH_HEADER_CREDENTIALS_RE = re.compile( + r"\b(bearer|basic)((?:\s*:\s*|\s+)['\"]?)([A-Za-z0-9._~+/-]+=*)", re.IGNORECASE +) + +# Shorter values are prose, such as "the bearer of", and so is a lowercase word of up to +# 15 letters, such as "bearer transportation". A random lowercase token is longer than +# that. A `Basic` credential of any length is still redacted when it decodes to +# `user:password`, e.g. `YTpi` for `a:b`. +_AUTH_HEADER_CREDENTIAL_MIN_LENGTH = 8 +_AUTH_HEADER_PROSE_WORD_MAX_LENGTH = 15 + + +def _is_basic_credential(credential): + try: + decoded = base64.b64decode(credential, validate=True).decode("utf-8") + except (binascii.Error, ValueError): + return False + return ":" in decoded + + +def _is_prose_word(credential): + return ( + len(credential) <= _AUTH_HEADER_PROSE_WORD_MAX_LENGTH + and credential.isalpha() + and credential.islower() + ) + + +def _redact_auth_header_match(match): + scheme, credential = match.group(1), match.group(3) + is_basic_pair = scheme.lower() == "basic" and _is_basic_credential(credential) + if not is_basic_pair and ( + len(credential) < _AUTH_HEADER_CREDENTIAL_MIN_LENGTH + or _is_prose_word(credential) + ): + return match.group(0) + return scheme + match.group(2) + CODE_VARIABLES_REDACTED_VALUE + + +def _redact_auth_header_credentials(value): + return _AUTH_HEADER_CREDENTIALS_RE.sub(_redact_auth_header_match, value) + + +def _redact_embedded_credentials(value, config): + """Scrub credentials embedded in otherwise safe text: URL credentials when that toggle + is on, then `Authorization` values always. URLs go first, because the `Authorization` + pass can consume a URL scheme, as in `Bearer postgresql://user:pass@host`.""" + if config.mask_url_credentials: + value = _redact_url_credentials(value) + return _redact_auth_header_credentials(value) + + DEFAULT_TOTAL_VARIABLES_SIZE_LIMIT = 10 * 1024 @@ -1270,16 +1332,14 @@ def _looks_like_secret(value): def _mask_string(value, config): """Apply the string masking policy: over-length cap, name/value patterns, - entropy-based secret detection, then embedded URL credentials.""" + entropy-based secret detection, then embedded `Authorization` and URL credentials.""" if len(value) > _MAX_VALUE_LENGTH_FOR_PATTERN_MATCH: return CODE_VARIABLES_TOO_LONG_VALUE if _matcher_matches(value, config.mask): return CODE_VARIABLES_REDACTED_VALUE if config.detect_secrets and _looks_like_secret(value): return CODE_VARIABLES_REDACTED_VALUE - if config.mask_url_credentials: - return _redact_url_credentials(value) - return value + return _redact_embedded_credentials(value, config) def _safe_type_name(value): @@ -1306,9 +1366,7 @@ def _safe_repr(value, config): # A __repr__ that is itself a bare secret would otherwise bypass detection. if config.detect_secrets and _looks_like_secret(rendered): return CODE_VARIABLES_REDACTED_VALUE - if config.mask_url_credentials: - return _redact_url_credentials(rendered) - return rendered + return _redact_embedded_credentials(rendered, config) def _extract_object_attrs(value): @@ -1485,8 +1543,8 @@ def _mask_mapping(items, config, seen, depth): out_key = _redacted_key(result) elif not key_is_json_safe and _key_parts_fail_masking(key, config, seen, depth): out_key = _redacted_key(result) - elif config.mask_url_credentials and isinstance(out_key, str): - out_key = _redact_url_credentials(out_key) + elif isinstance(out_key, str): + out_key = _redact_embedded_credentials(out_key, config) if out_key in result: # Two keys can end up with the same text, for example URLs that differ only in # their credentials. A placeholder keeps the later entry from overwriting. diff --git a/posthog/test/test_code_variables.py b/posthog/test/test_code_variables.py index 3d8abe0fb..6263b4b8d 100644 --- a/posthog/test/test_code_variables.py +++ b/posthog/test/test_code_variables.py @@ -238,6 +238,70 @@ def test_strings_without_credentials_are_left_untouched(self, value): assert _redact_url_credentials(value) == value +# --- 2b. Authorization credential scrubbing ------------------------------------------ + +_BEARER_TOKEN = _key("tok_", "Zx81Qm7Lp2Vb9Nc4") +_BASIC_CREDENTIAL = _key("c3ZjOmZha2Ut", "cGFzcy0xMjM=") # svc:fake-pass-123 + + +class TestAuthorizationCredentialScrubbing: + """A `Bearer` or `Basic` credential is removed even when no header name sits next + to it, so name patterns can't catch it. The scheme stays for context.""" + + @pytest.mark.parametrize( + "value, credential", + [ + # header pair list: the name is redacted, the value used to survive + ([("authorization", "Bearer " + _BEARER_TOKEN)], _BEARER_TOKEN), + # ASGI scope: raw byte headers + ( + { + "headers": [ + (b"authorization", b"Basic " + _BASIC_CREDENTIAL.encode()) + ] + }, + _BASIC_CREDENTIAL, + ), + # a local with a neutral name + ("Basic " + _BASIC_CREDENTIAL, _BASIC_CREDENTIAL), + # a short Basic credential: `YTpi` is `a:b` + ("Basic YTpi", "YTpi"), + # a credential of lowercase letters only + ("Bearer " + _key("zqwklmno", "pxyzrstu"), _key("zqwklmno", "pxyzrstu")), + # a DSN after the scheme keeps its own credential redacted + ( + "Bearer postgresql://alice:" + _key("sunfl", "ower99") + "@db/app", + _key("sunfl", "ower99"), + ), + # a colon or a quote between the scheme and the credential + ("Bearer: " + _BEARER_TOKEN, _BEARER_TOKEN), + ("Bearer '" + _BEARER_TOKEN + "'", _BEARER_TOKEN), + ], + ) + def test_credential_is_removed(self, value, credential): + result = json.dumps(mask(value)) + assert credential not in result + assert REDACTED in result + + def test_scheme_is_kept(self): + assert mask("Bearer " + _BEARER_TOKEN) == "Bearer " + REDACTED + + @pytest.mark.parametrize( + "value", + [ + "basicConfig(level=10)", + "basic auth", + "basic: configuration", + "Bearer token", + "bearer transportation", + "the bearer of bad news", + ], + ) + def test_prose_is_left_untouched(self, value): + # name patterns off, so only the Authorization check can change the text + assert mask(value, patterns=[]) == value + + # --- 3. scalar masking --------------------------------------------------------------- @@ -288,6 +352,17 @@ def test_url_scrubbing_can_be_turned_off(self): result = mask("postgresql://user:p4ss@host/db", patterns=[], mask_urls=False) assert result == "postgresql://user:p4ss@host/db" + def test_signed_url_is_redacted(self): + # an Azure SAS URL carries its signature in the `sig` query parameter + url = "https://acct.blob.core.windows.net/c/f?sv=2022-11-02&sp=r&sig=" + _key( + "q2VxT8", "fKz1aB3dE%3D" + ) + assert mask(url) == REDACTED + + @pytest.mark.parametrize("value", ["design", "signal_handler", "/signup?step=2"]) + def test_sig_inside_other_words_is_left_untouched(self, value): + assert mask(value) == value + # --- 5. collection masking ----------------------------------------------------------- diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index 524d22c4c..51ee00d9d 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -825,7 +825,7 @@ attribute posthog.exception_utils.CODE_VARIABLES_REDACTED_VALUE = '$$_posthog_re attribute posthog.exception_utils.CODE_VARIABLES_TOO_LONG_VALUE = '$$_posthog_value_too_long_$$' attribute posthog.exception_utils.DEFAULT_CODE_VARIABLES_DETECT_SECRETS = True attribute posthog.exception_utils.DEFAULT_CODE_VARIABLES_IGNORE_PATTERNS = ['^__.*'] -attribute posthog.exception_utils.DEFAULT_CODE_VARIABLES_MASK_PATTERNS = ['(?i)password', '(?i)secret', '(?i)passwd', '(?i)pwd', '(?i)api_key', '(?i)apikey', '(?i)auth', '(?i)credentials', '(?i)privatekey', '(?i)private_key', '(?i)token', '(?i)aws_access_key_id', '(?i)_pass', '(?i)sk_', '(?i)jwt', '(?i)connection_string', '(?i)connectionstring', '(?i)conn_str', '(?i)connstr', '(?i)dsn'] +attribute posthog.exception_utils.DEFAULT_CODE_VARIABLES_MASK_PATTERNS = ['(?i)password', '(?i)secret', '(?i)passwd', '(?i)pwd', '(?i)api_key', '(?i)apikey', '(?i)auth', '(?i)credentials', '(?i)privatekey', '(?i)private_key', '(?i)token', '(?i)aws_access_key_id', '(?i)_pass', '(?i)sk_', '(?i)jwt', '(?i)connection_string', '(?i)connectionstring', '(?i)conn_str', '(?i)connstr', '(?i)dsn', '(?i)[?&]sig='] attribute posthog.exception_utils.DEFAULT_CODE_VARIABLES_MASK_URL_CREDENTIALS = True attribute posthog.exception_utils.DEFAULT_MAX_VALUE_LENGTH = 1024 attribute posthog.exception_utils.DEFAULT_TOTAL_VARIABLES_SIZE_LIMIT = 10 * 1024 @@ -1408,6 +1408,7 @@ function posthog.load_feature_flags() function posthog.mcp.asgi.autowire_stateless_mint(server: Any) -> None function posthog.mcp.asgi.get_mcp_session(request_or_scope: Any) -> Optional[SessionTokenPayload] function posthog.mcp.feedback.send_feedback_result() -> Dict[str, Any] +function posthog.mcp.get_tool_input_properties(input_value: Any, input_schema: Any = None, options: Optional[ToolInputOptions] = None) function posthog.mcp.instrument(server: Any, posthog_client: Optional[Client] = None, options: Optional[MCPAnalyticsOptions] = None) -> McpAnalytics function posthog.mcp.logger.set_logger(logger: Optional[LoggerFn]) -> None function posthog.mcp.request_headers.get_request_headers(extra: Any) -> Optional[RequestHeaderBag]