From f5d6cf1ef4c94005ec3c5f48f7e63bc5d7d2725c Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:25:45 +0800 Subject: [PATCH] fix: report closed failed-stage cause signals Co-Authored-By: Codex --- scripts/diagnose_cached_stage_failure.py | 87 ++++++++++++- tests/test_cached_stage_failure_diagnostic.py | 122 ++++++++++++++++++ 2 files changed, 203 insertions(+), 6 deletions(-) diff --git a/scripts/diagnose_cached_stage_failure.py b/scripts/diagnose_cached_stage_failure.py index 5d8622a..d4c430d 100644 --- a/scripts/diagnose_cached_stage_failure.py +++ b/scripts/diagnose_cached_stage_failure.py @@ -23,6 +23,10 @@ "port", "startup", "missing_secret", + "update_mask", + "environment", + "network", + "quota", ) _REASON_PATTERNS = { "act_as": re.compile(r"iam\.serviceaccounts\.actas|actas|service account user", re.I), @@ -39,6 +43,41 @@ "port": re.compile(r"port[^\n]*(listen|start|bind|set)|listen on the port", re.I), "startup": re.compile(r"startup probe|container failed to start|startup[^\n]*failed", re.I), "permission": re.compile(r"permission denied|permission_denied|not authorized|forbidden|\b403\b", re.I), + "update_mask": re.compile(r"update[ _-]?mask|field mask", re.I), + "environment": re.compile(r"environment variable|environment configuration|env var", re.I), + "network": re.compile(r"\bvpc\b|network|subnet|connector", re.I), + "quota": re.compile(r"quota|resource[_ ]exhausted|limit exceeded", re.I), +} +_KNOWN_RPC_STATUS_CODES = frozenset(range(17)) +_AUDIT_MESSAGE_TERMS = ( + "container", + "name", + "image", + "traffic", + "tag", + "revision", + "env", + "secret", + "update_mask", + "port", + "vpc", + "quota", + "immutable", +) +_AUDIT_MESSAGE_TERM_PATTERNS = { + "container": re.compile(r"\bcontainers?\b", re.I), + "name": re.compile(r"\bname\b", re.I), + "image": re.compile(r"\bimages?\b", re.I), + "traffic": re.compile(r"\btraffic\b", re.I), + "tag": re.compile(r"\btags?\b", re.I), + "revision": re.compile(r"\brevisions?\b", re.I), + "env": re.compile(r"\benv(?:ironment)?\b", re.I), + "secret": re.compile(r"\bsecret(?:keyref)?\b", re.I), + "update_mask": _REASON_PATTERNS["update_mask"], + "port": re.compile(r"\bports?\b", re.I), + "vpc": re.compile(r"\bvpc\b", re.I), + "quota": _REASON_PATTERNS["quota"], + "immutable": re.compile(r"\bimmutab(?:le|ility)\b", re.I), } _COMBINED_TAG_NAME_LIMIT = re.compile( r"traffic[^\n]*tag[^\n]*too long|combined traffic tag and service name cannot exceed 46", @@ -116,6 +155,24 @@ def _condition_error_texts(resource: Any) -> list[str]: return texts +def _condition_error_status_codes(resource: Any) -> list[int]: + status = resource.get("status") if isinstance(resource, dict) else None + conditions = status.get("conditions") if isinstance(status, dict) else None + if not isinstance(conditions, list): + return [] + codes = set() + for condition in conditions: + if not isinstance(condition, dict): + continue + state = condition.get("state", condition.get("status")) + if state not in ("CONDITION_FAILED", "False", False): + continue + code = condition.get("code") + if isinstance(code, int) and not isinstance(code, bool) and code in _KNOWN_RPC_STATUS_CODES: + codes.add(code) + return sorted(codes) + + def _audit_identity_matches( entry: dict[str, Any], *, project: str, region: str, service: str ) -> bool: @@ -149,20 +206,20 @@ def _audit_error_texts( service: str, window_start: str, window_end: str, -) -> tuple[int | None, int | None, list[str]]: +) -> tuple[int | None, int | None, list[str], list[int] | None]: if not isinstance(entries, list): - return None, None, [] + return None, None, [], None try: start_time = datetime.fromisoformat(window_start.replace("Z", "+00:00")) end_time = datetime.fromisoformat(window_end.replace("Z", "+00:00")) except (TypeError, ValueError): - return None, None, [] + return None, None, [], None if ( start_time.tzinfo is None or end_time.tzinfo is None or end_time < start_time ): - return None, None, [] + return None, None, [], None failed = [] matching_count = 0 for entry in entries: @@ -199,7 +256,11 @@ def _audit_error_texts( texts = [message for _, message in failed if message] if any(code == 7 for code, _ in failed): texts.append("permission_denied") - return matching_count, len(failed), texts + status_codes = sorted({ + code for code, _ in failed + if isinstance(code, int) and not isinstance(code, bool) and code in _KNOWN_RPC_STATUS_CODES + }) + return matching_count, len(failed), texts, status_codes def _control_summary(policy: Any, jobs: Any) -> dict[str, Any]: @@ -234,6 +295,13 @@ def _control_summary(policy: Any, jobs: Any) -> dict[str, Any]: } +def _audit_message_terms(texts: list[str], *, available: bool) -> dict[str, bool | None]: + if not available: + return {term: None for term in _AUDIT_MESSAGE_TERMS} + joined = "\n".join(texts) + return {term: bool(_AUDIT_MESSAGE_TERM_PATTERNS[term].search(joined)) for term in _AUDIT_MESSAGE_TERMS} + + def summarize( *, service: Any, @@ -277,7 +345,7 @@ def summarize( if latest_ready is True else ["unknown"] ) - audit_entries_count, audit_error_count, audit_texts = _audit_error_texts( + audit_entries_count, audit_error_count, audit_texts, audit_status_codes = _audit_error_texts( audit_entries, project=expected_project, region=expected_region, @@ -301,6 +369,7 @@ def summarize( failure_texts = [] combined_tag_name_error = _has_combined_tag_name_limit(failure_texts) target_matches = bool(isinstance(metadata, dict) and metadata.get("name") == expected_service) + revision_status_codes = _condition_error_status_codes(latest) tag_budget_ok = bool( target_matches and isinstance(metadata, dict) @@ -323,11 +392,17 @@ def summarize( and all(isinstance(item, dict) for item in revisions), "latest_created_revision_ready": latest_ready, "latest_revision_error_categories": revision_categories, + "latest_revision_error_status_codes": revision_status_codes, + "audit_error_message_terms": _audit_message_terms( + audit_texts, + available=audit_status == "ok" and audit_entries_count is not None, + ), **_control_summary(policy, jobs), "audit_query_status": audit_status, "audit_entry_count": audit_entries_count if audit_status == "ok" else None, "audit_error_count": audit_error_count if audit_status == "ok" else None, "audit_error_categories": audit_categories, + "audit_error_status_codes": audit_status_codes if audit_status == "ok" else None, "failure_source": failure_source, "failure_categories": failure_categories, } diff --git a/tests/test_cached_stage_failure_diagnostic.py b/tests/test_cached_stage_failure_diagnostic.py index 837c6bb..4a6de0a 100644 --- a/tests/test_cached_stage_failure_diagnostic.py +++ b/tests/test_cached_stage_failure_diagnostic.py @@ -99,6 +99,22 @@ def test_summary_exposes_only_closed_statuses_counts_and_categories(): "revision_list_readable": True, "latest_created_revision_ready": False, "latest_revision_error_categories": ["image"], + "latest_revision_error_status_codes": [], + "audit_error_message_terms": { + "container": False, + "name": False, + "image": True, + "traffic": False, + "tag": False, + "revision": False, + "env": False, + "secret": True, + "update_mask": False, + "port": False, + "vpc": False, + "quota": False, + "immutable": False, + }, "iam_policy_readable": True, "iam_binding_count": 1, "scheduler_readable": True, @@ -108,6 +124,7 @@ def test_summary_exposes_only_closed_statuses_counts_and_categories(): "audit_entry_count": 1, "audit_error_count": 1, "audit_error_categories": ["image"], + "audit_error_status_codes": [13], "failure_source": "audit", "failure_categories": ["image"], } @@ -126,6 +143,10 @@ def test_summary_exposes_only_closed_statuses_counts_and_categories(): ("container failed to listen on the port", ["port"]), ("startup probe failed", ["startup"]), ("secret version was not found", ["missing_secret"]), + ("updateMask contains a private field", ["update_mask"]), + ("environment variable is invalid", ["environment"]), + ("VPC connector could not be resolved", ["network"]), + ("RESOURCE_EXHAUSTED: quota exceeded", ["quota"]), ( "traffic[].tag: traffic tag [TAG] and service name [SERVICE] together are too long. " "Combined traffic tag and service name cannot exceed 46 characters.", @@ -185,6 +206,107 @@ def test_combined_traffic_tag_length_is_closed_subcategory_and_budget_stays_priv assert PRIVATE_SERVICE not in json.dumps(summary) +def test_status_codes_and_message_terms_are_numeric_and_closed(): + message = ( + "permission denied: updateMask for VPC connector private-vpc " + "failed on immutable revision private-revision-placeholder" + ) + service, revisions, policy, jobs, audit = _evidence(message) + revisions[0]["status"]["conditions"][0]["code"] = 14 + audit[0]["protoPayload"]["status"]["code"] = 7 + summary = diagnostic.summarize( + service=service, + revisions=revisions, + expected_service=PRIVATE_SERVICE, + expected_project=PRIVATE_PROJECT, + expected_region=PRIVATE_REGION, + audit_window_start=WINDOW_START, + audit_window_end=WINDOW_END, + policy=policy, + jobs=jobs, + audit_entries=audit, + audit_status="ok", + ) + assert summary["latest_revision_error_status_codes"] == [14] + assert summary["audit_error_status_codes"] == [7] + assert summary["failure_categories"] == ["permission", "update_mask", "network"] + assert summary["audit_error_message_terms"] == { + "container": False, + "name": False, + "image": False, + "traffic": False, + "tag": False, + "revision": True, + "env": False, + "secret": False, + "update_mask": True, + "port": False, + "vpc": True, + "quota": False, + "immutable": True, + } + serialized = json.dumps(summary) + for private_value in ( + PRIVATE_SERVICE, + PRIVATE_NAME, + PRIVATE_MESSAGE, + "private-vpc", + "private-revision-placeholder", + "updateMask for VPC connector private-vpc", + ): + assert private_value not in serialized + + +@pytest.mark.parametrize( + ("message", "present"), + [ + ("multiple containers are unsupported on private-revision-placeholder", {"container", "revision"}), + ("revision private-revision-placeholder is immutable", {"revision", "immutable"}), + ("traffic target references tag private-tag on revision private-revision-placeholder", {"traffic", "tag", "revision"}), + ("cannot update container name or image private-image-reference", {"container", "name", "image"}), + ], +) +def test_audit_message_terms_are_closed_and_private_values_are_not_returned(message, present): + service, revisions, policy, jobs, audit = _evidence(message) + summary = diagnostic.summarize( + service=service, + revisions=revisions, + expected_service=PRIVATE_SERVICE, + expected_project=PRIVATE_PROJECT, + expected_region=PRIVATE_REGION, + audit_window_start=WINDOW_START, + audit_window_end=WINDOW_END, + policy=policy, + jobs=jobs, + audit_entries=audit, + audit_status="ok", + ) + terms = summary["audit_error_message_terms"] + assert {term for term, value in terms.items() if value} == present + assert all(type(value) is bool for value in terms.values()) + for private_value in (PRIVATE_SERVICE, PRIVATE_NAME, PRIVATE_MESSAGE, "private-image-reference", "private-tag"): + assert private_value not in json.dumps(summary) + + +def test_audit_message_terms_ignore_events_outside_target(): + service, revisions, policy, jobs, audit = _evidence() + summary = diagnostic.summarize( + service=service, + revisions=revisions, + expected_service="different-service-placeholder", + expected_project=PRIVATE_PROJECT, + expected_region=PRIVATE_REGION, + audit_window_start=WINDOW_START, + audit_window_end=WINDOW_END, + policy=policy, + jobs=jobs, + audit_entries=audit, + audit_status="ok", + ) + assert summary["target_matches"] is False + assert not any(summary["audit_error_message_terms"].values()) + + def test_tag_budget_false_is_reported_as_boolean_without_service_name(): service, revisions, policy, jobs, audit = _evidence() long_name = "s" * 45