Skip to content

feat(gpush): print what a failed CI run reported #372

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
twistedmelonman merged 1 commit into from
Sep 25, 2026
Merged

feat(gpush): print what a failed CI run reported #372

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions bash/gpush-wrapper.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,82 @@
# guard/bypass distinction the way gh does, so this mode is just a
# thin dispatch, not a separate implementation.

# Print what a failed CI run reported, not only that it failed
# (smartwatermelon/dev-env#113). Before this, gpush printed only
# "CI failed (Standards Check: failure)" and the reader had to open the run in
# a browser to learn which linter failed and on which line.
#
# For each job in the run that did not succeed, print the job's check-run
# annotations: failure and warning levels only, oldest first, each prefixed
# with its title (the linter name, for the standards check). The GitHub API
# returns annotations newest first, so they are sorted by start_line to read
# in the order the job emitted them. Notices are dropped (runner-image
# migration notes, "no YAML files"), and so is the runner's own "Process
# completed with exit code N" line, which says nothing the conclusion does not.
#
# If the annotations come back empty or unreadable, fall back to the
# "##[error]" lines of the job's failed-step log. The raw log is never dumped:
# for a standards-check failure it is hundreds of lines of checkout noise.
#
# Output is bounded to max_lines per job, and always ends with the job URL.
# Every call here is advisory: a failure to read the detail never changes
# gpush's exit path, it only means less is printed.
#
# Token note: measured 2026-09-25 against twistedmelonman/claude-config run
# 35944355297 with a fine-grained PAT. Both the check-run annotations endpoint
# and `gh run view --job <id> --log-failed` returned data. The job's
# databaseId from `gh run view --json jobs` is the check-run id.
_gpush_print_failure_detail() {
local run_id="$1"
local run_name="$2"
local max_lines=40
local RED='\033[0;31m'
local NC='\033[0m'

local jobs
jobs=$(gh run view "${run_id}" --json jobs \
-q '.jobs[] | select(.conclusion != null and .conclusion != "success" and .conclusion != "skipped" and .conclusion != "neutral") | (.databaseId | tostring) + "\t" + .name + "\t" + .url' \
2>/dev/null) || jobs=""

if [[ -z "${jobs}" ]]; then
echo -e "${RED}[gpush]${NC} ${run_name}: could not list the failed jobs. See: gh run view ${run_id} --log-failed" >&2
return 0
fi

local job_id job_name job_url detail total
while IFS=$'\t' read -r job_id job_name job_url; do
[[ -z "${job_id}" ]] && continue
# `(... // "")` guards: a null field inside test() or + would raise, and jq
# drops a record that raises without failing the whole run.
detail=$(gh api "repos/{owner}/{repo}/check-runs/${job_id}/annotations" --jq '
sort_by(.start_line) | .[]
| select(.annotation_level == "failure" or .annotation_level == "warning")
| select((.message // "") | test("^Process completed with exit code [0-9]+\\.?$") | not)
| (if (.title // "") != "" then (.title + ": ") else "" end) + (.message // "")' \
</dev/null 2>/dev/null) || detail=""

if [[ -z "${detail}" ]]; then
detail=$(gh run view "${run_id}" --job "${job_id}" --log-failed </dev/null 2>/dev/null \
| grep -F '##[error]' \
| sed -e 's/^.*##\[error\]//' \
| grep -vE '^Process completed with exit code [0-9]+\.?$') || detail=""
fi

echo -e "${RED}[gpush]${NC} ${run_name} / ${job_name} reported:" >&2
if [[ -n "${detail}" ]]; then
total=$(printf '%s\n' "${detail}" | wc -l | tr -d ' ')
printf '%s\n' "${detail}" | head -n "${max_lines}" | sed 's/^/ /' >&2
if [[ "${total}" -gt "${max_lines}" ]]; then
echo " ... $((total - max_lines)) more line(s) not shown" >&2
fi
else
echo " (no annotations or error lines could be read)" >&2
fi
echo " Details: ${job_url}" >&2
done <<<"${jobs}"
return 0
}

gpush() {
# Validate arguments
case "${1:-}" in
Expand Down Expand Up @@ -125,6 +201,7 @@ gpush() {
local ci_passed=false
local ci_failed=false
local fail_detail=""
local -a failed_runs=()
local run_id run_name pattern
while IFS=$'\t' read -r run_id run_name; do
[[ -z "${run_id}" ]] && continue
Expand Down Expand Up @@ -162,10 +239,17 @@ gpush() {
else
ci_failed=true
fail_detail+="${run_name}: ${run_conclusion}; "
failed_runs+=("${run_id}"$'\t'"${run_name}")
fi
done <<<"${all_runs}"

if [[ "${ci_failed}" == true ]]; then
local failed_run
# The +-form keeps an empty array safe under set -u on older bash: a
# non-ignorable skipped run sets ci_failed without adding a failed run.
for failed_run in ${failed_runs[@]+"${failed_runs[@]}"}; do
_gpush_print_failure_detail "${failed_run%%$'\t'*}" "${failed_run#*$'\t'}"
done
fail_detail="${fail_detail%; }"
echo -e "${RED}[gpush]${NC} CI failed (${fail_detail}). Fix and re-run gpush." >&2
return 1
Expand Down
239 changes: 239 additions & 0 deletions bash/tests/test-gpush-wrapper-failure-detail.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
#!/usr/bin/env bash
# shellcheck shell=bash
# Standalone verification for bash/gpush-wrapper.sh's failed-CI detail
# (_gpush_print_failure_detail). Run directly:
# bash bash/tests/test-gpush-wrapper-failure-detail.sh
#
# smartwatermelon/dev-env#113: when a CI run fails, gpush printed only
# "CI failed (Standards Check: failure)". It must also print what the run
# reported: the failed job's check-run annotations (titled, oldest first,
# notices and the runner's exit-code line dropped), bounded in length, with a
# fallback to the "##[error]" lines of the failed-step log when there are no
# annotations, and the job URL.
#
# gpush runs end to end against stub `git` and `gh` binaries on PATH, so no
# real remote or GitHub state is touched. The stub gh evaluates each -q/--jq
# expression with the real jq, so the wrapper's own filters are what is tested.
# The fixture JSON shapes (annotation fields, newest-first order, the job
# databaseId doubling as the check-run id) were measured 2026-09-25 against
# twistedmelonman/claude-config run 35944355297 and its job 107458893019.
set -uo pipefail

unset CDPATH
# functions.sh defines `gh` and `git` shell functions that would win over the
# PATH stubs if a child bash sourced it through BASH_ENV.
unset BASH_ENV

REPO_ROOT="$(CDPATH='' cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
_tests_dir="$(CDPATH='' cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/git-env-isolation.sh
source "${_tests_dir}/lib/git-env-isolation.sh"
isolate_git_env

WRAPPER="${GPUSH_WRAPPER_UNDER_TEST:-${REPO_ROOT}/bash/gpush-wrapper.sh}"

if ! command -v jq >/dev/null 2>&1; then
echo "FAIL: jq is required by the stub gh and is not installed"
exit 1
fi

WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/gpush-failure-detail.XXXXXX")" || exit 1
trap 'rm -rf "${WORKDIR}"' EXIT
STUBS="${WORKDIR}/stubs"
mkdir -p "${STUBS}"

cat >"${STUBS}/git" <<'STUB'
#!/usr/bin/env bash
case "$1" in
symbolic-ref) echo "feature-x" ;;
rev-parse) echo "abcdef1234567890abcdef1234567890abcdef12" ;;
*) exit 0 ;;
esac
STUB

# sleep is stubbed so the wrapper's re-poll delay does not slow the test.
cat >"${STUBS}/sleep" <<'STUB'
#!/usr/bin/env bash
exit 0
STUB

# The stub gh reads its fixtures from STUB_DIR:
# conclusion the run conclusion
# jobs.json `gh run view --json jobs` payload
# annotations.json check-run annotations payload
# log.txt `gh run view --log-failed` output
# and appends each invocation to STUB_DIR/calls.
cat >"${STUBS}/gh" <<'STUB'
#!/usr/bin/env bash
echo "$*" >>"${STUB_DIR}/calls"
q=""
args=("$@")
for ((i = 0; i < ${#args[@]}; i++)); do
case "${args[i]}" in
-q | --jq) q="${args[i + 1]}" ;;
esac
done
filter() {
if [[ -n "${q}" ]]; then jq -r "${q}"; else cat; fi
}
case "$1 $2" in
"pr create") echo "https://github.com/o/r/pull/7" ;;
"run list") printf '101\tStandards Check\n' ;;
"run watch") exit 0 ;;
"run view")
if [[ " $* " == *" --log-failed "* ]]; then
cat "${STUB_DIR}/log.txt"
elif [[ " $* " == *" conclusion "* ]]; then
cat "${STUB_DIR}/conclusion"
else
filter <"${STUB_DIR}/jobs.json"
fi
;;
"api repos/{owner}/{repo}/check-runs/555/annotations") filter <"${STUB_DIR}/annotations.json" ;;
*)
echo "stub gh: unexpected call: $*" >&2
exit 1
;;
esac
STUB
chmod +x "${STUBS}"/*

# Sources the wrapper under test ($1) and runs gpush with the rest.
cat >"${WORKDIR}/run-gpush.sh" <<'RUNNER'
# shellcheck source=/dev/null
source "$1"
shift
gpush "$@"
RUNNER

JOBS_JSON='{"jobs":[
{"databaseId":554,"name":"setup","conclusion":"success","url":"https://github.com/o/r/actions/runs/101/job/554"},
{"databaseId":555,"name":"standards-check / run-standards-check","conclusion":"failure","url":"https://github.com/o/r/actions/runs/101/job/555"}
]}'

fail=0
_pass() { echo "PASS: $1"; }
_fail() {
echo "FAIL: $1"
fail=1
}

# new_case <name>: make a fresh fixture dir with a failed run and the jobs list.
new_case() {
CASE_DIR="${WORKDIR}/$1"
mkdir -p "${CASE_DIR}"
echo "failure" >"${CASE_DIR}/conclusion"
printf '%s\n' "${JOBS_JSON}" >"${CASE_DIR}/jobs.json"
echo '[]' >"${CASE_DIR}/annotations.json"
: >"${CASE_DIR}/log.txt"
}

# run_gpush [args...]: run gpush in a clean child bash with the stubs first on
# PATH. Sets OUT (stdout+stderr, colour codes stripped) and RC.
run_gpush() {
OUT="$(STUB_DIR="${CASE_DIR}" PATH="${STUBS}:${PATH}" /usr/bin/env bash \
"${WORKDIR}/run-gpush.sh" "${WRAPPER}" "$@" 2>&1)"
RC=$?
OUT="$(printf '%s' "${OUT}" | sed $'s/\033\\[[0-9;]*m//g')"
}

contains() { [[ "${OUT}" == *"$1"* ]]; }

# Case 1: annotations in the post-#176 standards-check format. The API returns
# them newest first; the linter's own titled, multi-line annotation must be
# printed, before the summary line, without notices or the exit-code line.
new_case annotations
cat >"${CASE_DIR}/annotations.json" <<'JSON'
[
{"path":".github","start_line":75,"annotation_level":"failure","title":"","message":"Process completed with exit code 1."},
{"path":".github","start_line":74,"annotation_level":"failure","title":"","message":"standards-check failed: shellcheck"},
{"path":".github","start_line":69,"annotation_level":"notice","title":"","message":"no Markdown files"},
{"path":".github","start_line":58,"annotation_level":"failure","title":"shellcheck","message":"shellcheck found problems\nIn scripts/foo.sh line 3:\necho $1\n ^-- SC2086 (info): Double quote to prevent globbing and word splitting."},
{"path":".github","start_line":1,"annotation_level":"notice","title":"","message":"The ubuntu-latest label will migrate to Ubuntu 26"}
]
JSON
run_gpush
if [[ ${RC} -ne 0 ]]; then _pass "failed CI still exits non-zero"; else _fail "failed CI exited 0"; fi
if contains "CI failed (Standards Check: failure)"; then
_pass "summary line is kept"
else
_fail "summary line missing"
fi
if contains "shellcheck: shellcheck found problems" && contains "SC2086"; then
_pass "linter annotation title and finding are printed"
else
_fail "linter annotation title or finding missing"
fi
if contains "standards-check failed: shellcheck"; then
_pass "final error annotation is printed"
else
_fail "final error annotation missing"
fi
if contains "no Markdown files" || contains "ubuntu-latest" || contains "Process completed with exit code"; then
_fail "notice or exit-code annotations were printed"
else
_pass "notices and the exit-code line are dropped"
fi
line_linter="$(printf '%s\n' "${OUT}" | grep -n 'SC2086' | head -1 | cut -d: -f1)"
line_summary="$(printf '%s\n' "${OUT}" | grep -n 'standards-check failed: shellcheck' | head -1 | cut -d: -f1)"
if [[ -n "${line_linter}" && -n "${line_summary}" && "${line_linter}" -lt "${line_summary}" ]]; then
_pass "annotations are printed oldest first"
else
_fail "annotations are not in emitted order (linter line ${line_linter:-none}, summary line ${line_summary:-none})"
fi
if contains "Details: https://github.com/o/r/actions/runs/101/job/555"; then
_pass "failed job URL is printed"
else
_fail "failed job URL missing"
fi
if grep -q 'check-runs/554' "${CASE_DIR}/calls"; then
_fail "a successful job was queried for annotations"
else
_pass "only the failed job is queried"
fi
if [[ "${fail}" -ne 0 ]]; then
printf '%s\n' "${OUT}" | sed 's/^/ /'
fi

# Case 2: no annotations -> the "##[error]" lines of the failed-step log.
new_case log-fallback
cat >"${CASE_DIR}/log.txt" <<'LOG'
standards-check / run-standards-check UNKNOWN STEP 2026-09-24T01:46:08.1Z [command]/usr/bin/git checkout noise
standards-check / run-standards-check UNKNOWN STEP 2026-09-24T01:46:08.3Z ##[error]shellcheck found problems
standards-check / run-standards-check UNKNOWN STEP 2026-09-24T01:46:08.4Z ##[error]Process completed with exit code 1.
LOG
run_gpush
if contains "shellcheck found problems" && ! contains "checkout noise" && ! contains "Process completed with exit code"; then
_pass "empty annotations fall back to the log's error lines only"
else
_fail "log fallback printed the wrong lines"
printf '%s\n' "${OUT}" | sed 's/^/ /'
fi

# Case 3: output is bounded. A 100-line annotation prints at most 40 lines.
new_case bounded
jq -n '[{"path":".github","start_line":5,"annotation_level":"failure","title":"shellcheck",
"message":([range(1;101)] | map("finding-\(.)") | join("\n"))}]' >"${CASE_DIR}/annotations.json"
run_gpush
if contains "finding-39" && ! contains "finding-41" && contains "more line(s) not shown"; then
_pass "long annotations are cut to the line limit with a count"
else
_fail "long annotations were not bounded"
fi

# Case 4: a passing run prints no failure detail and queries no annotations.
new_case success
echo "success" >"${CASE_DIR}/conclusion"
run_gpush --no-merge
if [[ ${RC} -eq 0 ]] && ! contains "reported:" && ! grep -q 'annotations' "${CASE_DIR}/calls"; then
_pass "a passing run prints no failure detail"
else
_fail "a passing run printed failure detail or failed (rc=${RC})"
printf '%s\n' "${OUT}" | sed 's/^/ /'
fi

if [[ "${fail}" -ne 0 ]]; then
echo "FAILED"
exit 1
fi
echo "All gpush-wrapper failure-detail tests passed"
Loading