From 79568458643b419ff07bd3a8160a20a7d40bf6c4 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Sun, 1 Nov 2026 14:34:06 -0800 Subject: [PATCH] devops(bots): publish the agent's GitHub writes as typed calls The triage and revision workflows switch to the publish-calls action of microsoft/playwright. The agent leaves what must happen on GitHub as typed calls in calls.json, and one job checks and makes them with the app token. A fix in microsoft/playwright is a pull-request call for that repo: the publish job checks it out in playwright/ for it. It replaces publish-patch and triage-comment. Depends on: https://github.com/microsoft/playwright/pull/43838 --- .claude/skills/playwright-triage/SKILL.md | 24 ++- .github/workflows/revision.yml | 25 ++-- .github/workflows/triage.yml | 172 ++++++++-------------- 3 files changed, 97 insertions(+), 124 deletions(-) diff --git a/.claude/skills/playwright-triage/SKILL.md b/.claude/skills/playwright-triage/SKILL.md index 510d87d..7a9e9c7 100644 --- a/.claude/skills/playwright-triage/SKILL.md +++ b/.claude/skills/playwright-triage/SKILL.md @@ -51,17 +51,31 @@ upstream commit that an alpha was built from. When the fix is in `$PLAYWRIGHT_DIR`, follow the Fix section of the guide there, in place of `$REPO_DIR`: create the branch `fix-` from `main` in `$PLAYWRIGHT_DIR`, put the regression test in its `tests/mcp/cli-*.spec.ts`, build and lint as its `AGENTS.md` says, and commit in its -convention, for example `fix(cli): ...`. Then leave the proposal in `$OUTPUT_DIR/upstream-fix/` in -place of `$OUTPUT_DIR/fix/`, with the same files: +convention, for example `fix(cli): ...`. Then leave the proposal in +`$OUTPUT_DIR/upstream-fix/` in place of `$OUTPUT_DIR/fix/`: - `fix.patch`: the output of `git diff --binary main HEAD` in `$PLAYWRIGHT_DIR`. - `pr-body.md`: as the guide says. The `Fixes` line has the full URL of the issue, `Fixes https://github.com/microsoft/playwright-cli/issues/`, so that the PR closes the issue in this repo. -- `proposal.json`: as the guide says. Write it last. -The publish job opens the draft PR in `microsoft/playwright`. The revision workflow of that repo -addresses the review. Leave one proposal: `fix/` or `upstream-fix/`, not both. Do not change +Add the `pull-request` call for `microsoft/playwright` last, as the guide says: + +```json +{ + "call": "pull-request", + "repo": "microsoft/playwright", + "branch": "triage/playwright-cli-/-", + "title": "fix(cli): description", + "patch": "upstream-fix/fix.patch", + "bodyFile": "upstream-fix/pr-body.md", + "commitBody": "Fixes: https://github.com/microsoft/playwright-cli/issues/" +} +``` + +The publish job checks out `microsoft/playwright` at the commit of `$PLAYWRIGHT_DIR` and opens the +draft PR there. The revision workflow of that repo addresses the review. Make one `pull-request` +call: in this repo or in `microsoft/playwright`, not both. Do not change `package.json` here to pick up the fix. It reaches `@playwright/cli` with the next roll. **Check the linked PRs in both repos.** A PR in `microsoft/playwright` links this issue with its diff --git a/.github/workflows/revision.yml b/.github/workflows/revision.yml index 3a2bb34..1042041 100644 --- a/.github/workflows/revision.yml +++ b/.github/workflows/revision.yml @@ -1,6 +1,8 @@ # Copilot addresses the maintainers' review comments on a draft PR that the triage workflow opened -# in this repo, without write credentials, on the PR rebased on main. A fresh job replaces the PR's -# branch with the rebased commits and the new one using the app, so CI runs on it. The PRs that the +# in this repo, without write credentials, on the PR rebased on main. The agent leaves the calls to +# make on GitHub in calls.json. A fresh job checks them and makes them using the app: it replaces +# the PR's branch with the rebased commits and the new one, so CI runs on it, and replies in the +# review threads. The PRs that the # triage workflow opens in microsoft/playwright are revised there. The steps that are the same for # every Playwright repo are actions in microsoft/playwright, and the shared guide they carry says how # to revise; the prompt below has what is specific to this repo. @@ -25,6 +27,7 @@ jobs: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest permissions: + issues: read pull-requests: write env: PLATFORM: ${{ fromJSON('{"needs-revision":"linux","needs-revision-mac":"mac","needs-revision-win":"windows"}')[github.event.label.name] }} @@ -101,14 +104,14 @@ jobs: Pull request: https://github.com/microsoft/playwright-cli/pull/${{ github.event.pull_request.number }} output-dir: ${{ env.OUTPUT_DIR }} - result-file: commit/commit.json + result-file: calls.json artifact-name: revision exec: ${{ env.CI_ENV_EXEC }} push: name: Push the revision needs: [prepare, revise] - # commit.json is written last, so a finished revision is pushed even when the job failed later. + # calls.json is written when the revision is done, so it is pushed even if the job failed later. if: ${{ !cancelled() && needs.revise.outputs.has_revision == 'true' }} runs-on: ubuntu-latest steps: @@ -117,6 +120,8 @@ jobs: with: client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }} private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ needs.prepare.outputs.head }} @@ -125,13 +130,9 @@ jobs: with: name: revision path: ${{ runner.temp }}/revision - # Never execute code or scripts from the proposed changes in the app job. - - name: Validate and push - uses: microsoft/playwright/.github/actions/publish-patch@main + # Never execute code or scripts from the artifact in the app job. The calls are data. + - name: Check and make the calls + uses: microsoft/playwright/.github/actions/publish-calls@main with: token: ${{ steps.app-token.outputs.token }} - dir: ${{ runner.temp }}/revision/commit - patch: revision.patch - metadata: commit.json - branch: ${{ needs.prepare.outputs.branch }} - onto: ${{ needs.prepare.outputs.base }} + dir: ${{ runner.temp }}/revision diff --git a/.github/workflows/triage.yml b/.github/workflows/triage.yml index c4201ae..e2c294a 100644 --- a/.github/workflows/triage.yml +++ b/.github/workflows/triage.yml @@ -1,9 +1,10 @@ # Copilot triages and fixes an issue, or works on a task given by hand, without write credentials. -# Fresh jobs post the run link on the issue, open the completed fix as a draft PR using the app, so -# CI runs on it, and comment on the issue. The source of the CLI is in microsoft/playwright, -# so most fixes open there, and that repo's revision workflow addresses the review. The steps that -# are the same for every Playwright repo are actions in microsoft/playwright. The skill in -# .claude/skills/playwright-triage/ has what is specific to this repo. +# The agent leaves the calls to make on GitHub in calls.json. A fresh job posts the run link on the +# issue first. After the agent, a fresh job checks the calls and makes them using the app: it opens +# the fix as a draft PR, so CI runs on it, and comments on the issue. The source of the CLI is in +# microsoft/playwright, so most fixes open there, and that repo's revision workflow addresses the +# review. The steps that are the same for every Playwright repo are actions in microsoft/playwright. +# The skill in .claude/skills/playwright-triage/ has what is specific to this repo. name: Issue triage on: @@ -88,7 +89,6 @@ jobs: upstream_base: ${{ steps.upstream-base.outputs.sha }} setup: ${{ steps.setup.outcome }} agent: ${{ steps.agent.outputs.outcome }} - has_fix: ${{ steps.agent.outputs.has-result }} has_upstream_fix: ${{ steps.upstream-fix.outputs.has-result }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -136,14 +136,15 @@ jobs: with: prompt: ${{ steps.prompt.outputs.prompt }} output-dir: ${{ env.OUTPUT_DIR }} - result-file: fix/proposal.json + result-file: calls.json artifact-name: triage github-tools: true exec: ${{ env.CI_ENV_EXEC }} + # The publish job checks out microsoft/playwright only for a pull request there. - name: Check the fix in microsoft/playwright id: upstream-fix run: | - if [[ -f "$OUTPUT_DIR/upstream-fix/proposal.json" ]]; then + if node -e 'const calls = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); process.exit(calls.some(c => c.call === "pull-request" && String(c.repo).toLowerCase() === "microsoft/playwright") ? 0 : 1)' "$OUTPUT_DIR/calls.json" 2> /dev/null; then echo "has-result=true" >> "$GITHUB_OUTPUT" echo "Fix proposed in microsoft/playwright" >> "$GITHUB_STEP_SUMMARY" else @@ -151,126 +152,83 @@ jobs: fi publish: - name: Open draft pull request - needs: triage - # proposal.json is written last, so a finished proposal is published even when the job failed later. - if: ${{ !cancelled() && needs.triage.outputs.has_fix == 'true' }} + name: Publish the results + needs: [start, triage] + # Runs after every triage run, also a failed one: without a comment from the agent, the comment + # says why the bot could not finish. + if: ${{ !cancelled() && needs.triage.result != 'skipped' }} runs-on: ubuntu-latest - outputs: - pr: ${{ steps.publish.outputs.pull-request }} steps: - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 id: app-token with: client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }} private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} + owner: microsoft + repositories: playwright-cli,playwright + permission-contents: write + permission-issues: write + permission-pull-requests: write - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ needs.triage.outputs.base }} token: ${{ steps.app-token.outputs.token }} - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: triage - path: ${{ runner.temp }}/triage - # Never execute code or scripts from the proposed changes in the app job. - - name: Validate, push and open draft - id: publish - uses: microsoft/playwright/.github/actions/publish-patch@main - with: - token: ${{ steps.app-token.outputs.token }} - dir: ${{ runner.temp }}/triage/fix - patch: fix.patch - metadata: proposal.json - body-file: pr-body.md - branch: triage/${{ github.event.issue.number || inputs.issue || 'task' }}/${{ github.run_id }}-${{ github.run_attempt }} - commit-body: "${{ (github.event.issue.number || inputs.issue) && format('Fixes: https://github.com/{0}/issues/{1}', github.repository, github.event.issue.number || inputs.issue) || '' }}" - pull-request: true - - publish-upstream: - name: Open draft pull request in microsoft/playwright - needs: triage - # proposal.json is written last, so a finished proposal is published even when the job failed later. - if: ${{ !cancelled() && needs.triage.outputs.has_upstream_fix == 'true' }} - runs-on: ubuntu-latest - outputs: - pr: ${{ steps.pr.outputs.url }} - env: - BRANCH: triage/playwright-cli-${{ github.event.issue.number || inputs.issue || 'task' }}/${{ github.run_id }}-${{ github.run_attempt }} - steps: - - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 - id: app-token - with: - client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }} - private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} - owner: microsoft - repositories: playwright + # A pull request in microsoft/playwright goes to its checkout in playwright/, at the commit that + # the agent worked on. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: needs.triage.outputs.has_upstream_fix == 'true' with: repository: microsoft/playwright ref: ${{ needs.triage.outputs.upstream_base }} + path: playwright token: ${{ steps.app-token.outputs.token }} - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: triage - path: ${{ runner.temp }}/triage - # Never execute code or scripts from the proposed changes in the app job. publish-patch opens - # pull requests in the repo of the workflow only, so it only pushes, and the next step opens it. - - name: Validate and push - uses: microsoft/playwright/.github/actions/publish-patch@main - with: - token: ${{ steps.app-token.outputs.token }} - dir: ${{ runner.temp }}/triage/upstream-fix - patch: fix.patch - metadata: proposal.json - branch: ${{ env.BRANCH }} - commit-body: "${{ (github.event.issue.number || inputs.issue) && format('Fixes: https://github.com/{0}/issues/{1}', github.repository, github.event.issue.number || inputs.issue) || '' }}" - - name: Open draft - id: pr - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - DIR: ${{ runner.temp }}/triage/upstream-fix - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - with: - github-token: ${{ steps.app-token.outputs.token }} - script: | - const fs = require('fs'); - const path = require('path'); - const { BRANCH, DIR, RUN_URL } = process.env; - const { title } = JSON.parse(fs.readFileSync(path.join(DIR, 'proposal.json'), 'utf8')); - const description = fs.readFileSync(path.join(DIR, 'pr-body.md'), 'utf8').trim(); - const { data: pr } = await github.rest.pulls.create({ - owner: 'microsoft', repo: 'playwright', head: BRANCH, base: 'main', title, - body: `${description}\n\nPrepared by the [triage workflow of ${context.repo.owner}/${context.repo.repo}](${RUN_URL}).`, - draft: true, - }); - core.setOutput('url', pr.html_url); - await core.summary.addRaw(`Draft pull request: ${pr.html_url}\n`).write(); - - comment: - name: Comment on the issue - needs: [start, triage, publish, publish-upstream] - # Runs after every triage run that has an issue, including failed ones, and whatever publish did. - if: ${{ !cancelled() && needs.triage.result != 'skipped' && (github.event_name == 'issues' || inputs.issue != '') }} - runs-on: ubuntu-latest - permissions: - issues: write - steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 # A failed triage run may not have uploaded anything. continue-on-error: true with: name: triage path: ${{ runner.temp }}/triage - - name: Post the triage comment - uses: microsoft/playwright/.github/actions/triage-comment@main + # Without a comment from the agent on the issue, for example after a failed run, the issue gets + # one that says that the bot could not finish. + - name: Add a comment when the agent left none + if: github.event.issue.number || inputs.issue + env: + CALLS: ${{ runner.temp }}/triage/calls.json + ISSUE: ${{ github.event.issue.number || inputs.issue }} + SETUP: ${{ needs.triage.outputs.setup }} + AGENT: ${{ needs.triage.outputs.agent }} + FOOTER: Triaged on ${{ fromJSON('{"linux":"Linux","mac":"macOS","windows":"Windows","needs-triage":"Linux","needs-triage-mac":"macOS","needs-triage-win":"Windows"}')[inputs.platform || github.event.label.name] }} by the Playwright bot + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + mkdir -p "$(dirname "$CALLS")" + if jq -e --argjson issue "$ISSUE" 'any(.[]; .call == "comment" and .issue == $issue)' "$CALLS" > /dev/null 2>&1; then + exit 0 + fi + if [[ "$SETUP" == "failure" ]]; then + reason="setting up the runner failed" + elif [[ "$AGENT" == "failure" ]]; then + reason="the Copilot session failed or ran out of budget" + else + reason="the session ended without a comment" + fi + body=$(printf "Hi, I'm the Playwright bot. I couldn't finish triaging this issue: %s. The agent run has the details.\n\n%s - [agent run](%s)\n" "$reason" "$FOOTER" "$RUN_URL") + comment=$(jq -n --arg repo "$GITHUB_REPOSITORY" --argjson issue "$ISSUE" --arg body "$body" '{call: "comment", repo: $repo, issue: $issue, body: $body}') + # A calls.json that is not an array is replaced: its calls would be rejected anyway. + jq --argjson comment "$comment" '. + [$comment]' "$CALLS" > "$CALLS.tmp" 2> /dev/null \ + || jq -n --argjson comment "$comment" '[$comment]' > "$CALLS.tmp" + mv "$CALLS.tmp" "$CALLS" + # Never execute code or scripts from the artifact in the app job. The calls are data. + - name: Check and make the calls + id: publish + uses: microsoft/playwright/.github/actions/publish-calls@main with: - issue: ${{ github.event.issue.number || inputs.issue }} + token: ${{ steps.app-token.outputs.token }} dir: ${{ runner.temp }}/triage - footer: Triaged on ${{ fromJSON('{"linux":"Linux","mac":"macOS","windows":"Windows","needs-triage":"Linux","needs-triage-mac":"macOS","needs-triage-win":"Windows"}')[inputs.platform || github.event.label.name] }} by the Playwright bot - job-result: ${{ needs.triage.result }} - setup-outcome: ${{ needs.triage.outputs.setup }} - agent-outcome: ${{ needs.triage.outputs.agent }} - setup-failure-reason: installing the dependencies, building microsoft/playwright or downloading the browsers failed - pull-request: ${{ needs.publish.outputs.pr || needs.publish-upstream.outputs.pr }} - publish-result: ${{ (needs.publish.result == 'failure' || needs.publish-upstream.result == 'failure') && 'failure' || '' }} - start-comment: ${{ needs.start.outputs.comment }} + - name: Hide the comment with the run link + if: ${{ !cancelled() && steps.publish.outputs.comment == 'true' && needs.start.outputs.comment != '' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + ID: ${{ needs.start.outputs.comment }} + run: | + gh api graphql -f query='mutation($id: ID!) { minimizeComment(input: { subjectId: $id, classifier: OUTDATED }) { clientMutationId } }' -f id="$ID" \ + || echo "::warning::Could not hide the comment with the run link"