devops(bots): publish the agent's GitHub writes as typed calls

.claude/skills/playwright-triage/SKILL.md

@@ -51,17 +51,31 @@ upstream commit that an alpha was built from.
 When the fix is in `$PLAYWRIGHT_DIR`, follow the Fix section of the guide there, in place of
 `$REPO_DIR`: create the branch `fix-<n>` from `main` in `$PLAYWRIGHT_DIR`, put the regression test
 in its `tests/mcp/cli-*.spec.ts`, build and lint as its `AGENTS.md` says, and commit in its
-convention, for example `fix(cli): ...`. Then leave the proposal in `$OUTPUT_DIR/upstream-fix/` in
-place of `$OUTPUT_DIR/fix/`, with the same files:
+convention, for example `fix(cli): ...`. Then leave the proposal in
+`$OUTPUT_DIR/upstream-fix/` in place of `$OUTPUT_DIR/fix/`:
 
 - `fix.patch`: the output of `git diff --binary main HEAD` in `$PLAYWRIGHT_DIR`.
 - `pr-body.md`: as the guide says. The `Fixes` line has the full URL of the issue,
   `Fixes https://github.com/microsoft/playwright-cli/issues/<n>`, so that the PR closes the issue in
   this repo.
-- `proposal.json`: as the guide says. Write it last.
 
-The publish job opens the draft PR in `microsoft/playwright`. The revision workflow of that repo
-addresses the review. Leave one proposal: `fix/` or `upstream-fix/`, not both. Do not change
+Add the `pull-request` call for `microsoft/playwright` last, as the guide says:
+
+```json
+{
+  "call": "pull-request",
+  "repo": "microsoft/playwright",
+  "branch": "triage/playwright-cli-<issue>/<run id>-<run attempt>",
+  "title": "fix(cli): description",
+  "patch": "upstream-fix/fix.patch",
+  "bodyFile": "upstream-fix/pr-body.md",
+  "commitBody": "Fixes: https://github.com/microsoft/playwright-cli/issues/<n>"
+}
+```
+
+The publish job checks out `microsoft/playwright` at the commit of `$PLAYWRIGHT_DIR` and opens the
+draft PR there. The revision workflow of that repo addresses the review. Make one `pull-request`
+call: in this repo or in `microsoft/playwright`, not both. Do not change
 `package.json` here to pick up the fix. It reaches `@playwright/cli` with the next roll.
 
 **Check the linked PRs in both repos.** A PR in `microsoft/playwright` links this issue with its

.github/workflows/revision.yml

@@ -1,6 +1,8 @@
 # Copilot addresses the maintainers' review comments on a draft PR that the triage workflow opened
-# in this repo, without write credentials, on the PR rebased on main. A fresh job replaces the PR's
-# branch with the rebased commits and the new one using the app, so CI runs on it. The PRs that the
+# in this repo, without write credentials, on the PR rebased on main. The agent leaves the calls to
+# make on GitHub in calls.json. A fresh job checks them and makes them using the app: it replaces
+# the PR's branch with the rebased commits and the new one, so CI runs on it, and replies in the
+# review threads. The PRs that the
 # triage workflow opens in microsoft/playwright are revised there. The steps that are the same for
 # every Playwright repo are actions in microsoft/playwright, and the shared guide they carry says how
 # to revise; the prompt below has what is specific to this repo.
@@ -25,6 +27,7 @@ jobs:
       github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
     permissions:
+      issues: read
       pull-requests: write
     env:
       PLATFORM: ${{ fromJSON('{"needs-revision":"linux","needs-revision-mac":"mac","needs-revision-win":"windows"}')[github.event.label.name] }}
@@ -101,14 +104,14 @@ jobs:
 
           Pull request: https://github.com/microsoft/playwright-cli/pull/${{ github.event.pull_request.number }}
         output-dir: ${{ env.OUTPUT_DIR }}
-        result-file: commit/commit.json
+        result-file: calls.json
         artifact-name: revision
         exec: ${{ env.CI_ENV_EXEC }}
 
   push:
     name: Push the revision
     needs: [prepare, revise]
-    # commit.json is written last, so a finished revision is pushed even when the job failed later.
+    # calls.json is written when the revision is done, so it is pushed even if the job failed later.
     if: ${{ !cancelled() && needs.revise.outputs.has_revision == 'true' }}
     runs-on: ubuntu-latest
     steps:
@@ -117,6 +120,8 @@ jobs:
       with:
         client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }}
         private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }}
+        permission-contents: write
+        permission-pull-requests: write
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       with:
         ref: ${{ needs.prepare.outputs.head }}
@@ -125,13 +130,9 @@ jobs:
       with:
         name: revision
         path: ${{ runner.temp }}/revision
-    # Never execute code or scripts from the proposed changes in the app job.
-    - name: Validate and push
-      uses: microsoft/playwright/.github/actions/publish-patch@main
+    # Never execute code or scripts from the artifact in the app job. The calls are data.
+    - name: Check and make the calls
+      uses: microsoft/playwright/.github/actions/publish-calls@main
       with:
         token: ${{ steps.app-token.outputs.token }}
-        dir: ${{ runner.temp }}/revision/commit
-        patch: revision.patch
-        metadata: commit.json
-        branch: ${{ needs.prepare.outputs.branch }}
-        onto: ${{ needs.prepare.outputs.base }}
+        dir: ${{ runner.temp }}/revision

.github/workflows/triage.yml

@@ -1,9 +1,10 @@
 # Copilot triages and fixes an issue, or works on a task given by hand, without write credentials.
-# Fresh jobs post the run link on the issue, open the completed fix as a draft PR using the app, so
-# CI runs on it, and comment on the issue. The source of the CLI is in microsoft/playwright,
-# so most fixes open there, and that repo's revision workflow addresses the review. The steps that
-# are the same for every Playwright repo are actions in microsoft/playwright. The skill in
-# .claude/skills/playwright-triage/ has what is specific to this repo.
+# The agent leaves the calls to make on GitHub in calls.json. A fresh job posts the run link on the
+# issue first. After the agent, a fresh job checks the calls and makes them using the app: it opens
+# the fix as a draft PR, so CI runs on it, and comments on the issue. The source of the CLI is in
+# microsoft/playwright, so most fixes open there, and that repo's revision workflow addresses the
+# review. The steps that are the same for every Playwright repo are actions in microsoft/playwright.
+# The skill in .claude/skills/playwright-triage/ has what is specific to this repo.
 name: Issue triage
 
 on:
@@ -88,7 +89,6 @@ jobs:
       upstream_base: ${{ steps.upstream-base.outputs.sha }}
       setup: ${{ steps.setup.outcome }}
       agent: ${{ steps.agent.outputs.outcome }}
-      has_fix: ${{ steps.agent.outputs.has-result }}
       has_upstream_fix: ${{ steps.upstream-fix.outputs.has-result }}
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -136,141 +136,99 @@ jobs:
       with:
         prompt: ${{ steps.prompt.outputs.prompt }}
         output-dir: ${{ env.OUTPUT_DIR }}
-        result-file: fix/proposal.json
+        result-file: calls.json
         artifact-name: triage
         github-tools: true
         exec: ${{ env.CI_ENV_EXEC }}
+    # The publish job checks out microsoft/playwright only for a pull request there.
     - name: Check the fix in microsoft/playwright
       id: upstream-fix
       run: |
-        if [[ -f "$OUTPUT_DIR/upstream-fix/proposal.json" ]]; then
+        if node -e 'const calls = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); process.exit(calls.some(c => c.call === "pull-request" && String(c.repo).toLowerCase() === "microsoft/playwright") ? 0 : 1)' "$OUTPUT_DIR/calls.json" 2> /dev/null; then
           echo "has-result=true" >> "$GITHUB_OUTPUT"
           echo "Fix proposed in microsoft/playwright" >> "$GITHUB_STEP_SUMMARY"
         else
           echo "has-result=false" >> "$GITHUB_OUTPUT"
         fi
 
   publish:
-    name: Open draft pull request
-    needs: triage
-    # proposal.json is written last, so a finished proposal is published even when the job failed later.
-    if: ${{ !cancelled() && needs.triage.outputs.has_fix == 'true' }}
+    name: Publish the results
+    needs: [start, triage]
+    # Runs after every triage run, also a failed one: without a comment from the agent, the comment
+    # says why the bot could not finish.
+    if: ${{ !cancelled() && needs.triage.result != 'skipped' }}
     runs-on: ubuntu-latest
-    outputs:
-      pr: ${{ steps.publish.outputs.pull-request }}
     steps:
     - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
       id: app-token
       with:
         client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }}
         private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }}
+        owner: microsoft
+        repositories: playwright-cli,playwright
+        permission-contents: write
+        permission-issues: write
+        permission-pull-requests: write
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       with:
         ref: ${{ needs.triage.outputs.base }}
         token: ${{ steps.app-token.outputs.token }}
-    - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
-      with:
-        name: triage
-        path: ${{ runner.temp }}/triage
-    # Never execute code or scripts from the proposed changes in the app job.
-    - name: Validate, push and open draft
-      id: publish
-      uses: microsoft/playwright/.github/actions/publish-patch@main
-      with:
-        token: ${{ steps.app-token.outputs.token }}
-        dir: ${{ runner.temp }}/triage/fix
-        patch: fix.patch
-        metadata: proposal.json
-        body-file: pr-body.md
-        branch: triage/${{ github.event.issue.number || inputs.issue || 'task' }}/${{ github.run_id }}-${{ github.run_attempt }}
-        commit-body: "${{ (github.event.issue.number || inputs.issue) && format('Fixes: https://github.com/{0}/issues/{1}', github.repository, github.event.issue.number || inputs.issue) || '' }}"
-        pull-request: true
-
-  publish-upstream:
-    name: Open draft pull request in microsoft/playwright
-    needs: triage
-    # proposal.json is written last, so a finished proposal is published even when the job failed later.
-    if: ${{ !cancelled() && needs.triage.outputs.has_upstream_fix == 'true' }}
-    runs-on: ubuntu-latest
-    outputs:
-      pr: ${{ steps.pr.outputs.url }}
-    env:
-      BRANCH: triage/playwright-cli-${{ github.event.issue.number || inputs.issue || 'task' }}/${{ github.run_id }}-${{ github.run_attempt }}
-    steps:
-    - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
-      id: app-token
-      with:
-        client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }}
-        private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }}
-        owner: microsoft
-        repositories: playwright
+    # A pull request in microsoft/playwright goes to its checkout in playwright/, at the commit that
+    # the agent worked on.
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      if: needs.triage.outputs.has_upstream_fix == 'true'
       with:
         repository: microsoft/playwright
         ref: ${{ needs.triage.outputs.upstream_base }}
+        path: playwright
         token: ${{ steps.app-token.outputs.token }}
-    - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
-      with:
-        name: triage
-        path: ${{ runner.temp }}/triage
-    # Never execute code or scripts from the proposed changes in the app job. publish-patch opens
-    # pull requests in the repo of the workflow only, so it only pushes, and the next step opens it.
-    - name: Validate and push
-      uses: microsoft/playwright/.github/actions/publish-patch@main
-      with:
-        token: ${{ steps.app-token.outputs.token }}
-        dir: ${{ runner.temp }}/triage/upstream-fix
-        patch: fix.patch
-        metadata: proposal.json
-        branch: ${{ env.BRANCH }}
-        commit-body: "${{ (github.event.issue.number || inputs.issue) && format('Fixes: https://github.com/{0}/issues/{1}', github.repository, github.event.issue.number || inputs.issue) || '' }}"
-    - name: Open draft
-      id: pr
-      uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
-      env:
-        DIR: ${{ runner.temp }}/triage/upstream-fix
-        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-      with:
-        github-token: ${{ steps.app-token.outputs.token }}
-        script: |
-          const fs = require('fs');
-          const path = require('path');
-          const { BRANCH, DIR, RUN_URL } = process.env;
-          const { title } = JSON.parse(fs.readFileSync(path.join(DIR, 'proposal.json'), 'utf8'));
-          const description = fs.readFileSync(path.join(DIR, 'pr-body.md'), 'utf8').trim();
-          const { data: pr } = await github.rest.pulls.create({
-            owner: 'microsoft', repo: 'playwright', head: BRANCH, base: 'main', title,
-            body: `${description}\n\n<sub>Prepared by the [triage workflow of ${context.repo.owner}/${context.repo.repo}](${RUN_URL}).</sub>`,
-            draft: true,
-          });
-          core.setOutput('url', pr.html_url);
-          await core.summary.addRaw(`Draft pull request: ${pr.html_url}\n`).write();
-
-  comment:
-    name: Comment on the issue
-    needs: [start, triage, publish, publish-upstream]
-    # Runs after every triage run that has an issue, including failed ones, and whatever publish did.
-    if: ${{ !cancelled() && needs.triage.result != 'skipped' && (github.event_name == 'issues' || inputs.issue != '') }}
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-    steps:
     - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
       # A failed triage run may not have uploaded anything.
       continue-on-error: true
       with:
         name: triage
         path: ${{ runner.temp }}/triage
-    - name: Post the triage comment
-      uses: microsoft/playwright/.github/actions/triage-comment@main
+    # Without a comment from the agent on the issue, for example after a failed run, the issue gets
+    # one that says that the bot could not finish.
+    - name: Add a comment when the agent left none
+      if: github.event.issue.number || inputs.issue
+      env:
+        CALLS: ${{ runner.temp }}/triage/calls.json
+        ISSUE: ${{ github.event.issue.number || inputs.issue }}
+        SETUP: ${{ needs.triage.outputs.setup }}
+        AGENT: ${{ needs.triage.outputs.agent }}
+        FOOTER: Triaged on ${{ fromJSON('{"linux":"Linux","mac":"macOS","windows":"Windows","needs-triage":"Linux","needs-triage-mac":"macOS","needs-triage-win":"Windows"}')[inputs.platform || github.event.label.name] }} by the Playwright bot
+        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        mkdir -p "$(dirname "$CALLS")"
+        if jq -e --argjson issue "$ISSUE" 'any(.[]; .call == "comment" and .issue == $issue)' "$CALLS" > /dev/null 2>&1; then
+          exit 0
+        fi
+        if [[ "$SETUP" == "failure" ]]; then
+          reason="setting up the runner failed"
+        elif [[ "$AGENT" == "failure" ]]; then
+          reason="the Copilot session failed or ran out of budget"
+        else
+          reason="the session ended without a comment"
+        fi
+        body=$(printf "Hi, I'm the Playwright bot. I couldn't finish triaging this issue: %s. The agent run has the details.\n\n<sub>%s - [agent run](%s)</sub>\n<!-- playwright-ai-triage -->" "$reason" "$FOOTER" "$RUN_URL")
+        comment=$(jq -n --arg repo "$GITHUB_REPOSITORY" --argjson issue "$ISSUE" --arg body "$body" '{call: "comment", repo: $repo, issue: $issue, body: $body}')
+        # A calls.json that is not an array is replaced: its calls would be rejected anyway.
+        jq --argjson comment "$comment" '. + [$comment]' "$CALLS" > "$CALLS.tmp" 2> /dev/null \
+          || jq -n --argjson comment "$comment" '[$comment]' > "$CALLS.tmp"
+        mv "$CALLS.tmp" "$CALLS"
+    # Never execute code or scripts from the artifact in the app job. The calls are data.
+    - name: Check and make the calls
+      id: publish
+      uses: microsoft/playwright/.github/actions/publish-calls@main
       with:
-        issue: ${{ github.event.issue.number || inputs.issue }}
+        token: ${{ steps.app-token.outputs.token }}
         dir: ${{ runner.temp }}/triage
-        footer: Triaged on ${{ fromJSON('{"linux":"Linux","mac":"macOS","windows":"Windows","needs-triage":"Linux","needs-triage-mac":"macOS","needs-triage-win":"Windows"}')[inputs.platform || github.event.label.name] }} by the Playwright bot
-        job-result: ${{ needs.triage.result }}
-        setup-outcome: ${{ needs.triage.outputs.setup }}
-        agent-outcome: ${{ needs.triage.outputs.agent }}
-        setup-failure-reason: installing the dependencies, building microsoft/playwright or downloading the browsers failed
-        pull-request: ${{ needs.publish.outputs.pr || needs.publish-upstream.outputs.pr }}
-        publish-result: ${{ (needs.publish.result == 'failure' || needs.publish-upstream.result == 'failure') && 'failure' || '' }}
-        start-comment: ${{ needs.start.outputs.comment }}
+    - name: Hide the comment with the run link
+      if: ${{ !cancelled() && steps.publish.outputs.comment == 'true' && needs.start.outputs.comment != '' }}
+      env:
+        GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        ID: ${{ needs.start.outputs.comment }}
+      run: |
+        gh api graphql -f query='mutation($id: ID!) { minimizeComment(input: { subjectId: $id, classifier: OUTDATED }) { clientMutationId } }' -f id="$ID" \
+          || echo "::warning::Could not hide the comment with the run link"