From b59d9fa3bb182d0ba031d81a068f19b7ce224d8c Mon Sep 17 00:00:00 2001 From: sourcehold Date: Wed, 11 Nov 2026 15:39:11 +0100 Subject: [PATCH] Cover every fork's reimpl/ branches from the rollup tooling itself status_rollup.py gains --fetch-forks (REIMPL_FETCH_FORKS=1): it enumerates the root repository's forks with gh, adds each as a remote and shallow-fetches its reimpl/ branches before scanning, so a local run covers what CI covers. It also reports per-fork branch counts, and says so when nothing but the local repo is fetched. Fork identity comes from the remote's url rather than its name, since an 'owner-name' remote cannot be split unambiguously. status_regression.py understands the same two variables. The touched-file list is now settled before the fetch: a shallow fork fetch grafts the commit under test as parentless when the PR comes from one of those branches, which would turn the normal case into the script's "no merge base" hard error. The three workflows drop their fetch_fork_branches.sh step in favour of that flag, so the fetch and the scan cannot drift apart. The script stays as a standalone utility, with a PowerShell port beside it; both now default to sourcehold/OpenSHC. pick_work.py reports where to get each pick: the URL of the file behind the rollup's best_source, and with --all-sources every fork branch that has the function, collapsed by blob so the dozens of branches cut from one another show as one entry. New --format urls. Co-Authored-By: Claude Opus 5 --- .github/workflows/status-regression.yml | 17 +- .github/workflows/status-rollup-publish.yml | 21 +- .github/workflows/status-rollup.yml | 15 +- tools/pick_work.py | 216 +++++++++++++++++- .../todo/Fetch-ForkBranches.ps1 | 66 ++++++ .../todo/fetch_fork_branches.sh | 7 +- .../todo/status_regression.py | 33 ++- .../todo/status_rollup.py | 140 +++++++++++- 8 files changed, 472 insertions(+), 43 deletions(-) create mode 100644 tools/reimplementation-helper/todo/Fetch-ForkBranches.ps1 diff --git a/.github/workflows/status-regression.yml b/.github/workflows/status-regression.yml index 3c4aca4658..ec2c474bd4 100644 --- a/.github/workflows/status-regression.yml +++ b/.github/workflows/status-regression.yml @@ -164,15 +164,6 @@ jobs: git diff --name-only --diff-filter=d "$mb" "$PR_REF" > touched.txt echo "files changed since the merge base: $(wc -l < touched.txt)" - - name: Fetch fork reimpl/ branches - if: ${{ steps.ctx.outputs.mode != 'skip' && steps.fetch.outputs.gone != 'yes' }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - run: | - chmod +x tools/reimplementation-helper/todo/fetch_fork_branches.sh - tools/reimplementation-helper/todo/fetch_fork_branches.sh "$REPO" - # A fork's own reimpl/ branch must not count as prior art against the PR # that proposes it, so collect the head SHA of every PR under test. - name: Collect the PR head SHAs to exclude @@ -206,6 +197,14 @@ jobs: PR_HEAD_SHA: ${{ steps.heads.outputs.shas }} LABEL: ${{ steps.ctx.outputs.mode == 'queue' && 'this merge group' || 'this PR' }} OUT: status_regression.md + # The script adds every fork as a remote and shallow-fetches its + # reimpl/ branches itself, after it has settled the touched-file list + # (TOUCHED_LIST above) -- that fetch grafts the commit under test as + # parentless when the PR comes from one of those branches, so the + # order matters and belongs with the code that depends on it. + REIMPL_FETCH_FORKS: '1' + REIMPL_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail python -I tools/reimplementation-helper/todo/status_regression.py | tee regression.log diff --git a/.github/workflows/status-rollup-publish.yml b/.github/workflows/status-rollup-publish.yml index 937fa5aa80..08a8290ea9 100644 --- a/.github/workflows/status-rollup-publish.yml +++ b/.github/workflows/status-rollup-publish.yml @@ -92,15 +92,6 @@ jobs: echo "mode=skip" >> "$GITHUB_OUTPUT" fi - - name: Fetch fork reimpl/ branches - if: ${{ steps.ctx.outputs.mode != 'skip' }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - run: | - chmod +x tools/reimplementation-helper/todo/fetch_fork_branches.sh - tools/reimplementation-helper/todo/fetch_fork_branches.sh "$REPO" - # Data, not code: the PR head arrives through the base repo's own # refs/pull/N/head, and the rollup only ever reads status/*.txt from it. - name: Fetch the PR head as data @@ -118,14 +109,24 @@ jobs: fi echo "Included PR #$PR head $got as data." + # --fetch-forks adds every fork as a remote and shallow-fetches its + # reimpl/ branches before the rollup reads them, so the fetch and the scan + # cannot drift apart. It runs after the PR head has been fetched above, so + # both are in the clone by the time anything is scanned. Only branch tips + # are fetched and nothing from a fetched branch is executed: one status + # file per ref is read. - name: Roll up status files if: ${{ steps.ctx.outputs.mode != 'skip' }} env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REIMPL_REPO: ${{ github.repository }} # include the one fetched PR head alongside the reimpl/ branches. # Anchored on the exact number: a bare ^pr/ would also pick up every # other refs/pull ref that happens to be present in the clone. REIMPL_FILTER: ${{ steps.ctx.outputs.mode == 'pr' && format('/reimpl/|^pr/{0}$', steps.ctx.outputs.pr) || '/reimpl/' }} - run: python -I tools/reimplementation-helper/todo/status_rollup.py | tee rollup.log + run: | + python -I tools/reimplementation-helper/todo/status_rollup.py \ + --fetch-forks | tee rollup.log - name: Check the output before publishing it if: ${{ steps.ctx.outputs.mode != 'skip' }} diff --git a/.github/workflows/status-rollup.yml b/.github/workflows/status-rollup.yml index 2dc372cae7..5c2de12913 100644 --- a/.github/workflows/status-rollup.yml +++ b/.github/workflows/status-rollup.yml @@ -36,16 +36,17 @@ jobs: with: persist-credentials: false - - name: Fetch fork reimpl/ branches + # --fetch-forks adds every fork as a remote and shallow-fetches its + # reimpl/ branches before the rollup reads them, so the fetch and the scan + # cannot drift apart. Only branch tips are fetched and nothing from a + # fetched branch is executed: one status file per ref is read. + - name: Roll up status files env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} + REIMPL_REPO: ${{ github.repository }} run: | - chmod +x tools/reimplementation-helper/todo/fetch_fork_branches.sh - tools/reimplementation-helper/todo/fetch_fork_branches.sh "$REPO" - - - name: Roll up status files - run: python -I tools/reimplementation-helper/todo/status_rollup.py | tee rollup.log + python -I tools/reimplementation-helper/todo/status_rollup.py \ + --fetch-forks | tee rollup.log - name: Summary run: | diff --git a/tools/pick_work.py b/tools/pick_work.py index 6c1c87f0d3..e4185ce043 100644 --- a/tools/pick_work.py +++ b/tools/pick_work.py @@ -7,10 +7,16 @@ The rollup is fetched from the project's latest-sarif release by default; pass --tsv with a local path (or another URL) to override it. +Every pick is reported with the URL of the place its best match lives, derived +from the rollup's best_source (``main`` or a remote-tracking branch such as +``owner-OpenSHC/reimpl/foo``) and the local git remotes. With --all-sources the +local reimpl/ branches are scanned as well, so each pick lists every fork +branch that has the function and the URL of the file on that branch. + Usage: python pick_worst.py [-n N] [--tsv PATH_OR_URL] [--namespace NS] [--include-done] [--max-pct P] [--spread] - [--format table|tsv|json|addresses] + [--all-sources] [--format table|tsv|json|addresses|urls] """ import argparse @@ -18,6 +24,8 @@ import io import json import os +import re +import subprocess import sys import urllib.error import urllib.request @@ -87,10 +95,167 @@ def load(source): "cpp": cpp, "namespace": namespace_of(cpp), "function": function_of(cpp), + "url": "", + "sources": [], }) return rows +DEFAULT_REPO_URL = "https://github.com/sourcehold/OpenSHC" +#: one entry per line: SHC__0x | % | +ENTRY = re.compile(r"^\s*(SHC_[0-9A-Fa-f]+_0x[0-9A-Fa-f]+)\s*\|\s*([0-9.]+)%\s*\|\s*(.*?)\s*$") +STATUS_GLOB = re.compile(r"^status/addresses-SHC-[0-9A-Fa-f]+\.txt$") + + +def git(args): + """-> (returncode, stdout); never raises, so a missing repo degrades gracefully.""" + try: + p = subprocess.run(["git"] + args, capture_output=True, text=True, + errors="replace") + except OSError: + return 1, "" + return p.returncode, p.stdout + + +def web_base(url): + """git remote url -> browsable https base, or '' if it is not a known host.""" + url = url.strip() + if url.startswith("git@"): + host, _, path = url[4:].partition(":") + url = "https://{}/{}".format(host, path) + elif url.startswith("ssh://git@"): + url = "https://" + url[len("ssh://git@"):] + if not url.startswith(("http://", "https://")): + return "" + return url[:-4].rstrip("/") if url.endswith(".git") else url.rstrip("/") + + +def remote_bases(): + """-> {remote name: browsable https base} for the local remotes.""" + rc, out = git(["remote", "-v"]) + bases = {} + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0] not in bases: + base = web_base(parts[1]) + if base: + bases[parts[0]] = base + return bases + + +def source_url(source, cpp, bases): + """'owner-OpenSHC/reimpl/foo' + a cpp path -> the URL of that file on GitHub. + + 'main' (the rollup's name for HEAD) is the project's own repository, + sourcehold/OpenSHC. Remote names never contain a slash, so the + first component of a remote-tracking branch is the remote. + """ + if not source or source == "main": + # the rollup's 'main' is the project's own main, not whatever this + # checkout's origin happens to be + base, branch = DEFAULT_REPO_URL, "main" + else: + remote, _, branch = source.partition("/") + if not branch: + return "" + base = bases.get(remote) + if base is None: + # not fetched locally: rebuild owner/name from the remote name, which + # fetch_fork_branches made by replacing the slash with a dash + owner, _, name = remote.partition("-") + if not name: + return "" + base = "https://github.com/{}/{}".format(owner, name) + if cpp: + return "{}/blob/{}/{}".format(base, branch, cpp.replace("\\", "/")) + return "{}/tree/{}".format(base, branch) + + +def reimpl_refs(filter_re): + """-> ['HEAD'] + the remote-tracking branches whose name matches the filter.""" + rc, out = git(["branch", "-r", "--format=%(refname:short)"]) + refs = [b.strip() for b in out.splitlines() + if b.strip() and "->" not in b and filter_re.search(b)] + return ["HEAD"] + sorted(refs) + + +def ref_rows(ref, wanted): + """-> {shc id: (pct, status)} for the wanted ids in one ref's status files.""" + rc, out = git(["ls-tree", "-r", "--name-only", ref, "--", "status"]) + rows = {} + for path in out.splitlines(): + path = path.strip() + if not STATUS_GLOB.match(path): + continue + rc, text = git(["show", "{}:{}".format(ref, path)]) + if rc != 0: + continue + for line in text.splitlines(): + m = ENTRY.match(line) + if m and m.group(1) in wanted: + rows[m.group(1)] = (float(m.group(2)), m.group(3).strip()) + return rows + + +def collect_sources(picks, pattern="/reimpl/", min_pct=None, progress=False, + keep_duplicates=False): + """Fill each pick's 'sources' with every ref that reports the function. + + Branch contents are taken as truth exactly as status_rollup.py does: nothing + is re-measured, only the per-address status files are read. + + Most reimpl/ branches are cut from one another, so the same file turns up + verbatim on dozens of them. Sources are therefore grouped by the blob id of + the cpp on that branch: one entry per distinct version of the function, + carrying the other branches that hold the identical file in 'also_on'. + """ + wanted = {r["shc_id"]: r for r in picks} + bases = remote_bases() + refs = reimpl_refs(re.compile(pattern)) + for i, ref in enumerate(refs, 1): + if progress: + print("scanning {}/{}: {}".format(i, len(refs), ref), + end="\r", file=sys.stderr) + for sid, (pct, status) in ref_rows(ref, wanted).items(): + # every branch carries a status line for every address, so a branch + # that has not touched the function reports 0% and has no file + if pct <= (0.0 if min_pct is None else min_pct - 1e-9): + continue + row = wanted[sid] + source = "main" if ref == "HEAD" else ref + rc, blob = git(["rev-parse", "--verify", "--quiet", + "{}:{}".format(ref, row["cpp"])]) + row["sources"].append({ + "source": source, + "pct": pct, + "status": status, + "url": source_url(source, row["cpp"], bases), + "blob": blob.strip() if rc == 0 else "", + "also_on": [], + }) + if progress: + print(" " * 60, end="\r", file=sys.stderr) + for row in picks: + row["sources"].sort(key=lambda s: (-s["pct"], s["source"])) + if not keep_duplicates: + row["sources"] = dedupe_sources(row["sources"]) + return picks + + +def dedupe_sources(sources): + """Collapse sources whose cpp is byte-identical into one entry each.""" + out, by_blob = [], {} + for src in sources: + first = by_blob.get(src["blob"]) if src["blob"] else None + if first is None: + if src["blob"]: + by_blob[src["blob"]] = src + out.append(src) + else: + first["also_on"].append(src["source"]) + return out + + #: namespaces that hold generated / runtime glue rather than game logic SKIPPED_NAMESPACES = ("OS", "Global::Init") @@ -150,8 +315,17 @@ def render(picks, fmt): return json.dumps(picks, indent=2) if fmt == "addresses": return "\n".join(r["address"] for r in picks) + if fmt == "urls": + out = [] + for r in picks: + for src in r["sources"] or [{"source": r["source"], "pct": r["pct"], + "url": r["url"], "also_on": []}]: + out.append("{}\t{:.1f}\t{}\t{}".format( + r["address"], src["pct"], src["source"], src["url"])) + return "\n".join(out) if fmt == "tsv": - cols = ["address", "pct", "namespace", "function", "refs", "status", "cpp"] + cols = ["address", "pct", "namespace", "function", "refs", "status", + "source", "url", "cpp"] out = ["\t".join(cols)] out += ["\t".join(str(r[c]) for c in cols) for r in picks] return "\n".join(out) @@ -166,6 +340,15 @@ def render(picks, fmt): r["pct"], r["address"], r["refs"], r["function"] or r["shc_id"])) if r["status"]: lines.append(" status: {}".format(r["status"])) + if r["sources"]: + for src in r["sources"]: + lines.append(" {:>6.1f}% {}".format( + src["pct"], src["url"] or src["source"])) + if src["also_on"]: + lines.append(" (identical on {} other " + "branch(es))".format(len(src["also_on"]))) + elif r["url"]: + lines.append(" {:>6.1f}% {}".format(r["pct"], r["url"])) counts = defaultdict(int) for r in picks: counts[r["namespace"]] += 1 @@ -201,7 +384,21 @@ def main(argv=None): help="only consider rows with at least this many refs_at_100") ap.add_argument("--spread", action="store_true", help="ignore namespace grouping; take the globally worst N") - ap.add_argument("--format", choices=["table", "tsv", "json", "addresses"], + ap.add_argument("--all-sources", action="store_true", + help="scan the local reimpl/ branches and list every fork " + "branch that has the function, with the URL of its file " + "there (slow: one pass over every remote-tracking branch)") + ap.add_argument("--sources-filter", default="/reimpl/", + help="regex selecting the remote-tracking branches to scan " + "with --all-sources (default '/reimpl/')") + ap.add_argument("--sources-min-pct", type=float, + help="with --all-sources, only list branches at or above " + "this match percentage (default: above 0%%, i.e. every " + "branch that has actually touched the function)") + ap.add_argument("--keep-duplicate-sources", action="store_true", + help="with --all-sources, list every branch separately " + "instead of collapsing branches whose file is identical") + ap.add_argument("--format", choices=["table", "tsv", "json", "addresses", "urls"], default="table") ap.add_argument("--list-namespaces", action="store_true", help="print a per-namespace badness summary instead of picks") @@ -245,7 +442,18 @@ def main(argv=None): min(x["pct"] for x in members), ns)) return 0 - print(render(select(pool, args.count, spread=args.spread), args.format)) + picks = select(pool, args.count, spread=args.spread) + + bases = remote_bases() + for row in picks: + row["url"] = source_url(row["source"], row["cpp"], bases) + if args.all_sources: + collect_sources(picks, pattern=args.sources_filter, + min_pct=args.sources_min_pct, + progress=sys.stderr.isatty(), + keep_duplicates=args.keep_duplicate_sources) + + print(render(picks, args.format)) return 0 diff --git a/tools/reimplementation-helper/todo/Fetch-ForkBranches.ps1 b/tools/reimplementation-helper/todo/Fetch-ForkBranches.ps1 new file mode 100644 index 0000000000..48bbd3c79f --- /dev/null +++ b/tools/reimplementation-helper/todo/Fetch-ForkBranches.ps1 @@ -0,0 +1,66 @@ +<# +.SYNOPSIS +Add every fork of the root repository as a remote and shallow-fetch its +reimpl/ branches, so that status_rollup.py sees them as ordinary +remote-tracking branches. + +.DESCRIPTION +Only branch tips are fetched: the rollup reads one file per ref, and nothing +from a fetched branch is ever executed. + +Requires: gh (authenticated via GH_TOKEN), git. Run from the repo root. + +The repository defaults to the project's own, sourcehold/OpenSHC; pass another +one to walk up from a fork instead. + +.EXAMPLE +tools\reimplementation-helper\todo\Fetch-ForkBranches.ps1 +#> +[CmdletBinding()] +param( + [Parameter(Position = 0)] + [string]$RepoFullName = 'sourcehold/OpenSHC' +) + +$ErrorActionPreference = 'Stop' + +# Walk up to the root repository, then enumerate its forks. +$root = gh repo view $RepoFullName --json parent,nameWithOwner ` + --jq 'if .parent then .parent.owner.login + "/" + .parent.name else .nameWithOwner end' +if ($LASTEXITCODE -ne 0) { throw "gh repo view failed for $RepoFullName" } +$root = $root.Trim() +Write-Host "Root repository: $root" + +$forks = gh api "repos/$root/forks" --paginate --jq '.[].full_name' +if ($LASTEXITCODE -ne 0) { throw "gh api repos/$root/forks failed" } + +$repos = @($root) + @($forks) | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } | + Sort-Object -Unique +Write-Host "Repositories to scan: $($repos.Count)" + +foreach ($repo in $repos) { + # Ignore anything that is not a plain owner/name, so that a surprising API + # response cannot turn into command arguments or a path. + if ($repo -notmatch '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$') { + Write-Host " skipped (unexpected name): $repo" + continue + } + $remote = $repo -replace '/', '-' + git remote remove $remote 2>$null | Out-Null + git remote add $remote "https://github.com/$repo" + + # A fork with no reimpl/ branches is normal, not an error. + git fetch --no-tags --depth=1 $remote "+refs/heads/reimpl/*:refs/remotes/$remote/reimpl/*" 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { + $n = @(git branch -r --list "$remote/reimpl/*").Count + if ($n -gt 0) { Write-Host " ${repo}: $n reimpl branch(es)" } + } + else { + Write-Host " ${repo}: skipped (unreachable or no reimpl/ branches)" + } +} + +$total = @(git branch -r | Where-Object { $_ -match '/reimpl/' }).Count +Write-Host "Total reimpl/ branches: $total" diff --git a/tools/reimplementation-helper/todo/fetch_fork_branches.sh b/tools/reimplementation-helper/todo/fetch_fork_branches.sh index ac76b28f7b..08d4fca1b5 100644 --- a/tools/reimplementation-helper/todo/fetch_fork_branches.sh +++ b/tools/reimplementation-helper/todo/fetch_fork_branches.sh @@ -7,11 +7,14 @@ # from a fetched branch is ever executed. # # Requires: gh (authenticated via GH_TOKEN), git. Run from the repo root. -# Usage: tools/reimplementation-helper/todo/fetch_fork_branches.sh +# Usage: tools/reimplementation-helper/todo/fetch_fork_branches.sh [owner/repo] +# +# The repository defaults to the project's own; pass another one to walk up from +# a fork instead. set -euo pipefail -repo_full_name="${1:?usage: fetch_fork_branches.sh }" +repo_full_name="${1:-sourcehold/OpenSHC}" # Walk up to the root repository, then enumerate its forks. root=$(gh repo view "$repo_full_name" --json parent,nameWithOwner \ diff --git a/tools/reimplementation-helper/todo/status_regression.py b/tools/reimplementation-helper/todo/status_regression.py index 5b4eb82c7b..560998b67f 100644 --- a/tools/reimplementation-helper/todo/status_regression.py +++ b/tools/reimplementation-helper/todo/status_regression.py @@ -30,6 +30,13 @@ PR that proposes it. A merge group may hold several PRs, hence the list. REIMPL_FILTER regex of remote-tracking branches to include (default '/reimpl/') + REIMPL_FETCH_FORKS '1' to add every fork as a remote and shallow-fetch its + reimpl/ branches first, so the baseline covers every fork + without a separate fetch step (needs gh, authenticated). The + fetch happens only after the touched-file list is settled, + because it grafts the commit under test as parentless. + REIMPL_REPO repository whose forks are enumerated, as owner/name (default: + the origin remote, else sourcehold/OpenSHC) TOUCHED_LIST file of one changed path per line; when set it is used instead of asking git, so the list can be worked out before anything shallow is fetched into the repository @@ -42,7 +49,8 @@ import os, sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from status_rollup import rows_at, header_cpp_map, reimpl_refs, best_per_id, run, parse_resolvers +from status_rollup import (rows_at, header_cpp_map, reimpl_refs, best_per_id, run, + parse_resolvers, fetch_fork_branches) # status files carry one decimal, so compare with a tolerance below that EPS = 0.05 @@ -110,18 +118,14 @@ def main(): return 0 base_rows = rows_at(base_ref) - # The PR's own branch may also be one of the fetched fork reimpl/ branches. - exclude = set(os.environ.get('PR_HEAD_SHA', '').replace(',', ' ').split()) - refs = reimpl_refs(exclude_shas=exclude) - best, at100, seen = best_per_id(refs) - print('baseline refs with a status file: %d' % len(seen)) - - cpp_of = header_cpp_map() - # Which .cpp files does the change actually touch? A status line can differ # for reasons that are not this change's doing -- most often a branch that # has not been updated since main improved the same function -- so an entry # only counts when the change also edits the file behind it. + # + # This is answered BEFORE any fork branch is fetched: a shallow fetch grafts + # the commit under test as parentless whenever the PR comes from one of those + # branches, and the merge base is then unanswerable. touched, mb = touched_files(base_ref, pr_ref) if touched is None: # Not something to work around: without the file list this would compare @@ -134,6 +138,17 @@ def main(): ) return 1 + if os.environ.get('REIMPL_FETCH_FORKS') == '1': + fetch_fork_branches(os.environ.get('REIMPL_REPO')) + + # The PR's own branch may also be one of the fetched fork reimpl/ branches. + exclude = set(os.environ.get('PR_HEAD_SHA', '').replace(',', ' ').split()) + refs = reimpl_refs(exclude_shas=exclude) + best, at100, seen = best_per_id(refs) + print('baseline refs with a status file: %d' % len(seen)) + + cpp_of = header_cpp_map() + # A resolver the change itself adds or moves is not in the base headers, so # re-read the .func.hpp files it touches at the PR's own ref. cpp_of = dict(cpp_of) diff --git a/tools/reimplementation-helper/todo/status_rollup.py b/tools/reimplementation-helper/todo/status_rollup.py index 03221fd13e..4af2b217ac 100644 --- a/tools/reimplementation-helper/todo/status_rollup.py +++ b/tools/reimplementation-helper/todo/status_rollup.py @@ -7,6 +7,19 @@ Run from the repo root: python -I tools/reimplementation-helper/todo/status_rollup.py +Only branches that are present as remote-tracking refs can be read, so a plain +clone sees nothing but its own. Pass --fetch-forks (or set REIMPL_FETCH_FORKS=1) +to add every fork of the root repository as a remote first and shallow-fetch its +reimpl/ branches, which is what fetch_fork_branches.sh does in CI: + + python -I tools/reimplementation-helper/todo/status_rollup.py --fetch-forks + +That needs gh, authenticated (GH_TOKEN). Only branch tips are fetched, and +nothing from a fetched branch is ever executed -- one status file per ref is +read. The repository is taken from --repo, else REIMPL_REPO, else the origin +remote, else sourcehold/OpenSHC. Without the flag the fetch is skipped and only +what is already fetched is scanned, so the CI workflow keeps working unchanged. + Output: status_rollup.tsv in the current working directory. Set REIMPL_FILTER to a regex to widen or narrow which remote-tracking branches @@ -19,10 +32,12 @@ importable: everything that runs belongs in main(). """ -import os, re, subprocess +import argparse, os, re, subprocess from collections import Counter ROOT = 'src/OpenSHC' +#: the project's own repository; forks are enumerated from its root +DEFAULT_REPO = 'sourcehold/OpenSHC' STATUS_GLOB = re.compile(r'^status/addresses-SHC-[0-9A-Fa-f]+\.txt$') # one entry per line: SHC__0x | % | ENTRY = re.compile(r'^\s*(SHC_[0-9A-Fa-f]+_0x([0-9A-Fa-f]+))\s*\|\s*([0-9.]+)%\s*\|\s*(.*?)\s*$') @@ -102,6 +117,68 @@ def header_cpp_map(): return cpp_of +REPO_NAME = re.compile(r'^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$') + + +def origin_repo(): + """-> 'owner/name' of the origin remote, or '' if it is not a GitHub url.""" + rc, out = run(['git', 'remote', 'get-url', 'origin']) + url = out.strip() + if url.endswith('.git'): + url = url[:-4] + m = re.search(r'github\.com[:/]+([A-Za-z0-9._-]+/[A-Za-z0-9._-]+)/?$', url) + return m.group(1) if m else '' + + +def fork_repos(repo_full_name): + """-> [root repo, every fork of it]; the root is the repo's parent, if any.""" + rc, out = run(['gh', 'repo', 'view', repo_full_name, '--json', 'parent,nameWithOwner', + '--jq', 'if .parent then .parent.owner.login + "/" + .parent.name ' + 'else .nameWithOwner end']) + if rc != 0 or not out.strip(): + raise RuntimeError('gh repo view %s failed (is gh installed and authenticated?)' + % repo_full_name) + root = out.strip() + rc, out = run(['gh', 'api', 'repos/%s/forks' % root, '--paginate', + '--jq', '.[].full_name']) + if rc != 0: + raise RuntimeError('gh api repos/%s/forks failed' % root) + repos = {root} | {ln.strip() for ln in out.splitlines() if ln.strip()} + # ignore anything that is not a plain owner/name, so that a surprising API + # response cannot turn into command arguments or a path + return root, sorted(r for r in repos if REPO_NAME.match(r)) + + +def fetch_fork_branches(repo_full_name=None): + """Add every fork as a remote and shallow-fetch its reimpl/ branches. + + The Python counterpart of fetch_fork_branches.sh, so that a local run covers + the same branches CI does. Only branch tips are fetched. + """ + repo_full_name = (repo_full_name or os.environ.get('REIMPL_REPO') + or origin_repo() or DEFAULT_REPO) + root, repos = fork_repos(repo_full_name) + print('root repository: %s' % root) + print('repositories to scan: %d' % len(repos)) + fetched = 0 + for repo in repos: + remote = repo.replace('/', '-') + run(['git', 'remote', 'remove', remote]) + run(['git', 'remote', 'add', remote, 'https://github.com/%s' % repo]) + # a fork with no reimpl/ branches is normal, not an error + rc, out = run(['git', 'fetch', '--no-tags', '--depth=1', remote, + '+refs/heads/reimpl/*:refs/remotes/%s/reimpl/*' % remote]) + if rc != 0: + print(' %s: skipped (unreachable or no reimpl/ branches)' % repo) + continue + rc, out = run(['git', 'branch', '-r', '--list', '%s/reimpl/*' % remote]) + n = len([b for b in out.splitlines() if b.strip()]) + if n: + fetched += n + print(' %s: %d reimpl branch(es)' % (repo, n)) + print('fetched reimpl/ branches: %d' % fetched) + + def reimpl_refs(filter_re=None, exclude_shas=()): """-> ['HEAD', ] (sorted).""" if filter_re is None: @@ -141,7 +218,55 @@ def best_per_id(refs): return best, at100, seen -def main(): +def remote_repos(): + """-> {remote name: 'owner/name'} from the remotes' urls. + + The url is the only reliable source: a remote added by hand is not named + after the repository, and an owner or a name may itself contain a dash, so + splitting the 'owner-name' remote fetch_fork_branches creates is ambiguous. + """ + rc, out = run(['git', 'remote', '-v']) + repos = {} + for line in out.splitlines(): + parts = line.split() + if len(parts) < 2 or parts[0] in repos: + continue + url = parts[1][:-4] if parts[1].endswith('.git') else parts[1] + m = re.search(r'github\.com[:/]+([A-Za-z0-9._-]+/[A-Za-z0-9._-]+)/?$', url) + if m: + repos[parts[0]] = m.group(1) + return repos + + +def fork_of(ref, repos=None): + """'owner-name/reimpl/foo' -> 'owner/name'; the remote name has no slash.""" + if ref == 'HEAD': + return 'main' + remote = ref.split('/', 1)[0] + if repos is None: + repos = remote_repos() + if remote in repos: + return repos[remote] + owner, _, name = remote.partition('-') + return '%s/%s' % (owner, name) if name else remote + + +def main(argv=None): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--fetch-forks', action='store_true', + default=os.environ.get('REIMPL_FETCH_FORKS') == '1', + help='add every fork as a remote and shallow-fetch its reimpl/ ' + 'branches before rolling up (needs gh; REIMPL_FETCH_FORKS=1)') + ap.add_argument('--repo', default=os.environ.get('REIMPL_REPO'), + help="repository to enumerate the forks of, as owner/name " + "(default: REIMPL_REPO, else the origin remote, else " + + DEFAULT_REPO + ")") + args = ap.parse_args(argv) + + if args.fetch_forks: + fetch_fork_branches(args.repo) + # data files are written to the current working directory (run from the repo root) cpp_of = header_cpp_map() print('resolvers in main headers:', len(cpp_of)) @@ -149,6 +274,17 @@ def main(): refs = reimpl_refs() best, at100, seen = best_per_id(refs) print('refs scanned: %d (%d with a status file)' % (len(refs), len(seen))) + repos = remote_repos() + # GitHub repository names are case-insensitive, and the same fork is often + # reachable under two remotes, so count one entry per repository + forks = Counter(fork_of(r, repos).lower() for r in refs) + print('forks covered: %d' % len([f for f in forks if f != 'main'])) + for fork, n in sorted(forks.items(), key=lambda kv: (-kv[1], kv[0])): + if fork != 'main': + print(' %-40s %d branch(es)' % (fork, n)) + if len(forks) <= 1 and not args.fetch_forks: + print('NOTE: no fork branches are fetched; rerun with --fetch-forks to ' + 'include every fork of the repository') missing = [r for r in refs if r not in seen] if missing: print('refs with NO status file:', len(missing))