diff --git a/.github/workflows/status-regression.yml b/.github/workflows/status-regression.yml index 3c4aca4658..ec2c474bd4 100644 --- a/.github/workflows/status-regression.yml +++ b/.github/workflows/status-regression.yml @@ -164,15 +164,6 @@ jobs: git diff --name-only --diff-filter=d "$mb" "$PR_REF" > touched.txt echo "files changed since the merge base: $(wc -l < touched.txt)" - - name: Fetch fork reimpl/ branches - if: ${{ steps.ctx.outputs.mode != 'skip' && steps.fetch.outputs.gone != 'yes' }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - run: | - chmod +x tools/reimplementation-helper/todo/fetch_fork_branches.sh - tools/reimplementation-helper/todo/fetch_fork_branches.sh "$REPO" - # A fork's own reimpl/ branch must not count as prior art against the PR # that proposes it, so collect the head SHA of every PR under test. - name: Collect the PR head SHAs to exclude @@ -206,6 +197,14 @@ jobs: PR_HEAD_SHA: ${{ steps.heads.outputs.shas }} LABEL: ${{ steps.ctx.outputs.mode == 'queue' && 'this merge group' || 'this PR' }} OUT: status_regression.md + # The script adds every fork as a remote and shallow-fetches its + # reimpl/ branches itself, after it has settled the touched-file list + # (TOUCHED_LIST above) -- that fetch grafts the commit under test as + # parentless when the PR comes from one of those branches, so the + # order matters and belongs with the code that depends on it. + REIMPL_FETCH_FORKS: '1' + REIMPL_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail python -I tools/reimplementation-helper/todo/status_regression.py | tee regression.log diff --git a/.github/workflows/status-rollup-publish.yml b/.github/workflows/status-rollup-publish.yml index 937fa5aa80..08a8290ea9 100644 --- a/.github/workflows/status-rollup-publish.yml +++ b/.github/workflows/status-rollup-publish.yml @@ -92,15 +92,6 @@ jobs: echo "mode=skip" >> "$GITHUB_OUTPUT" fi - - name: Fetch fork reimpl/ branches - if: ${{ steps.ctx.outputs.mode != 'skip' }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - run: | - chmod +x tools/reimplementation-helper/todo/fetch_fork_branches.sh - tools/reimplementation-helper/todo/fetch_fork_branches.sh "$REPO" - # Data, not code: the PR head arrives through the base repo's own # refs/pull/N/head, and the rollup only ever reads status/*.txt from it. - name: Fetch the PR head as data @@ -118,14 +109,24 @@ jobs: fi echo "Included PR #$PR head $got as data." + # --fetch-forks adds every fork as a remote and shallow-fetches its + # reimpl/ branches before the rollup reads them, so the fetch and the scan + # cannot drift apart. It runs after the PR head has been fetched above, so + # both are in the clone by the time anything is scanned. Only branch tips + # are fetched and nothing from a fetched branch is executed: one status + # file per ref is read. - name: Roll up status files if: ${{ steps.ctx.outputs.mode != 'skip' }} env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REIMPL_REPO: ${{ github.repository }} # include the one fetched PR head alongside the reimpl/ branches. # Anchored on the exact number: a bare ^pr/ would also pick up every # other refs/pull ref that happens to be present in the clone. REIMPL_FILTER: ${{ steps.ctx.outputs.mode == 'pr' && format('/reimpl/|^pr/{0}$', steps.ctx.outputs.pr) || '/reimpl/' }} - run: python -I tools/reimplementation-helper/todo/status_rollup.py | tee rollup.log + run: | + python -I tools/reimplementation-helper/todo/status_rollup.py \ + --fetch-forks | tee rollup.log - name: Check the output before publishing it if: ${{ steps.ctx.outputs.mode != 'skip' }} diff --git a/.github/workflows/status-rollup.yml b/.github/workflows/status-rollup.yml index 2dc372cae7..5c2de12913 100644 --- a/.github/workflows/status-rollup.yml +++ b/.github/workflows/status-rollup.yml @@ -36,16 +36,17 @@ jobs: with: persist-credentials: false - - name: Fetch fork reimpl/ branches + # --fetch-forks adds every fork as a remote and shallow-fetches its + # reimpl/ branches before the rollup reads them, so the fetch and the scan + # cannot drift apart. Only branch tips are fetched and nothing from a + # fetched branch is executed: one status file per ref is read. + - name: Roll up status files env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} + REIMPL_REPO: ${{ github.repository }} run: | - chmod +x tools/reimplementation-helper/todo/fetch_fork_branches.sh - tools/reimplementation-helper/todo/fetch_fork_branches.sh "$REPO" - - - name: Roll up status files - run: python -I tools/reimplementation-helper/todo/status_rollup.py | tee rollup.log + python -I tools/reimplementation-helper/todo/status_rollup.py \ + --fetch-forks | tee rollup.log - name: Summary run: | diff --git a/tools/pick_work.py b/tools/pick_work.py index 6c1c87f0d3..e4185ce043 100644 --- a/tools/pick_work.py +++ b/tools/pick_work.py @@ -7,10 +7,16 @@ The rollup is fetched from the project's latest-sarif release by default; pass --tsv with a local path (or another URL) to override it. +Every pick is reported with the URL of the place its best match lives, derived +from the rollup's best_source (``main`` or a remote-tracking branch such as +``owner-OpenSHC/reimpl/foo``) and the local git remotes. With --all-sources the +local reimpl/ branches are scanned as well, so each pick lists every fork +branch that has the function and the URL of the file on that branch. + Usage: python pick_worst.py [-n N] [--tsv PATH_OR_URL] [--namespace NS] [--include-done] [--max-pct P] [--spread] - [--format table|tsv|json|addresses] + [--all-sources] [--format table|tsv|json|addresses|urls] """ import argparse @@ -18,6 +24,8 @@ import io import json import os +import re +import subprocess import sys import urllib.error import urllib.request @@ -87,10 +95,167 @@ def load(source): "cpp": cpp, "namespace": namespace_of(cpp), "function": function_of(cpp), + "url": "", + "sources": [], }) return rows +DEFAULT_REPO_URL = "https://github.com/sourcehold/OpenSHC" +#: one entry per line: SHC__0x | % | +ENTRY = re.compile(r"^\s*(SHC_[0-9A-Fa-f]+_0x[0-9A-Fa-f]+)\s*\|\s*([0-9.]+)%\s*\|\s*(.*?)\s*$") +STATUS_GLOB = re.compile(r"^status/addresses-SHC-[0-9A-Fa-f]+\.txt$") + + +def git(args): + """-> (returncode, stdout); never raises, so a missing repo degrades gracefully.""" + try: + p = subprocess.run(["git"] + args, capture_output=True, text=True, + errors="replace") + except OSError: + return 1, "" + return p.returncode, p.stdout + + +def web_base(url): + """git remote url -> browsable https base, or '' if it is not a known host.""" + url = url.strip() + if url.startswith("git@"): + host, _, path = url[4:].partition(":") + url = "https://{}/{}".format(host, path) + elif url.startswith("ssh://git@"): + url = "https://" + url[len("ssh://git@"):] + if not url.startswith(("http://", "https://")): + return "" + return url[:-4].rstrip("/") if url.endswith(".git") else url.rstrip("/") + + +def remote_bases(): + """-> {remote name: browsable https base} for the local remotes.""" + rc, out = git(["remote", "-v"]) + bases = {} + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0] not in bases: + base = web_base(parts[1]) + if base: + bases[parts[0]] = base + return bases + + +def source_url(source, cpp, bases): + """'owner-OpenSHC/reimpl/foo' + a cpp path -> the URL of that file on GitHub. + + 'main' (the rollup's name for HEAD) is the project's own repository, + sourcehold/OpenSHC. Remote names never contain a slash, so the + first component of a remote-tracking branch is the remote. + """ + if not source or source == "main": + # the rollup's 'main' is the project's own main, not whatever this + # checkout's origin happens to be + base, branch = DEFAULT_REPO_URL, "main" + else: + remote, _, branch = source.partition("/") + if not branch: + return "" + base = bases.get(remote) + if base is None: + # not fetched locally: rebuild owner/name from the remote name, which + # fetch_fork_branches made by replacing the slash with a dash + owner, _, name = remote.partition("-") + if not name: + return "" + base = "https://github.com/{}/{}".format(owner, name) + if cpp: + return "{}/blob/{}/{}".format(base, branch, cpp.replace("\\", "/")) + return "{}/tree/{}".format(base, branch) + + +def reimpl_refs(filter_re): + """-> ['HEAD'] + the remote-tracking branches whose name matches the filter.""" + rc, out = git(["branch", "-r", "--format=%(refname:short)"]) + refs = [b.strip() for b in out.splitlines() + if b.strip() and "->" not in b and filter_re.search(b)] + return ["HEAD"] + sorted(refs) + + +def ref_rows(ref, wanted): + """-> {shc id: (pct, status)} for the wanted ids in one ref's status files.""" + rc, out = git(["ls-tree", "-r", "--name-only", ref, "--", "status"]) + rows = {} + for path in out.splitlines(): + path = path.strip() + if not STATUS_GLOB.match(path): + continue + rc, text = git(["show", "{}:{}".format(ref, path)]) + if rc != 0: + continue + for line in text.splitlines(): + m = ENTRY.match(line) + if m and m.group(1) in wanted: + rows[m.group(1)] = (float(m.group(2)), m.group(3).strip()) + return rows + + +def collect_sources(picks, pattern="/reimpl/", min_pct=None, progress=False, + keep_duplicates=False): + """Fill each pick's 'sources' with every ref that reports the function. + + Branch contents are taken as truth exactly as status_rollup.py does: nothing + is re-measured, only the per-address status files are read. + + Most reimpl/ branches are cut from one another, so the same file turns up + verbatim on dozens of them. Sources are therefore grouped by the blob id of + the cpp on that branch: one entry per distinct version of the function, + carrying the other branches that hold the identical file in 'also_on'. + """ + wanted = {r["shc_id"]: r for r in picks} + bases = remote_bases() + refs = reimpl_refs(re.compile(pattern)) + for i, ref in enumerate(refs, 1): + if progress: + print("scanning {}/{}: {}".format(i, len(refs), ref), + end="\r", file=sys.stderr) + for sid, (pct, status) in ref_rows(ref, wanted).items(): + # every branch carries a status line for every address, so a branch + # that has not touched the function reports 0% and has no file + if pct <= (0.0 if min_pct is None else min_pct - 1e-9): + continue + row = wanted[sid] + source = "main" if ref == "HEAD" else ref + rc, blob = git(["rev-parse", "--verify", "--quiet", + "{}:{}".format(ref, row["cpp"])]) + row["sources"].append({ + "source": source, + "pct": pct, + "status": status, + "url": source_url(source, row["cpp"], bases), + "blob": blob.strip() if rc == 0 else "", + "also_on": [], + }) + if progress: + print(" " * 60, end="\r", file=sys.stderr) + for row in picks: + row["sources"].sort(key=lambda s: (-s["pct"], s["source"])) + if not keep_duplicates: + row["sources"] = dedupe_sources(row["sources"]) + return picks + + +def dedupe_sources(sources): + """Collapse sources whose cpp is byte-identical into one entry each.""" + out, by_blob = [], {} + for src in sources: + first = by_blob.get(src["blob"]) if src["blob"] else None + if first is None: + if src["blob"]: + by_blob[src["blob"]] = src + out.append(src) + else: + first["also_on"].append(src["source"]) + return out + + #: namespaces that hold generated / runtime glue rather than game logic SKIPPED_NAMESPACES = ("OS", "Global::Init") @@ -150,8 +315,17 @@ def render(picks, fmt): return json.dumps(picks, indent=2) if fmt == "addresses": return "\n".join(r["address"] for r in picks) + if fmt == "urls": + out = [] + for r in picks: + for src in r["sources"] or [{"source": r["source"], "pct": r["pct"], + "url": r["url"], "also_on": []}]: + out.append("{}\t{:.1f}\t{}\t{}".format( + r["address"], src["pct"], src["source"], src["url"])) + return "\n".join(out) if fmt == "tsv": - cols = ["address", "pct", "namespace", "function", "refs", "status", "cpp"] + cols = ["address", "pct", "namespace", "function", "refs", "status", + "source", "url", "cpp"] out = ["\t".join(cols)] out += ["\t".join(str(r[c]) for c in cols) for r in picks] return "\n".join(out) @@ -166,6 +340,15 @@ def render(picks, fmt): r["pct"], r["address"], r["refs"], r["function"] or r["shc_id"])) if r["status"]: lines.append(" status: {}".format(r["status"])) + if r["sources"]: + for src in r["sources"]: + lines.append(" {:>6.1f}% {}".format( + src["pct"], src["url"] or src["source"])) + if src["also_on"]: + lines.append(" (identical on {} other " + "branch(es))".format(len(src["also_on"]))) + elif r["url"]: + lines.append(" {:>6.1f}% {}".format(r["pct"], r["url"])) counts = defaultdict(int) for r in picks: counts[r["namespace"]] += 1 @@ -201,7 +384,21 @@ def main(argv=None): help="only consider rows with at least this many refs_at_100") ap.add_argument("--spread", action="store_true", help="ignore namespace grouping; take the globally worst N") - ap.add_argument("--format", choices=["table", "tsv", "json", "addresses"], + ap.add_argument("--all-sources", action="store_true", + help="scan the local reimpl/ branches and list every fork " + "branch that has the function, with the URL of its file " + "there (slow: one pass over every remote-tracking branch)") + ap.add_argument("--sources-filter", default="/reimpl/", + help="regex selecting the remote-tracking branches to scan " + "with --all-sources (default '/reimpl/')") + ap.add_argument("--sources-min-pct", type=float, + help="with --all-sources, only list branches at or above " + "this match percentage (default: above 0%%, i.e. every " + "branch that has actually touched the function)") + ap.add_argument("--keep-duplicate-sources", action="store_true", + help="with --all-sources, list every branch separately " + "instead of collapsing branches whose file is identical") + ap.add_argument("--format", choices=["table", "tsv", "json", "addresses", "urls"], default="table") ap.add_argument("--list-namespaces", action="store_true", help="print a per-namespace badness summary instead of picks") @@ -245,7 +442,18 @@ def main(argv=None): min(x["pct"] for x in members), ns)) return 0 - print(render(select(pool, args.count, spread=args.spread), args.format)) + picks = select(pool, args.count, spread=args.spread) + + bases = remote_bases() + for row in picks: + row["url"] = source_url(row["source"], row["cpp"], bases) + if args.all_sources: + collect_sources(picks, pattern=args.sources_filter, + min_pct=args.sources_min_pct, + progress=sys.stderr.isatty(), + keep_duplicates=args.keep_duplicate_sources) + + print(render(picks, args.format)) return 0 diff --git a/tools/reimplementation-helper/todo/Fetch-ForkBranches.ps1 b/tools/reimplementation-helper/todo/Fetch-ForkBranches.ps1 new file mode 100644 index 0000000000..48bbd3c79f --- /dev/null +++ b/tools/reimplementation-helper/todo/Fetch-ForkBranches.ps1 @@ -0,0 +1,66 @@ +<# +.SYNOPSIS +Add every fork of the root repository as a remote and shallow-fetch its +reimpl/ branches, so that status_rollup.py sees them as ordinary +remote-tracking branches. + +.DESCRIPTION +Only branch tips are fetched: the rollup reads one file per ref, and nothing +from a fetched branch is ever executed. + +Requires: gh (authenticated via GH_TOKEN), git. Run from the repo root. + +The repository defaults to the project's own, sourcehold/OpenSHC; pass another +one to walk up from a fork instead. + +.EXAMPLE +tools\reimplementation-helper\todo\Fetch-ForkBranches.ps1 +#> +[CmdletBinding()] +param( + [Parameter(Position = 0)] + [string]$RepoFullName = 'sourcehold/OpenSHC' +) + +$ErrorActionPreference = 'Stop' + +# Walk up to the root repository, then enumerate its forks. +$root = gh repo view $RepoFullName --json parent,nameWithOwner ` + --jq 'if .parent then .parent.owner.login + "/" + .parent.name else .nameWithOwner end' +if ($LASTEXITCODE -ne 0) { throw "gh repo view failed for $RepoFullName" } +$root = $root.Trim() +Write-Host "Root repository: $root" + +$forks = gh api "repos/$root/forks" --paginate --jq '.[].full_name' +if ($LASTEXITCODE -ne 0) { throw "gh api repos/$root/forks failed" } + +$repos = @($root) + @($forks) | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } | + Sort-Object -Unique +Write-Host "Repositories to scan: $($repos.Count)" + +foreach ($repo in $repos) { + # Ignore anything that is not a plain owner/name, so that a surprising API + # response cannot turn into command arguments or a path. + if ($repo -notmatch '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$') { + Write-Host " skipped (unexpected name): $repo" + continue + } + $remote = $repo -replace '/', '-' + git remote remove $remote 2>$null | Out-Null + git remote add $remote "https://github.com/$repo" + + # A fork with no reimpl/ branches is normal, not an error. + git fetch --no-tags --depth=1 $remote "+refs/heads/reimpl/*:refs/remotes/$remote/reimpl/*" 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { + $n = @(git branch -r --list "$remote/reimpl/*").Count + if ($n -gt 0) { Write-Host " ${repo}: $n reimpl branch(es)" } + } + else { + Write-Host " ${repo}: skipped (unreachable or no reimpl/ branches)" + } +} + +$total = @(git branch -r | Where-Object { $_ -match '/reimpl/' }).Count +Write-Host "Total reimpl/ branches: $total" diff --git a/tools/reimplementation-helper/todo/fetch_fork_branches.sh b/tools/reimplementation-helper/todo/fetch_fork_branches.sh index ac76b28f7b..08d4fca1b5 100644 --- a/tools/reimplementation-helper/todo/fetch_fork_branches.sh +++ b/tools/reimplementation-helper/todo/fetch_fork_branches.sh @@ -7,11 +7,14 @@ # from a fetched branch is ever executed. # # Requires: gh (authenticated via GH_TOKEN), git. Run from the repo root. -# Usage: tools/reimplementation-helper/todo/fetch_fork_branches.sh +# Usage: tools/reimplementation-helper/todo/fetch_fork_branches.sh [owner/repo] +# +# The repository defaults to the project's own; pass another one to walk up from +# a fork instead. set -euo pipefail -repo_full_name="${1:?usage: fetch_fork_branches.sh }" +repo_full_name="${1:-sourcehold/OpenSHC}" # Walk up to the root repository, then enumerate its forks. root=$(gh repo view "$repo_full_name" --json parent,nameWithOwner \ diff --git a/tools/reimplementation-helper/todo/status_regression.py b/tools/reimplementation-helper/todo/status_regression.py index 5b4eb82c7b..560998b67f 100644 --- a/tools/reimplementation-helper/todo/status_regression.py +++ b/tools/reimplementation-helper/todo/status_regression.py @@ -30,6 +30,13 @@ PR that proposes it. A merge group may hold several PRs, hence the list. REIMPL_FILTER regex of remote-tracking branches to include (default '/reimpl/') + REIMPL_FETCH_FORKS '1' to add every fork as a remote and shallow-fetch its + reimpl/ branches first, so the baseline covers every fork + without a separate fetch step (needs gh, authenticated). The + fetch happens only after the touched-file list is settled, + because it grafts the commit under test as parentless. + REIMPL_REPO repository whose forks are enumerated, as owner/name (default: + the origin remote, else sourcehold/OpenSHC) TOUCHED_LIST file of one changed path per line; when set it is used instead of asking git, so the list can be worked out before anything shallow is fetched into the repository @@ -42,7 +49,8 @@ import os, sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from status_rollup import rows_at, header_cpp_map, reimpl_refs, best_per_id, run, parse_resolvers +from status_rollup import (rows_at, header_cpp_map, reimpl_refs, best_per_id, run, + parse_resolvers, fetch_fork_branches) # status files carry one decimal, so compare with a tolerance below that EPS = 0.05 @@ -110,18 +118,14 @@ def main(): return 0 base_rows = rows_at(base_ref) - # The PR's own branch may also be one of the fetched fork reimpl/ branches. - exclude = set(os.environ.get('PR_HEAD_SHA', '').replace(',', ' ').split()) - refs = reimpl_refs(exclude_shas=exclude) - best, at100, seen = best_per_id(refs) - print('baseline refs with a status file: %d' % len(seen)) - - cpp_of = header_cpp_map() - # Which .cpp files does the change actually touch? A status line can differ # for reasons that are not this change's doing -- most often a branch that # has not been updated since main improved the same function -- so an entry # only counts when the change also edits the file behind it. + # + # This is answered BEFORE any fork branch is fetched: a shallow fetch grafts + # the commit under test as parentless whenever the PR comes from one of those + # branches, and the merge base is then unanswerable. touched, mb = touched_files(base_ref, pr_ref) if touched is None: # Not something to work around: without the file list this would compare @@ -134,6 +138,17 @@ def main(): ) return 1 + if os.environ.get('REIMPL_FETCH_FORKS') == '1': + fetch_fork_branches(os.environ.get('REIMPL_REPO')) + + # The PR's own branch may also be one of the fetched fork reimpl/ branches. + exclude = set(os.environ.get('PR_HEAD_SHA', '').replace(',', ' ').split()) + refs = reimpl_refs(exclude_shas=exclude) + best, at100, seen = best_per_id(refs) + print('baseline refs with a status file: %d' % len(seen)) + + cpp_of = header_cpp_map() + # A resolver the change itself adds or moves is not in the base headers, so # re-read the .func.hpp files it touches at the PR's own ref. cpp_of = dict(cpp_of) diff --git a/tools/reimplementation-helper/todo/status_rollup.py b/tools/reimplementation-helper/todo/status_rollup.py index 03221fd13e..4af2b217ac 100644 --- a/tools/reimplementation-helper/todo/status_rollup.py +++ b/tools/reimplementation-helper/todo/status_rollup.py @@ -7,6 +7,19 @@ Run from the repo root: python -I tools/reimplementation-helper/todo/status_rollup.py +Only branches that are present as remote-tracking refs can be read, so a plain +clone sees nothing but its own. Pass --fetch-forks (or set REIMPL_FETCH_FORKS=1) +to add every fork of the root repository as a remote first and shallow-fetch its +reimpl/ branches, which is what fetch_fork_branches.sh does in CI: + + python -I tools/reimplementation-helper/todo/status_rollup.py --fetch-forks + +That needs gh, authenticated (GH_TOKEN). Only branch tips are fetched, and +nothing from a fetched branch is ever executed -- one status file per ref is +read. The repository is taken from --repo, else REIMPL_REPO, else the origin +remote, else sourcehold/OpenSHC. Without the flag the fetch is skipped and only +what is already fetched is scanned, so the CI workflow keeps working unchanged. + Output: status_rollup.tsv in the current working directory. Set REIMPL_FILTER to a regex to widen or narrow which remote-tracking branches @@ -19,10 +32,12 @@ importable: everything that runs belongs in main(). """ -import os, re, subprocess +import argparse, os, re, subprocess from collections import Counter ROOT = 'src/OpenSHC' +#: the project's own repository; forks are enumerated from its root +DEFAULT_REPO = 'sourcehold/OpenSHC' STATUS_GLOB = re.compile(r'^status/addresses-SHC-[0-9A-Fa-f]+\.txt$') # one entry per line: SHC__0x | % | ENTRY = re.compile(r'^\s*(SHC_[0-9A-Fa-f]+_0x([0-9A-Fa-f]+))\s*\|\s*([0-9.]+)%\s*\|\s*(.*?)\s*$') @@ -102,6 +117,68 @@ def header_cpp_map(): return cpp_of +REPO_NAME = re.compile(r'^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$') + + +def origin_repo(): + """-> 'owner/name' of the origin remote, or '' if it is not a GitHub url.""" + rc, out = run(['git', 'remote', 'get-url', 'origin']) + url = out.strip() + if url.endswith('.git'): + url = url[:-4] + m = re.search(r'github\.com[:/]+([A-Za-z0-9._-]+/[A-Za-z0-9._-]+)/?$', url) + return m.group(1) if m else '' + + +def fork_repos(repo_full_name): + """-> [root repo, every fork of it]; the root is the repo's parent, if any.""" + rc, out = run(['gh', 'repo', 'view', repo_full_name, '--json', 'parent,nameWithOwner', + '--jq', 'if .parent then .parent.owner.login + "/" + .parent.name ' + 'else .nameWithOwner end']) + if rc != 0 or not out.strip(): + raise RuntimeError('gh repo view %s failed (is gh installed and authenticated?)' + % repo_full_name) + root = out.strip() + rc, out = run(['gh', 'api', 'repos/%s/forks' % root, '--paginate', + '--jq', '.[].full_name']) + if rc != 0: + raise RuntimeError('gh api repos/%s/forks failed' % root) + repos = {root} | {ln.strip() for ln in out.splitlines() if ln.strip()} + # ignore anything that is not a plain owner/name, so that a surprising API + # response cannot turn into command arguments or a path + return root, sorted(r for r in repos if REPO_NAME.match(r)) + + +def fetch_fork_branches(repo_full_name=None): + """Add every fork as a remote and shallow-fetch its reimpl/ branches. + + The Python counterpart of fetch_fork_branches.sh, so that a local run covers + the same branches CI does. Only branch tips are fetched. + """ + repo_full_name = (repo_full_name or os.environ.get('REIMPL_REPO') + or origin_repo() or DEFAULT_REPO) + root, repos = fork_repos(repo_full_name) + print('root repository: %s' % root) + print('repositories to scan: %d' % len(repos)) + fetched = 0 + for repo in repos: + remote = repo.replace('/', '-') + run(['git', 'remote', 'remove', remote]) + run(['git', 'remote', 'add', remote, 'https://github.com/%s' % repo]) + # a fork with no reimpl/ branches is normal, not an error + rc, out = run(['git', 'fetch', '--no-tags', '--depth=1', remote, + '+refs/heads/reimpl/*:refs/remotes/%s/reimpl/*' % remote]) + if rc != 0: + print(' %s: skipped (unreachable or no reimpl/ branches)' % repo) + continue + rc, out = run(['git', 'branch', '-r', '--list', '%s/reimpl/*' % remote]) + n = len([b for b in out.splitlines() if b.strip()]) + if n: + fetched += n + print(' %s: %d reimpl branch(es)' % (repo, n)) + print('fetched reimpl/ branches: %d' % fetched) + + def reimpl_refs(filter_re=None, exclude_shas=()): """-> ['HEAD', ] (sorted).""" if filter_re is None: @@ -141,7 +218,55 @@ def best_per_id(refs): return best, at100, seen -def main(): +def remote_repos(): + """-> {remote name: 'owner/name'} from the remotes' urls. + + The url is the only reliable source: a remote added by hand is not named + after the repository, and an owner or a name may itself contain a dash, so + splitting the 'owner-name' remote fetch_fork_branches creates is ambiguous. + """ + rc, out = run(['git', 'remote', '-v']) + repos = {} + for line in out.splitlines(): + parts = line.split() + if len(parts) < 2 or parts[0] in repos: + continue + url = parts[1][:-4] if parts[1].endswith('.git') else parts[1] + m = re.search(r'github\.com[:/]+([A-Za-z0-9._-]+/[A-Za-z0-9._-]+)/?$', url) + if m: + repos[parts[0]] = m.group(1) + return repos + + +def fork_of(ref, repos=None): + """'owner-name/reimpl/foo' -> 'owner/name'; the remote name has no slash.""" + if ref == 'HEAD': + return 'main' + remote = ref.split('/', 1)[0] + if repos is None: + repos = remote_repos() + if remote in repos: + return repos[remote] + owner, _, name = remote.partition('-') + return '%s/%s' % (owner, name) if name else remote + + +def main(argv=None): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--fetch-forks', action='store_true', + default=os.environ.get('REIMPL_FETCH_FORKS') == '1', + help='add every fork as a remote and shallow-fetch its reimpl/ ' + 'branches before rolling up (needs gh; REIMPL_FETCH_FORKS=1)') + ap.add_argument('--repo', default=os.environ.get('REIMPL_REPO'), + help="repository to enumerate the forks of, as owner/name " + "(default: REIMPL_REPO, else the origin remote, else " + + DEFAULT_REPO + ")") + args = ap.parse_args(argv) + + if args.fetch_forks: + fetch_fork_branches(args.repo) + # data files are written to the current working directory (run from the repo root) cpp_of = header_cpp_map() print('resolvers in main headers:', len(cpp_of)) @@ -149,6 +274,17 @@ def main(): refs = reimpl_refs() best, at100, seen = best_per_id(refs) print('refs scanned: %d (%d with a status file)' % (len(refs), len(seen))) + repos = remote_repos() + # GitHub repository names are case-insensitive, and the same fork is often + # reachable under two remotes, so count one entry per repository + forks = Counter(fork_of(r, repos).lower() for r in refs) + print('forks covered: %d' % len([f for f in forks if f != 'main'])) + for fork, n in sorted(forks.items(), key=lambda kv: (-kv[1], kv[0])): + if fork != 'main': + print(' %-40s %d branch(es)' % (fork, n)) + if len(forks) <= 1 and not args.fetch_forks: + print('NOTE: no fork branches are fetched; rerun with --fetch-forks to ' + 'include every fork of the repository') missing = [r for r in refs if r not in seen] if missing: print('refs with NO status file:', len(missing))