Mirror pkg:git tags and submodules

docs/architecture.md

@@ -363,7 +363,7 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g
 
 ### `internal/mirror`
 
-Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and commit in the git cache that `/git` serves.
+Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and the commit or tag, with its submodules, in the git cache that `/git` serves.
 
 The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints.
 

docs/configuration.md

@@ -1910,16 +1910,18 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403
 
 ### Git repositories
 
-`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID:
+`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID or a tag:
 
 ```bash
 proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567
+proxy mirror pkg:git/github.com/git-pkgs/purl@v0.2.1
 proxy mirror pkg:git/github.com/git-pkgs/purl
 ```
 
 - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail.
 - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails.
-- Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID.
+- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name as the version fails too; give the commit ID.
+- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log.
 - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead.
 
 ### Scheduled refresh

internal/gitcache/mirror.go

@@ -4,27 +4,44 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os/exec"
+	"slices"
 	"strings"
 	"time"
 )
 
 // MirrorResult is what Mirror did.
 type MirrorResult struct {
-	// Cached reports that the repository, and the commit if one was given,
-	// were cached already.
+	// Cached reports that the repository, and the commit if a version was
+	// given, were cached already.
 	Cached bool
 	// Bytes is how much the cached repository grew.
 	Bytes int64
+	// Commit is the commit the version names, empty without a version.
+	Commit string
+	// Submodules are the submodules Commit records, sorted by path.
+	Submodules []Submodule
+}
+
+// Submodule is a submodule a commit records: its path, the URL .gitmodules
+// gives for it, which may be relative to the repository's, and the commit
+// the gitlink pins.
+type Submodule struct {
+	Path, URL, Commit string
 }
 
 // Mirror brings the repository cached under key up to date with repoURL,
-// cloning it if needed, and waits for that to finish. With commit, an object
-// ID, it also makes sure the commit is cached. One on no branch or tag is
-// fetched by ID when fetchWants allows it (see OpenOptions.FetchWants), and
-// is kept as long as a commit a client fetched by ID.
-func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWants bool) (MirrorResult, error) {
-	if commit != "" && !isObjectID(commit) {
-		return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit)
+// cloning it if needed, and waits for that to finish. version, if given, is
+// a full commit ID or a tag. A commit is made sure to be cached: one on no
+// branch or tag is fetched by ID when fetchWants allows it (see
+// OpenOptions.FetchWants), and is kept as long as a commit a client fetched
+// by ID. A tag is cached with the repository's tags.
+func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) {
+	tag := version != "" && !isObjectID(version)
+	if tag {
+		if err := c.checkTagName(ctx, version); err != nil {
+			return MirrorResult{}, err
+		}
 	}
 	lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants})
 	if err != nil {
@@ -53,17 +70,108 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa
 	if errors.Is(refreshErr, ErrTooLarge) {
 		return MirrorResult{}, refreshErr
 	}
-	if commit != "" {
+	commit := version
+	switch {
+	case tag:
+		if commit, err = c.resolveTag(ctx, rp.dir, version); err != nil {
+			if refreshErr != nil {
+				// The tag may be one the failed refresh would have brought.
+				return MirrorResult{}, refreshErr
+			}
+			return MirrorResult{}, err
+		}
+	case version != "":
 		if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil {
 			return MirrorResult{}, err
 		}
 	}
 	// A refresh that failed leaves the commit cached all the same.
-	if refreshErr != nil && commit == "" {
+	if refreshErr != nil && version == "" {
 		return MirrorResult{}, refreshErr
 	}
+	var submodules []Submodule
+	if commit != "" {
+		if submodules, err = c.submodules(ctx, rp.dir, commit); err != nil {
+			return MirrorResult{}, err
+		}
+	}
 	grown := max(rp.size.Load()-before, 0)
-	return MirrorResult{Cached: cached && grown == 0, Bytes: grown}, nil
+	return MirrorResult{Cached: cached && grown == 0, Bytes: grown, Commit: commit, Submodules: submodules}, nil
+}
+
+// checkTagName refuses a version that is no valid tag name, such as one
+// with .. or a revision suffix like ^{tree}.
+func (c *Cache) checkTagName(ctx context.Context, name string) error {
+	if err := c.command(ctx, nil, "check-ref-format", "refs/tags/"+name).Run(); err != nil {
+		return fmt.Errorf("%q is neither a full commit ID nor a tag name", name)
+	}
+	return nil
+}
+
+// resolveTag returns the commit tag points at.
+func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) {
+	out, err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", "refs/tags/"+tag+"^{commit}").Output()
+	var exit *exec.ExitError
+	if errors.As(err, &exit) && exit.ExitCode() == 1 {
+		return "", fmt.Errorf("tag %s: %w", tag, ErrNotFound)
+	}
+	if err != nil {
+		return "", fmt.Errorf("resolving tag %s: %w", tag, err)
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+// submodules lists the submodules commit records: its gitlinks that
+// .gitmodules names a URL for.
+func (c *Cache) submodules(ctx context.Context, dir, commit string) ([]Submodule, error) {
+	if c.command(ctx, nil, "-C", dir, "cat-file", "-e", commit+":.gitmodules").Run() != nil {
+		return nil, nil
+	}
+	// --list rather than --get-regexp, which fails when nothing matches.
+	out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list")
+	if err != nil {
+		return nil, fmt.Errorf("reading .gitmodules: %w", err)
+	}
+	paths, urls := map[string]string{}, map[string]string{}
+	for entry := range strings.SplitSeq(out, "\x00") {
+		key, value, ok := strings.Cut(entry, "\n")
+		section, ok2 := strings.CutPrefix(key, "submodule.")
+		name, field, dotted := cutLast(section, ".")
+		if !ok || !ok2 || !dotted {
+			continue
+		}
+		switch field {
+		case "path":
+			paths[value] = name
+		case "url":
+			urls[name] = value
+		}
+	}
+	tree, err := c.run(ctx, dir, "ls-tree", "-r", "-z", "--full-tree", commit)
+	if err != nil {
+		return nil, err
+	}
+	var submodules []Submodule
+	for entry := range strings.SplitSeq(tree, "\x00") {
+		meta, path, ok := strings.Cut(entry, "\t")
+		fields := strings.Fields(meta)
+		if !ok || len(fields) != 3 || fields[0] != "160000" {
+			continue
+		}
+		if url := urls[paths[path]]; url != "" && isObjectID(fields[2]) {
+			submodules = append(submodules, Submodule{Path: path, URL: url, Commit: fields[2]})
+		}
+	}
+	slices.SortFunc(submodules, func(a, b Submodule) int { return strings.Compare(a.Path, b.Path) })
+	return submodules, nil
+}
+
+func cutLast(s, sep string) (before, after string, found bool) {
+	i := strings.LastIndex(s, sep)
+	if i < 0 {
+		return s, "", false
+	}
+	return s[:i], s[i+len(sep):], true
 }
 
 // ensureCommit makes sure the repository has commit, fetching it by ID when

internal/gitcache/mirror_test.go

@@ -212,3 +212,106 @@ func TestMirrorReportsRefreshTooLarge(t *testing.T) {
 		t.Errorf("mirror with a refresh past max_repo_size = %v, want ErrTooLarge", err)
 	}
 }
+
+func TestMirrorResolvesTag(t *testing.T) {
+	upstream := newFakeUpstream(t)
+	work := upstream.newRepo("repo")
+	tagged := runGit(t, work, "rev-parse", "v1^{commit}")
+	tc := newTestCache(t, upstream.server.URL, nil)
+
+	got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true)
+	if err != nil {
+		t.Fatalf("Mirror: %v", err)
+	}
+	if got.Commit != tagged {
+		t.Errorf("commit %q, want %s the tag points at", got.Commit, tagged)
+	}
+}
+
+func TestMirrorReportsMissingTag(t *testing.T) {
+	upstream := newFakeUpstream(t)
+	upstream.newRepo("repo")
+	tc := newTestCache(t, upstream.server.URL, nil)
+
+	for _, version := range []string{"v9", "main"} {
+		_, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true)
+		if !errors.Is(err, ErrNotFound) {
+			t.Errorf("mirror at %q = %v, want ErrNotFound", version, err)
+		}
+	}
+}
+
+func TestMirrorRefusesInvalidTag(t *testing.T) {
+	upstream := newFakeUpstream(t)
+	upstream.newRepo("repo")
+	tc := newTestCache(t, upstream.server.URL, nil)
+
+	for _, version := range []string{"a..b", "v1^{tree}", "v1~1", "v 1", "v1:x"} {
+		_, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true)
+		if err == nil || errors.Is(err, ErrNotFound) {
+			t.Errorf("mirror at %q = %v, want an invalid version error", version, err)
+		}
+	}
+}
+
+func TestMirrorListsSubmodules(t *testing.T) {
+	upstream := newFakeUpstream(t)
+	libWork := upstream.newRepo("lib")
+	libCommit := runGit(t, libWork, "rev-parse", "HEAD")
+	work := upstream.newRepo("repo")
+	gitmodules := "[submodule \"vendor/lib\"]\n\tpath = vendor/lib\n\turl = ../lib.git\n" +
+		"[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n"
+	if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	otherCommit := strings.Repeat("ab", 20)
+	runGit(t, work, "add", ".gitmodules")
+	runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib")
+	runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other")
+	runGit(t, work, "commit", "--quiet", "-m", "submodules")
+	runGit(t, work, "push", "--quiet", "origin", "main")
+	commit := runGit(t, work, "rev-parse", "HEAD")
+	tc := newTestCache(t, upstream.server.URL, nil)
+
+	got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true)
+	if err != nil {
+		t.Fatalf("Mirror: %v", err)
+	}
+	want := []Submodule{
+		{Path: "third_party/other", URL: "https://example.com/other.git", Commit: otherCommit},
+		{Path: "vendor/lib", URL: "../lib.git", Commit: libCommit},
+	}
+	if !slices.Equal(got.Submodules, want) {
+		t.Errorf("submodules %+v, want %+v", got.Submodules, want)
+	}
+
+	// Without a version, nothing names a commit to read submodules from.
+	got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true)
+	if err != nil || len(got.Submodules) != 0 {
+		t.Errorf("unversioned mirror = %+v, %v, want no submodules", got, err)
+	}
+}
+
+// TestMirrorWithEmptyGitmodules checks a .gitmodules with no path or url,
+// for which git config --get-regexp finds nothing, lists no submodules.
+func TestMirrorWithEmptyGitmodules(t *testing.T) {
+	upstream := newFakeUpstream(t)
+	work := upstream.newRepo("repo")
+	for i, content := range []string{"", "[submodule \"x\"]\n\tbranch = main\n"} {
+		if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(content), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		runGit(t, work, "add", ".gitmodules")
+		runGit(t, work, "commit", "--quiet", "--allow-empty", "-m", "gitmodules "+strconv.Itoa(i))
+	}
+	runGit(t, work, "push", "--quiet", "origin", "main")
+	tc := newTestCache(t, upstream.server.URL, nil)
+
+	for _, rev := range []string{"HEAD~1", "HEAD"} {
+		commit := runGit(t, work, "rev-parse", rev)
+		got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true)
+		if err != nil || len(got.Submodules) != 0 {
+			t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err)
+		}
+	}
+}