diff --git a/docs/architecture.md b/docs/architecture.md index 655b3098..80120ee9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -363,7 +363,7 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g ### `internal/mirror` -Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and commit in the git cache that `/git` serves. +Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and the commit or tag, with its submodules, in the git cache that `/git` serves. The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. diff --git a/docs/configuration.md b/docs/configuration.md index 584896bc..1c015890 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1910,16 +1910,18 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403 ### Git repositories -`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID: +`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID or a tag: ```bash proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 +proxy mirror pkg:git/github.com/git-pkgs/purl@v0.2.1 proxy mirror pkg:git/github.com/git-pkgs/purl ``` - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. -- Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. +- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name as the version fails too; give the commit ID. +- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. ### Scheduled refresh diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index b5bde40f..5aaa6fba 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -4,27 +4,44 @@ import ( "context" "errors" "fmt" + "os/exec" + "slices" "strings" "time" ) // MirrorResult is what Mirror did. type MirrorResult struct { - // Cached reports that the repository, and the commit if one was given, - // were cached already. + // Cached reports that the repository, and the commit if a version was + // given, were cached already. Cached bool // Bytes is how much the cached repository grew. Bytes int64 + // Commit is the commit the version names, empty without a version. + Commit string + // Submodules are the submodules Commit records, sorted by path. + Submodules []Submodule +} + +// Submodule is a submodule a commit records: its path, the URL .gitmodules +// gives for it, which may be relative to the repository's, and the commit +// the gitlink pins. +type Submodule struct { + Path, URL, Commit string } // Mirror brings the repository cached under key up to date with repoURL, -// cloning it if needed, and waits for that to finish. With commit, an object -// ID, it also makes sure the commit is cached. One on no branch or tag is -// fetched by ID when fetchWants allows it (see OpenOptions.FetchWants), and -// is kept as long as a commit a client fetched by ID. -func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWants bool) (MirrorResult, error) { - if commit != "" && !isObjectID(commit) { - return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit) +// cloning it if needed, and waits for that to finish. version, if given, is +// a full commit ID or a tag. A commit is made sure to be cached: one on no +// branch or tag is fetched by ID when fetchWants allows it (see +// OpenOptions.FetchWants), and is kept as long as a commit a client fetched +// by ID. A tag is cached with the repository's tags. +func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) { + tag := version != "" && !isObjectID(version) + if tag { + if err := c.checkTagName(ctx, version); err != nil { + return MirrorResult{}, err + } } lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants}) if err != nil { @@ -53,17 +70,108 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa if errors.Is(refreshErr, ErrTooLarge) { return MirrorResult{}, refreshErr } - if commit != "" { + commit := version + switch { + case tag: + if commit, err = c.resolveTag(ctx, rp.dir, version); err != nil { + if refreshErr != nil { + // The tag may be one the failed refresh would have brought. + return MirrorResult{}, refreshErr + } + return MirrorResult{}, err + } + case version != "": if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { return MirrorResult{}, err } } // A refresh that failed leaves the commit cached all the same. - if refreshErr != nil && commit == "" { + if refreshErr != nil && version == "" { return MirrorResult{}, refreshErr } + var submodules []Submodule + if commit != "" { + if submodules, err = c.submodules(ctx, rp.dir, commit); err != nil { + return MirrorResult{}, err + } + } grown := max(rp.size.Load()-before, 0) - return MirrorResult{Cached: cached && grown == 0, Bytes: grown}, nil + return MirrorResult{Cached: cached && grown == 0, Bytes: grown, Commit: commit, Submodules: submodules}, nil +} + +// checkTagName refuses a version that is no valid tag name, such as one +// with .. or a revision suffix like ^{tree}. +func (c *Cache) checkTagName(ctx context.Context, name string) error { + if err := c.command(ctx, nil, "check-ref-format", "refs/tags/"+name).Run(); err != nil { + return fmt.Errorf("%q is neither a full commit ID nor a tag name", name) + } + return nil +} + +// resolveTag returns the commit tag points at. +func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) { + out, err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", "refs/tags/"+tag+"^{commit}").Output() + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 1 { + return "", fmt.Errorf("tag %s: %w", tag, ErrNotFound) + } + if err != nil { + return "", fmt.Errorf("resolving tag %s: %w", tag, err) + } + return strings.TrimSpace(string(out)), nil +} + +// submodules lists the submodules commit records: its gitlinks that +// .gitmodules names a URL for. +func (c *Cache) submodules(ctx context.Context, dir, commit string) ([]Submodule, error) { + if c.command(ctx, nil, "-C", dir, "cat-file", "-e", commit+":.gitmodules").Run() != nil { + return nil, nil + } + // --list rather than --get-regexp, which fails when nothing matches. + out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list") + if err != nil { + return nil, fmt.Errorf("reading .gitmodules: %w", err) + } + paths, urls := map[string]string{}, map[string]string{} + for entry := range strings.SplitSeq(out, "\x00") { + key, value, ok := strings.Cut(entry, "\n") + section, ok2 := strings.CutPrefix(key, "submodule.") + name, field, dotted := cutLast(section, ".") + if !ok || !ok2 || !dotted { + continue + } + switch field { + case "path": + paths[value] = name + case "url": + urls[name] = value + } + } + tree, err := c.run(ctx, dir, "ls-tree", "-r", "-z", "--full-tree", commit) + if err != nil { + return nil, err + } + var submodules []Submodule + for entry := range strings.SplitSeq(tree, "\x00") { + meta, path, ok := strings.Cut(entry, "\t") + fields := strings.Fields(meta) + if !ok || len(fields) != 3 || fields[0] != "160000" { + continue + } + if url := urls[paths[path]]; url != "" && isObjectID(fields[2]) { + submodules = append(submodules, Submodule{Path: path, URL: url, Commit: fields[2]}) + } + } + slices.SortFunc(submodules, func(a, b Submodule) int { return strings.Compare(a.Path, b.Path) }) + return submodules, nil +} + +func cutLast(s, sep string) (before, after string, found bool) { + i := strings.LastIndex(s, sep) + if i < 0 { + return s, "", false + } + return s[:i], s[i+len(sep):], true } // ensureCommit makes sure the repository has commit, fetching it by ID when diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 86f652f9..1f85a6cc 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -212,3 +212,106 @@ func TestMirrorReportsRefreshTooLarge(t *testing.T) { t.Errorf("mirror with a refresh past max_repo_size = %v, want ErrTooLarge", err) } } + +func TestMirrorResolvesTag(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tagged := runGit(t, work, "rev-parse", "v1^{commit}") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Commit != tagged { + t.Errorf("commit %q, want %s the tag points at", got.Commit, tagged) + } +} + +func TestMirrorReportsMissingTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, version := range []string{"v9", "main"} { + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror at %q = %v, want ErrNotFound", version, err) + } + } +} + +func TestMirrorRefusesInvalidTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, version := range []string{"a..b", "v1^{tree}", "v1~1", "v 1", "v1:x"} { + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) + if err == nil || errors.Is(err, ErrNotFound) { + t.Errorf("mirror at %q = %v, want an invalid version error", version, err) + } + } +} + +func TestMirrorListsSubmodules(t *testing.T) { + upstream := newFakeUpstream(t) + libWork := upstream.newRepo("lib") + libCommit := runGit(t, libWork, "rev-parse", "HEAD") + work := upstream.newRepo("repo") + gitmodules := "[submodule \"vendor/lib\"]\n\tpath = vendor/lib\n\turl = ../lib.git\n" + + "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { + t.Fatal(err) + } + otherCommit := strings.Repeat("ab", 20) + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") + runGit(t, work, "commit", "--quiet", "-m", "submodules") + runGit(t, work, "push", "--quiet", "origin", "main") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + want := []Submodule{ + {Path: "third_party/other", URL: "https://example.com/other.git", Commit: otherCommit}, + {Path: "vendor/lib", URL: "../lib.git", Commit: libCommit}, + } + if !slices.Equal(got.Submodules, want) { + t.Errorf("submodules %+v, want %+v", got.Submodules, want) + } + + // Without a version, nothing names a commit to read submodules from. + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("unversioned mirror = %+v, %v, want no submodules", got, err) + } +} + +// TestMirrorWithEmptyGitmodules checks a .gitmodules with no path or url, +// for which git config --get-regexp finds nothing, lists no submodules. +func TestMirrorWithEmptyGitmodules(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + for i, content := range []string{"", "[submodule \"x\"]\n\tbranch = main\n"} { + if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "--allow-empty", "-m", "gitmodules "+strconv.Itoa(i)) + } + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, rev := range []string{"HEAD~1", "HEAD"} { + commit := runGit(t, work, "rev-parse", rev) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err) + } + } +} diff --git a/internal/handler/git_mirror.go b/internal/handler/git_mirror.go index 709df313..38be233c 100644 --- a/internal/handler/git_mirror.go +++ b/internal/handler/git_mirror.go @@ -3,22 +3,114 @@ package handler import ( "context" "fmt" + "log/slog" "net/url" + "path" "strings" "github.com/git-pkgs/proxy/internal/gitcache" ) +// maxSubmoduleDepth bounds how deeply Mirror follows submodules of +// submodules. +const maxSubmoduleDepth = 8 + // Mirror caches the repository at location, a host and path such as // github.com/acme/widgets as a pkg:git PURL names it, from the upstream.git -// host serving it, and with commit, a full commit ID, that commit too. -// Clones through /git are then served from that copy. -func (h *GitHandler) Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) { +// host serving it. With version, a full commit ID or a tag, it caches that +// commit too, and the commits its submodules pin, from the upstream.git +// hosts serving them. A submodule no upstream.git host serves is skipped. +// Clones through /git are then served from those copies. +func (h *GitHandler) Mirror(ctx context.Context, location, version string) (gitcache.MirrorResult, error) { + return h.mirror(ctx, location, version, map[string]bool{}, 0) +} + +func (h *GitHandler) mirror(ctx context.Context, location, version string, seen map[string]bool, depth int) (gitcache.MirrorResult, error) { repo, err := h.mirrorRepository(location) if err != nil { return gitcache.MirrorResult{}, err } - return h.cache.Mirror(ctx, repo.key, repo.url, commit, repo.upstream.fetchWants) + result, err := h.cache.Mirror(ctx, repo.key, repo.url, version, repo.upstream.fetchWants) + if err != nil { + return result, err + } + for _, sub := range result.Submodules { + subLocation, err := gitSubmoduleLocation(repo.url, sub.URL) + if err == nil { + _, err = h.mirrorRepository(subLocation) + } + if err != nil { + h.logger().Warn("git mirror: skipping submodule", "repository", location, "submodule", sub.Path, "url", sub.URL, "error", err) + continue + } + if seen[subLocation+"@"+sub.Commit] { + continue + } + seen[subLocation+"@"+sub.Commit] = true + if depth >= maxSubmoduleDepth { + return result, fmt.Errorf("submodule %s: nested more than %d deep", sub.Path, maxSubmoduleDepth) + } + subResult, err := h.mirror(ctx, subLocation, sub.Commit, seen, depth+1) + if err != nil { + return result, fmt.Errorf("submodule %s: %w", sub.Path, err) + } + result.Cached = result.Cached && subResult.Cached + result.Bytes += subResult.Bytes + } + return result, nil +} + +func (h *GitHandler) logger() *slog.Logger { + if h.proxy != nil && h.proxy.Logger != nil { + return h.proxy.Logger + } + return slog.Default() +} + +// gitSubmoduleLocation turns the URL .gitmodules gives a submodule into a +// location as Mirror takes it. A relative URL is resolved against the +// superproject's repoURL, as git does; ssh, git and scp-like URLs name the +// host and path the repository has there. +func gitSubmoduleLocation(repoURL, submoduleURL string) (string, error) { + var host, repoPath string + switch { + case strings.HasPrefix(submoduleURL, "./"), strings.HasPrefix(submoduleURL, "../"): + u, err := url.Parse(repoURL) + if err != nil { + return "", err + } + host, repoPath = u.Host, path.Join(u.Path, submoduleURL) + case strings.Contains(submoduleURL, "://"): + u, err := url.Parse(submoduleURL) + if err != nil { + return "", err + } + switch u.Scheme { + case "http", "https": + host = u.Host + case "ssh", "git", "git+ssh", "ssh+git": + host = u.Hostname() + default: + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + repoPath = u.Path + default: + // scp-like: [user@]host:path + hostPart, p, ok := strings.Cut(submoduleURL, ":") + if !ok || strings.Contains(hostPart, "/") { + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + _, host, _ = strings.Cut(hostPart, "@") + if host == "" { + host = hostPart + } + repoPath = p + } + repoPath = strings.Trim(path.Clean("/"+repoPath), "/") + if host == "" || repoPath == "" { + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + return strings.ToLower(host) + "/" + repoPath, nil } // mirrorRepository resolves location to a repository of the upstream.git diff --git a/internal/handler/git_mirror_test.go b/internal/handler/git_mirror_test.go index 4ebcf982..227bd71e 100644 --- a/internal/handler/git_mirror_test.go +++ b/internal/handler/git_mirror_test.go @@ -2,6 +2,8 @@ package handler import ( "context" + "os" + "path/filepath" "strings" "testing" @@ -71,3 +73,82 @@ func TestGitMirrorCachesCommitForClones(t *testing.T) { t.Errorf("cloned HEAD %s, want %s", head, sha) } } + +func TestGitSubmoduleLocation(t *testing.T) { + tests := []struct { + repoURL, submoduleURL, want string + }{ + {"https://github.com/acme/widgets", "../lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets.git", "../../other/lib", "github.com/other/lib"}, + {"https://github.com/acme/widgets", "./vendored", "github.com/acme/widgets/vendored"}, + {"http://127.0.0.1:8080/team/repo", "../lib.git", "127.0.0.1:8080/team/lib.git"}, + {"https://github.com/acme/widgets", "https://GitLab.com/group/lib.git/", "gitlab.com/group/lib.git"}, + {"https://github.com/acme/widgets", "ssh://git@github.com:22/acme/lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets", "git://git.example/lib", "git.example/lib"}, + {"https://github.com/acme/widgets", "git@github.com:acme/lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets", "/srv/git/lib.git", ""}, + {"https://github.com/acme/widgets", "file:///srv/git/lib.git", ""}, + {"https://github.com/acme/widgets", "", ""}, + } + for _, tt := range tests { + got, err := gitSubmoduleLocation(tt.repoURL, tt.submoduleURL) + if tt.want == "" { + if err == nil { + t.Errorf("location of %q = %q, want an error", tt.submoduleURL, got) + } + continue + } + if err != nil || got != tt.want { + t.Errorf("location of %q from %s = %q, %v; want %q", tt.submoduleURL, tt.repoURL, got, err, tt.want) + } + } +} + +// TestGitMirrorCachesSubmodules mirrors a commit with a submodule served by +// the same upstream and one on a host no upstream.git serves, which is +// skipped. +func TestGitMirrorCachesSubmodules(t *testing.T) { + g := newGitTestSetup(t, 0, 0) + libBare := filepath.Join(g.root, "lib.git") + runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", libBare) + libWork := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", libBare, libWork) + runGitCommand(t, libWork, "commit", "--quiet", "--allow-empty", "-m", "lib") + runGitCommand(t, libWork, "push", "--quiet", "origin", "HEAD:main") + libCommit := runGitCommand(t, libWork, "rev-parse", "HEAD") + + gitmodules := "[submodule \"lib\"]\n\tpath = lib\n\turl = ../lib.git\n" + + "[submodule \"elsewhere\"]\n\tpath = elsewhere\n\turl = https://elsewhere.example/x.git\n" + if err := os.WriteFile(filepath.Join(g.work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { + t.Fatal(err) + } + runGitCommand(t, g.work, "add", ".gitmodules") + runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",lib") + runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+strings.Repeat("ab", 20)+",elsewhere") + runGitCommand(t, g.work, "commit", "--quiet", "-m", "submodules") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + location := strings.TrimPrefix(g.upstreamURL, "http://") + "/repo" + + got, err := g.handler.Mirror(context.Background(), location, sha) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("first mirror = %+v, want fetched", got) + } + if got, err = g.handler.Mirror(context.Background(), location, sha); err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + if !strings.Contains(g.logs.String(), "elsewhere.example") { + t.Errorf("log doesn't name the skipped submodule:\n%s", g.logs.String()) + } + + // With the upstream gone, the submodule's commit is served from the cache. + g.upstream.Close() + clone := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", g.server.URL+"/git/up/lib", clone) + if head := runGitCommand(t, clone, "rev-parse", "HEAD"); head != libCommit { + t.Errorf("cloned lib HEAD %s, want %s", head, libCommit) + } +} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go index cf553d60..aeec2bc9 100644 --- a/internal/handler/git_test.go +++ b/internal/handler/git_test.go @@ -129,7 +129,9 @@ func runGitCommand(t *testing.T, dir string, args ...string) string { // gitTestSetup is a GitHandler in front of a cache of one upstream // repository, "repo", served by git http-backend. type gitTestSetup struct { - work string + work string + // root holds the upstream's bare repositories. + root string server *httptest.Server // upstream serves the repository, at upstreamURL/repo, and handler is // the GitHandler under test. @@ -161,7 +163,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe bare := filepath.Join(root, "repo.git") runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) runGitCommand(t, bare, "config", "uploadpack.allowAnySHA1InWant", "true") - g := &gitTestSetup{work: t.TempDir(), logs: &syncLog{}} + g := &gitTestSetup{work: t.TempDir(), root: root, logs: &syncLog{}} runGitCommand(t, "", "clone", "--quiet", bare, g.work) runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "first") runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") diff --git a/internal/mirror/git.go b/internal/mirror/git.go index a4908dc5..e9c04ffe 100644 --- a/internal/mirror/git.go +++ b/internal/mirror/git.go @@ -9,10 +9,10 @@ import ( // GitMirrorer caches Git repositories for pkg:git items. location is the // repository's host and path, such as codeberg.org/forgejo/forgejo, and -// commit a full commit ID, or empty to cache the repository's branches and -// tags only. +// version a full commit ID or a tag, or empty to cache the repository's +// branches and tags only. type GitMirrorer interface { - Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) + Mirror(ctx context.Context, location, version string) (gitcache.MirrorResult, error) } // UnavailableGit fails every pkg:git item with Reason. @@ -26,8 +26,8 @@ func (u UnavailableGit) Mirror(context.Context, string, string) (gitcache.Mirror var errNoGitUpstream = errors.New("git repositories are mirrored from upstream.git, and none is configured") -// mirrorGit caches a pkg:git repository, and the commit its version names, -// in the cache /git serves clones from. +// mirrorGit caches a pkg:git repository, and the commit its version names +// with the submodules it pins, in the cache /git serves clones from. func (m *Mirror) mirrorGit(ctx context.Context, pv PackageVersion, tracker *progressTracker) { if m.Git == nil { m.recordFailure(tracker, pv, errNoGitUpstream)