List cached Git repositories in the UI and metrics

README.md

@@ -1606,7 +1606,7 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre
 | `proxy_ecosystem_downloaded_bytes` | gauge | `ecosystem` | Accumulated bytes served from cache: cache hits multiplied by the artifact size they served. |
 | `proxy_ecosystem_artifact_downloads` | gauge | `ecosystem` | Accumulated artifact downloads served from cache. |
 | `proxy_ecosystem_cache_size_bytes` | gauge | `ecosystem` | Size of cached artifacts per ecosystem. |
-| `proxy_ecosystem_cached_artifacts` | gauge | `ecosystem` | Number of cached artifacts per ecosystem. |
+| `proxy_ecosystem_cached_artifacts` | gauge | `ecosystem` | Number of cached artifacts per ecosystem; for `git`, cached repositories. |
 | `proxy_ecosystem_packages` | gauge | `ecosystem` | Known packages per ecosystem. |
 | `proxy_ecosystem_versions` | gauge | `ecosystem` | Known package versions per ecosystem. |
 | `proxy_response_bytes_total` | counter | `ecosystem` | Response body bytes written to clients. Route-labelled, see the label caveat below. |

docs/configuration.md

@@ -946,6 +946,20 @@ those yourself. The proxy locks `cache_dir` while it runs, through a
 `.gitcache-lock` file in it, so a second process given the same directory,
 such as another replica on a shared volume, fails to start; give each its own.
 
+The web UI lists the cached repositories at `/ui/git`, linked as "Git" in the
+navigation: each one's upstream, clone URL, size, the newest commit of the
+branch HEAD names (ID, subject and age), how many branches and tags it has,
+when the upstream was last checked and a client last read it, how many clones and fetches it served,
+and, while the last refresh failed, the kind of error (such as
+`fetch_failed` or `too_large`). The request counts and errors are kept in
+memory and start over when the proxy restarts. Like the rest of the UI the
+page has no authentication, so it shows the names of repositories from
+upstreams with credentials to everyone who can reach it, as `/git` lets them
+clone those. The `proxy_ecosystem_cache_size_bytes` and
+`proxy_ecosystem_cached_artifacts` gauges report the git cache as
+`ecosystem="git"`, counting repositories as artifacts, and clones and
+fetches count as cache hits or misses there too.
+
 Git LFS objects are cached in the artifact cache, so they use the configured
 storage backend and retention rather than `cache_dir`. git-lfs finds the LFS
 API under the rewritten remote, at `/git/{name}/{repo}.git/info/lfs`, with no

internal/gitcache/cache.go

@@ -136,6 +136,12 @@ type repo struct {
 	lastUsed    atomic.Int64
 	lastChecked atomic.Int64
 	usedWritten atomic.Int64
+	// requests counts clones and fetches, and lastError is the last failed
+	// refresh, nil once one succeeds. Neither is kept across restarts.
+	requests  atomic.Int64
+	lastError atomic.Pointer[refreshError]
+	// state is the last RepositoryState read.
+	state atomic.Pointer[cachedState]
 }
 
 var errLocked = errors.New("locked by another process")

internal/gitcache/refresh.go

@@ -151,9 +151,13 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall {
 		cancel()
 		metrics.RecordUpstreamFetch(ecosystem, time.Since(start))
 		// A refresh cut short by Close is not an upstream failure.
-		if call.err != nil && c.ctx.Err() == nil {
+		switch {
+		case call.err == nil:
+			rp.lastError.Store(nil)
+		case c.ctx.Err() == nil:
 			metrics.RecordUpstreamError(ecosystem, errorType(call.err), metrics.FailureDetail{URL: repoURL, Err: call.err})
 			c.logger.Warn("git cache: refresh failed", "repository", rp.key, "error", call.err)
+			rp.lastError.Store(&refreshError{kind: errorType(call.err), at: time.Now()})
 		}
 		c.mu.Lock()
 		rp.refreshing = nil

internal/gitcache/serve.go

@@ -53,6 +53,9 @@ func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions)
 	}
 	rp := c.acquire(key)
 	hit := rp.exists()
+	if opts.Refresh {
+		rp.requests.Add(1)
+	}
 	if opts.Refresh || !hit {
 		if err := c.ensureFresh(ctx, rp, repoURL); err != nil {
 			c.release(rp)

internal/gitcache/stats.go

@@ -0,0 +1,151 @@
+package gitcache
+
+import (
+	"context"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// RepositoryStats describes a cached repository.
+type RepositoryStats struct {
+	// Key is the repository's path in the cache, such as
+	// github-1a2b3c4d/acme/widgets.git.
+	Key  string
+	Size int64
+	// LastChecked is when the upstream's refs were last compared, and
+	// LastUsed when a client last read the repository.
+	LastChecked, LastUsed time.Time
+	// Requests counts clones and fetches since the cache was opened.
+	Requests int64
+	// LastError names the kind of error the last refresh failed with, such
+	// as fetch_failed or too_large, and LastErrorAt when. Both are empty
+	// once a refresh succeeds.
+	LastError   string
+	LastErrorAt time.Time
+}
+
+// refreshError is the last failed refresh of a repository.
+type refreshError struct {
+	kind string
+	at   time.Time
+}
+
+// Repositories describes the repositories in the cache, sorted by key.
+func (c *Cache) Repositories() []RepositoryStats {
+	c.mu.Lock()
+	repos := make([]*repo, 0, len(c.repos))
+	for _, rp := range c.repos {
+		repos = append(repos, rp)
+	}
+	c.mu.Unlock()
+
+	var out []RepositoryStats
+	for _, rp := range repos {
+		if !rp.exists() {
+			continue
+		}
+		s := RepositoryStats{
+			Key:         rp.key,
+			Size:        rp.size.Load(),
+			LastChecked: unixNanoTime(rp.lastChecked.Load()),
+			LastUsed:    unixNanoTime(rp.lastUsed.Load()),
+			Requests:    rp.requests.Load(),
+		}
+		if e := rp.lastError.Load(); e != nil {
+			s.LastError, s.LastErrorAt = e.kind, e.at
+		}
+		out = append(out, s)
+	}
+	slices.SortFunc(out, func(a, b RepositoryStats) int { return strings.Compare(a.Key, b.Key) })
+	return out
+}
+
+func unixNanoTime(n int64) time.Time {
+	if n == 0 {
+		return time.Time{}
+	}
+	return time.Unix(0, n)
+}
+
+// RepositoryState is what a cached repository holds: the branch HEAD names
+// with its newest commit, and how many branches and tags there are.
+type RepositoryState struct {
+	Branch     string
+	Commit     string
+	CommitTime time.Time
+	// Subject is the first line of the commit message, at most
+	// maxSubjectLength bytes.
+	Subject        string
+	Branches, Tags int
+}
+
+// maxSubjectLength bounds RepositoryState.Subject.
+const maxSubjectLength = 200
+
+// headFields are the commit ID, time and subject readState asks git log for.
+const headFields = 3
+
+// cachedState is a repository's state, or the error reading it failed
+// with, as read after the check at checked.
+type cachedState struct {
+	checked int64
+	state   RepositoryState
+	err     error
+}
+
+// RepositoryState reads the state of the repository cached under key. It is
+// read again only after the repository's next check of the upstream, so a
+// page listing many repositories runs git for those that changed. A read
+// that failed is kept the same way.
+func (c *Cache) RepositoryState(ctx context.Context, key string) (RepositoryState, error) {
+	c.mu.Lock()
+	rp := c.repos[key]
+	c.mu.Unlock()
+	if rp == nil || !rp.exists() {
+		return RepositoryState{}, ErrNotFound
+	}
+	checked := rp.lastChecked.Load()
+	if cached := rp.state.Load(); cached != nil && cached.checked == checked {
+		return cached.state, cached.err
+	}
+	state, err := c.readState(ctx, rp.dir)
+	// A read the request gave up on says nothing about the repository.
+	if ctx.Err() == nil {
+		rp.state.Store(&cachedState{checked: checked, state: state, err: err})
+	}
+	return state, err
+}
+
+func (c *Cache) readState(ctx context.Context, dir string) (RepositoryState, error) {
+	var state RepositoryState
+	refs, err := c.run(ctx, dir, "for-each-ref", "--format=%(refname)", "refs/heads", "refs/tags")
+	if err != nil {
+		return state, err
+	}
+	for ref := range strings.Lines(refs) {
+		if strings.HasPrefix(ref, "refs/heads/") {
+			state.Branches++
+		} else {
+			state.Tags++
+		}
+	}
+	// A repository with no branch has no HEAD commit to show.
+	if branch, err := c.run(ctx, dir, "symbolic-ref", "--quiet", "--short", "HEAD"); err == nil {
+		state.Branch = strings.TrimSpace(branch)
+	}
+	if head, err := c.run(ctx, dir, "log", "-1", "--format=%H%x00%ct%x00%s", "HEAD", "--"); err == nil {
+		fields := strings.SplitN(strings.TrimSpace(head), "\x00", headFields)
+		if len(fields) == headFields {
+			state.Commit, state.Subject = fields[0], fields[2]
+			if unix, err := strconv.ParseInt(fields[1], 10, 64); err == nil {
+				state.CommitTime = time.Unix(unix, 0)
+			}
+			if len(state.Subject) > maxSubjectLength {
+				state.Subject = strings.ToValidUTF8(state.Subject[:maxSubjectLength], "") + "…"
+			}
+		}
+	}
+	return state, nil
+}

internal/gitcache/stats_test.go

@@ -0,0 +1,116 @@
+package gitcache
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+func TestRepositoriesDescribesCachedRepositories(t *testing.T) {
+	upstream := newFakeUpstream(t)
+	upstream.newRepo("repo")
+	tc := newTestCache(t, upstream.server.URL, nil)
+	if got := tc.Repositories(); len(got) != 0 {
+		t.Fatalf("empty cache lists %+v", got)
+	}
+
+	start := time.Now().Add(-time.Second)
+	tc.clone(t, "repo")
+	tc.clone(t, "repo")
+	waitIdle(t, tc.Cache)
+	got := tc.Repositories()
+	if len(got) != 1 {
+		t.Fatalf("repositories %+v, want one", got)
+	}
+	r := got[0]
+	if r.Key != "up/repo.git" || r.Size <= 0 || r.Requests != 2 || r.LastError != "" {
+		t.Errorf("repository %+v, want up/repo.git with a size, 2 requests and no error", r)
+	}
+	if r.LastChecked.Before(start) || r.LastUsed.Before(start) {
+		t.Errorf("checked %v, used %v, want both after %v", r.LastChecked, r.LastUsed, start)
+	}
+
+	// A refresh that fails is reported until one succeeds.
+	upstream.server.Close()
+	tc.clone(t, "repo")
+	waitIdle(t, tc.Cache)
+	r = tc.Repositories()[0]
+	if r.LastError != "fetch_failed" || r.LastErrorAt.Before(start) || r.Requests != 3 {
+		t.Errorf("after a failed refresh %+v, want fetch_failed and 3 requests", r)
+	}
+}
+
+// TestRepositoriesListsRepositoriesFromDisk checks a cache opened over a
+// cache_dir lists what is already there.
+func TestRepositoriesListsRepositoriesFromDisk(t *testing.T) {
+	upstream := newFakeUpstream(t)
+	upstream.newRepo("repo")
+	dir := t.TempDir()
+	first := newTestCache(t, upstream.server.URL, func(o *Options) { o.CacheDir = dir })
+	first.clone(t, "repo")
+	waitIdle(t, first.Cache)
+	if err := first.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	second := newTestCache(t, upstream.server.URL, func(o *Options) { o.CacheDir = dir })
+	got := second.Repositories()
+	if len(got) != 1 || got[0].Key != "up/repo.git" || got[0].Size <= 0 || got[0].LastChecked.IsZero() || got[0].Requests != 0 {
+		t.Errorf("repositories %+v, want up/repo.git with its size and last check", got)
+	}
+}
+
+func TestRepositoryState(t *testing.T) {
+	upstream := newFakeUpstream(t)
+	work := upstream.newRepo("repo")
+	head := runGit(t, work, "rev-parse", "HEAD")
+	tc := newTestCache(t, upstream.server.URL, nil)
+	tc.clone(t, "repo")
+	waitIdle(t, tc.Cache)
+
+	got, err := tc.RepositoryState(context.Background(), "up/repo.git")
+	if err != nil {
+		t.Fatalf("RepositoryState: %v", err)
+	}
+	if got.Branch != "main" || got.Commit != head || got.Subject != "first" || got.Branches != 1 || got.Tags != 1 ||
+		time.Since(got.CommitTime) > time.Hour {
+		t.Errorf("state %+v, want main at %s, first, 1 branch and 1 tag", got, head)
+	}
+
+	// The state follows a refresh that fetched.
+	runGit(t, work, "checkout", "--quiet", "-b", "feature")
+	runGit(t, work, "push", "--quiet", "origin", "feature")
+	runGit(t, work, "checkout", "--quiet", "main")
+	head = upstream.commit(work, "second")
+	runGit(t, work, "push", "--quiet", "origin", "main")
+	tc.clone(t, "repo")
+	waitIdle(t, tc.Cache)
+	if got, err = tc.RepositoryState(context.Background(), "up/repo.git"); err != nil || got.Commit != head || got.Subject != "second" || got.Branches != 2 {
+		t.Errorf("after a push: %+v, %v; want main at %s, second, 2 branches", got, err, head)
+	}
+
+	if _, err := tc.RepositoryState(context.Background(), "up/missing.git"); !errors.Is(err, ErrNotFound) {
+		t.Errorf("state of an uncached repository: %v, want ErrNotFound", err)
+	}
+}
+
+// TestRepositoryStateKeepsAFailedRead checks a repository whose state can't
+// be read isn't read again until its next upstream check.
+func TestRepositoryStateKeepsAFailedRead(t *testing.T) {
+	upstream := newFakeUpstream(t)
+	upstream.newRepo("repo")
+	tc := newTestCache(t, upstream.server.URL, nil)
+	tc.clone(t, "repo")
+	waitIdle(t, tc.Cache)
+	rp := tc.repo("up/repo.git")
+	rp.state.Store(&cachedState{checked: rp.lastChecked.Load(), err: errors.New("unreadable")})
+
+	if _, err := tc.RepositoryState(context.Background(), "up/repo.git"); err == nil || err.Error() != "unreadable" {
+		t.Errorf("state = %v, want the kept error", err)
+	}
+	rp.lastChecked.Add(1)
+	if _, err := tc.RepositoryState(context.Background(), "up/repo.git"); err != nil {
+		t.Errorf("state after a new check = %v, want it read again", err)
+	}
+}