Show versions held back by cooldown on the package page and in the API

docs/configuration.md

@@ -1179,7 +1179,7 @@ Exact `packages` entries take precedence over patterns. When several patterns ma
 
 Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families.
 
-The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported.
+The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`.
 
 Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below.
 

docs/swagger/docs.go

@@ -1062,6 +1062,10 @@ const docTemplate = `{
         "server.PackageResponse": {
             "type": "object",
             "properties": {
+                "cooldown": {
+                    "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").",
+                    "type": "string"
+                },
                 "description": {
                     "type": "string"
                 },

docs/swagger/swagger.json

@@ -1055,6 +1055,10 @@
         "server.PackageResponse": {
             "type": "object",
             "properties": {
+                "cooldown": {
+                    "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").",
+                    "type": "string"
+                },
                 "description": {
                     "type": "string"
                 },

internal/database/database_test.go

@@ -1164,3 +1164,43 @@ func TestVersionPublishedAtPreserved(t *testing.T) {
 		}
 	})
 }
+
+func TestGetFetchedVersionPURLs(t *testing.T) {
+	runWithBothDatabases(t, func(t *testing.T, db *DB) {
+		const pkg = "pkg:npm/leftpad"
+		for _, v := range []string{"1.0.0", "2.0.0", "3.0.0"} {
+			if err := db.UpsertVersion(&Version{PURL: pkg + "@" + v, PackagePURL: pkg}); err != nil {
+				t.Fatalf("UpsertVersion: %v", err)
+			}
+		}
+		if err := db.UpsertVersion(&Version{PURL: "pkg:npm/other@1.0.0", PackagePURL: "pkg:npm/other"}); err != nil {
+			t.Fatalf("UpsertVersion: %v", err)
+		}
+		artifacts := []Artifact{
+			{VersionPURL: pkg + "@1.0.0", Filename: "leftpad-1.0.0.tgz", UpstreamURL: "https://example.com/1",
+				StoragePath: sql.NullString{String: "npm/leftpad-1.0.0.tgz", Valid: true}},
+			// Evicted: the row stays without a storage path.
+			{VersionPURL: pkg + "@2.0.0", Filename: "leftpad-2.0.0.tgz", UpstreamURL: "https://example.com/2"},
+			{VersionPURL: "pkg:npm/other@1.0.0", Filename: "other-1.0.0.tgz", UpstreamURL: "https://example.com/3"},
+		}
+		for i := range artifacts {
+			if err := db.UpsertArtifact(&artifacts[i]); err != nil {
+				t.Fatalf("UpsertArtifact: %v", err)
+			}
+		}
+
+		got, err := db.GetFetchedVersionPURLs(pkg)
+		if err != nil {
+			t.Fatalf("GetFetchedVersionPURLs: %v", err)
+		}
+		want := map[string]bool{pkg + "@1.0.0": true, pkg + "@2.0.0": true}
+		if len(got) != len(want) {
+			t.Fatalf("got %v, want %v", got, want)
+		}
+		for purl := range want {
+			if !got[purl] {
+				t.Errorf("missing %s in %v", purl, got)
+			}
+		}
+	})
+}

internal/database/queries.go

@@ -224,6 +224,25 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt
 	return nil
 }
 
+// GetFetchedVersionPURLs returns the versions of a package that have an
+// artifact row, cached now or evicted since. Versions without one were only
+// seen in metadata.
+func (db *DB) GetFetchedVersionPURLs(packagePURL string) (map[string]bool, error) {
+	var purls []string
+	query := db.Rebind(`
+		SELECT DISTINCT version_purl FROM artifacts
+		WHERE version_purl IN (SELECT purl FROM versions WHERE package_purl = ?)
+	`)
+	if err := db.Select(&purls, query, packagePURL); err != nil {
+		return nil, fmt.Errorf("listing fetched versions: %w", err)
+	}
+	fetched := make(map[string]bool, len(purls))
+	for _, p := range purls {
+		fetched[p] = true
+	}
+	return fetched, nil
+}
+
 // Artifact queries
 
 func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) {

internal/server/api.go

@@ -1,10 +1,12 @@
 package server
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"net/http"
 	"strings"
+	"time"
 
 	shared "github.com/git-pkgs/enrichment"
 	"github.com/git-pkgs/proxy/internal/database"
@@ -21,9 +23,20 @@ const (
 
 // APIHandler provides REST endpoints for package enrichment data.
 type APIHandler struct {
-	enrichment *enrichment.Service
+	enrichment enricher
 	ecosystems *shared.EcosystemsClient
 	db         DBSearcher
+	// cooldown is nil with cooldown off.
+	cooldown *cooldownView
+}
+
+// enricher is the part of enrichment.Service the API uses.
+type enricher interface {
+	EnrichPackage(ctx context.Context, ecosystem, name string) (*enrichment.PackageInfo, error)
+	EnrichFull(ctx context.Context, ecosystem, name, version string) (*enrichment.EnrichmentResult, error)
+	CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]enrichment.VulnInfo, error)
+	GetLatestVersion(ctx context.Context, ecosystem, name string) (string, error)
+	BulkEnrichPackages(ctx context.Context, packages []struct{ Ecosystem, Name string }) map[string]*enrichment.PackageInfo
 }
 
 // DBSearcher defines the interface for database search operations.
@@ -47,6 +60,12 @@ func NewAPIHandler(svc *enrichment.Service, db DBSearcher) *APIHandler {
 	return h
 }
 
+func (s *Server) newAPIHandler(svc *enrichment.Service) *APIHandler {
+	h := NewAPIHandler(svc, s.db)
+	h.cooldown = s.cooldown
+	return h
+}
+
 // PackageResponse contains enriched package metadata.
 type PackageResponse struct {
 	Ecosystem       string `json:"ecosystem"`
@@ -58,6 +77,9 @@ type PackageResponse struct {
 	Homepage        string `json:"homepage,omitempty"`
 	Repository      string `json:"repository,omitempty"`
 	RegistryURL     string `json:"registry_url,omitempty"`
+	// Cooldown is the cooldown that applies to the package, in the
+	// configuration's syntax ("3d", "0").
+	Cooldown string `json:"cooldown,omitempty"`
 }
 
 // VersionResponse contains enriched version metadata.
@@ -67,11 +89,15 @@ type VersionResponse struct {
 	Version     string `json:"version"`
 	License     string `json:"license,omitempty"`
 	PublishedAt string `json:"published_at,omitempty"`
+	// AvailableAt is when a version held back by cooldown leaves it.
+	AvailableAt string `json:"available_at,omitempty"`
 	Integrity   string `json:"integrity,omitempty"`
 	Yanked      bool   `json:"yanked"`
 	IsOutdated  bool   `json:"is_outdated"`
 }
 
+const apiTimeFormat = "2006-01-02T15:04:05Z"
+
 // VulnResponse contains vulnerability information.
 type VulnResponse struct {
 	ID           string   `json:"id"`
@@ -180,6 +206,7 @@ func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) {
 			Homepage:        info.Homepage,
 			Repository:      info.Repository,
 			RegistryURL:     info.RegistryURL,
+			Cooldown:        h.packageCooldown(ecosystem, fullName),
 		}
 		writeJSON(w, resp)
 		return
@@ -213,6 +240,7 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste
 		Homepage:        info.Homepage,
 		Repository:      info.Repository,
 		RegistryURL:     info.RegistryURL,
+		Cooldown:        h.packageCooldown(ecosystem, name),
 	}
 
 	writeJSON(w, resp)
@@ -245,6 +273,7 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste
 			Homepage:        result.Package.Homepage,
 			Repository:      result.Package.Repository,
 			RegistryURL:     result.Package.RegistryURL,
+			Cooldown:        h.packageCooldown(ecosystem, name),
 		}
 	}
 
@@ -258,8 +287,11 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste
 			Yanked:     result.Version.Yanked,
 			IsOutdated: result.IsOutdated,
 		}
-		if !result.Version.PublishedAt.IsZero() {
-			resp.Version.PublishedAt = result.Version.PublishedAt.Format("2006-01-02T15:04:05Z")
+		if published := result.Version.PublishedAt; !published.IsZero() {
+			resp.Version.PublishedAt = published.UTC().Format(apiTimeFormat)
+			if at := h.cooldown.availableAt(ecosystem, name, published, time.Now()); !at.IsZero() {
+				resp.Version.AvailableAt = at.UTC().Format(apiTimeFormat)
+			}
 		}
 	}
 
@@ -277,6 +309,19 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste
 	writeJSON(w, resp)
 }
 
+// packageCooldown returns the cooldown for a package in the configuration's
+// syntax, or "" with cooldown off or not applied to the ecosystem.
+func (h *APIHandler) packageCooldown(ecosystem, name string) string {
+	c := h.cooldown.forPackage(ecosystem, name)
+	switch {
+	case c == nil:
+		return ""
+	case c.Duration == cooldownOff:
+		return "0"
+	}
+	return c.Duration
+}
+
 // HandleVulnsPath dispatches /api/vulns/{ecosystem}/* to the vulns handler.
 // Supports both {name} and {name}/{version} paths with namespaced package names.
 func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {

internal/server/api_cooldown_test.go

@@ -0,0 +1,76 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/git-pkgs/proxy/internal/config"
+	"github.com/git-pkgs/proxy/internal/enrichment"
+	"github.com/go-chi/chi/v5"
+)
+
+func TestAPIVersionAvailableAt(t *testing.T) {
+	now := time.Now().UTC().Truncate(time.Second)
+	recent, old := now.Add(-time.Hour), now.Add(-96*time.Hour)
+	h := &APIHandler{
+		enrichment: fakeEnricher{published: map[string]time.Time{"1.0.0": old, "2.0.0": recent}},
+		cooldown:   testCooldownView(t, config.CooldownConfig{Default: "3d"}),
+	}
+	r := chi.NewRouter()
+	r.Get("/api/package/{ecosystem}/*", h.HandlePackagePath)
+
+	tests := []struct{ version, availableAt string }{
+		{"2.0.0", recent.Add(72 * time.Hour).Format(apiTimeFormat)},
+		{"1.0.0", ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.version, func(t *testing.T) {
+			w := httptest.NewRecorder()
+			r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/package/npm/leftpad/"+tt.version, nil))
+			if w.Code != http.StatusOK {
+				t.Fatalf("status = %d, body = %s", w.Code, w.Body)
+			}
+			var resp EnrichmentResponse
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			if resp.Package == nil || resp.Package.Cooldown != "3d" {
+				t.Errorf("package = %+v, want cooldown 3d", resp.Package)
+			}
+			if resp.Version == nil || resp.Version.AvailableAt != tt.availableAt {
+				t.Errorf("version = %+v, want available_at %q", resp.Version, tt.availableAt)
+			}
+		})
+	}
+}
+
+// fakeEnricher answers version lookups with the given publish times and
+// knows no packages by name alone.
+type fakeEnricher struct{ published map[string]time.Time }
+
+func (fakeEnricher) EnrichPackage(context.Context, string, string) (*enrichment.PackageInfo, error) {
+	return nil, nil
+}
+
+func (f fakeEnricher) EnrichFull(_ context.Context, ecosystem, name, version string) (*enrichment.EnrichmentResult, error) {
+	return &enrichment.EnrichmentResult{
+		Package: &enrichment.PackageInfo{Ecosystem: ecosystem, Name: name},
+		Version: &enrichment.VersionInfo{Number: version, PublishedAt: f.published[version]},
+	}, nil
+}
+
+func (fakeEnricher) CheckVulnerabilities(context.Context, string, string, string) ([]enrichment.VulnInfo, error) {
+	return nil, nil
+}
+
+func (fakeEnricher) GetLatestVersion(context.Context, string, string) (string, error) {
+	return "", nil
+}
+
+func (fakeEnricher) BulkEnrichPackages(context.Context, []struct{ Ecosystem, Name string }) map[string]*enrichment.PackageInfo {
+	return nil
+}

internal/server/cooldown_view.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/git-pkgs/cooldown"
 	"github.com/git-pkgs/proxy/internal/config"
+	"github.com/git-pkgs/proxy/internal/database"
 	"github.com/git-pkgs/proxy/internal/handler"
 	"github.com/git-pkgs/proxy/internal/packageurl"
 )
@@ -30,6 +31,14 @@ type CooldownOverride struct {
 	Unsupported bool
 }
 
+// PackageVersion is a version on the package page.
+type PackageVersion struct {
+	database.Version
+	// AvailableAt is when the version leaves cooldown, zero when it is not
+	// held back.
+	AvailableAt time.Time
+}
+
 // PackageCooldown is the cooldown that applies to one package, and the kind
 // of rule it comes from.
 type PackageCooldown struct {
@@ -104,6 +113,37 @@ func (v *cooldownView) forPackage(ecosystem, name string) *PackageCooldown {
 	return &PackageCooldown{Duration: formatCooldown(v.policy.For(ecosystem, purl)), Rule: rule}
 }
 
+// availableAt returns when a version published at publishedAt leaves
+// cooldown, or zero when it is not held back at now.
+func (v *cooldownView) availableAt(ecosystem, name string, publishedAt, now time.Time) time.Time {
+	if !v.enabled() || !handler.CooldownSupported(ecosystem) || publishedAt.IsZero() {
+		return time.Time{}
+	}
+	d := v.policy.Evaluate(ecosystem, packageurl.MakeString(ecosystem, name, ""), publishedAt, now)
+	if d.Allowed {
+		return time.Time{}
+	}
+	return d.AvailableAt
+}
+
+// packageVersions returns the versions to list on a package page. A version
+// without an artifact row was only seen in metadata, by a cooldown filter or
+// a download check, and is listed only while it is held back.
+func (v *cooldownView) packageVersions(pkg *database.Package, versions []database.Version, fetched map[string]bool, now time.Time) []PackageVersion {
+	rows := make([]PackageVersion, 0, len(versions))
+	for _, ver := range versions {
+		var at time.Time
+		if ver.PublishedAt.Valid {
+			at = v.availableAt(pkg.Ecosystem, pkg.Name, ver.PublishedAt.Time, now)
+		}
+		if at.IsZero() && !fetched[ver.PURL] {
+			continue
+		}
+		rows = append(rows, PackageVersion{Version: ver, AvailableAt: at})
+	}
+	return rows
+}
+
 func formatConfigDuration(s string) string {
 	d, err := cooldown.ParseDuration(s)
 	if err != nil {
@@ -112,13 +152,16 @@ func formatConfigDuration(s string) string {
 	return formatCooldown(d)
 }
 
+// cooldownOff is how formatCooldown writes a zero cooldown.
+const cooldownOff = "off"
+
 // formatCooldown writes d as the configuration does: "3d", "12h", "30m",
 // and "off" for zero.
 func formatCooldown(d time.Duration) string {
 	const day = 24 * time.Hour
 	switch {
 	case d <= 0:
-		return "off"
+		return cooldownOff
 	case d%day == 0:
 		return fmt.Sprintf("%dd", d/day)
 	case d%time.Hour == 0: