From 1205fa759e60f3c484c2646be7e75ac6b036e1f8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 13:54:24 +0100 Subject: [PATCH 1/2] Show versions held back by cooldown on the package page and in the API Part of #617. The package page marks versions in cooldown with when they become available, and lists a version only seen in metadata (no artifact row) only while it is held back. /api/package returns the package's cooldown and available_at on a held-back version. FillVersionPublishedAt stores a publish time without overwriting one, for the cooldown filters to call once they record the versions they hold back. --- docs/configuration.md | 2 +- docs/swagger/docs.go | 4 + docs/swagger/swagger.json | 4 + internal/database/database_test.go | 76 +++++++++++++ internal/database/queries.go | 54 +++++++++ internal/server/api.go | 41 ++++++- internal/server/cooldown_view.go | 45 +++++++- internal/server/cooldown_view_test.go | 104 ++++++++++++++++++ internal/server/dashboard.go | 2 +- internal/server/server.go | 13 ++- internal/server/server_test.go | 5 + .../server/templates/pages/package_show.html | 1 + internal/server/templates_test.go | 11 +- 13 files changed, 352 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0b9c7799..9c67e980 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1125,7 +1125,7 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index e35a86f3..57a896ff 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -1062,6 +1062,10 @@ const docTemplate = `{ "server.PackageResponse": { "type": "object", "properties": { + "cooldown": { + "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").", + "type": "string" + }, "description": { "type": "string" }, diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 09e4f11e..1a95e146 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -1055,6 +1055,10 @@ "server.PackageResponse": { "type": "object", "properties": { + "cooldown": { + "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").", + "type": "string" + }, "description": { "type": "string" }, diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a63..30c6bff0 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1164,3 +1164,79 @@ func TestVersionPublishedAtPreserved(t *testing.T) { } }) } + +func TestFillVersionPublishedAt(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkg, held, known = "pkg:npm/leftpad", "pkg:npm/leftpad@2.0.0", "pkg:npm/leftpad@1.0.0" + stored := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + filtered := time.Date(2021, 6, 7, 8, 9, 10, 0, time.UTC) + + if err := db.SetVersionPublishedAt(known, pkg, stored); err != nil { + t.Fatalf("SetVersionPublishedAt: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: held, PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + + for _, purl := range []string{known, held, "pkg:npm/leftpad@3.0.0"} { + if err := db.FillVersionPublishedAt(purl, pkg, filtered); err != nil { + t.Fatalf("FillVersionPublishedAt(%s): %v", purl, err) + } + } + + want := map[string]time.Time{ + known: stored, + held: filtered, + "pkg:npm/leftpad@3.0.0": filtered, + } + for purl, at := range want { + got, err := db.GetVersionByPURL(purl) + if err != nil || got == nil { + t.Fatalf("GetVersionByPURL(%s): %v", purl, err) + } + if !got.PublishedAt.Valid || !got.PublishedAt.Time.Equal(at) { + t.Errorf("%s PublishedAt = %v, want %v", purl, got.PublishedAt, at) + } + } + }) +} + +func TestGetFetchedVersionPURLs(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkg = "pkg:npm/leftpad" + for _, v := range []string{"1.0.0", "2.0.0", "3.0.0"} { + if err := db.UpsertVersion(&Version{PURL: pkg + "@" + v, PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + } + if err := db.UpsertVersion(&Version{PURL: "pkg:npm/other@1.0.0", PackagePURL: "pkg:npm/other"}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + artifacts := []Artifact{ + {VersionPURL: pkg + "@1.0.0", Filename: "leftpad-1.0.0.tgz", UpstreamURL: "https://example.com/1", + StoragePath: sql.NullString{String: "npm/leftpad-1.0.0.tgz", Valid: true}}, + // Evicted: the row stays without a storage path. + {VersionPURL: pkg + "@2.0.0", Filename: "leftpad-2.0.0.tgz", UpstreamURL: "https://example.com/2"}, + {VersionPURL: "pkg:npm/other@1.0.0", Filename: "other-1.0.0.tgz", UpstreamURL: "https://example.com/3"}, + } + for i := range artifacts { + if err := db.UpsertArtifact(&artifacts[i]); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + } + + got, err := db.GetFetchedVersionPURLs(pkg) + if err != nil { + t.Fatalf("GetFetchedVersionPURLs: %v", err) + } + want := map[string]bool{pkg + "@1.0.0": true, pkg + "@2.0.0": true} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for purl := range want { + if !got[purl] { + t.Errorf("missing %s in %v", purl, got) + } + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e80..5ba8edfe 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -223,6 +223,60 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } +// FillVersionPublishedAt records a version's publish time like +// SetVersionPublishedAt, but keeps a time already stored. The cooldown +// filters call it for each version they hold back, on every uncached +// metadata request, so it writes without reading first. +func (db *DB) FillVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { + now := time.Now() + var query string + + if db.dialect == DialectPostgres { + query = ` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(purl) DO UPDATE SET + published_at = EXCLUDED.published_at, + updated_at = EXCLUDED.updated_at + WHERE versions.published_at IS NULL + ` + } else { + query = ` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO UPDATE SET + published_at = excluded.published_at, + updated_at = excluded.updated_at + WHERE versions.published_at IS NULL + ` + } + + _, err := db.Exec(query, versionPURL, packagePURL, publishedAt, now, now) + if err != nil { + return fmt.Errorf("filling version publish time: %w", err) + } + return nil +} + +// GetFetchedVersionPURLs returns the versions of a package that have an +// artifact row, cached now or evicted since. Versions without one were only +// seen in metadata. +func (db *DB) GetFetchedVersionPURLs(packagePURL string) (map[string]bool, error) { + var purls []string + query := db.Rebind(` + SELECT DISTINCT version_purl FROM artifacts + WHERE version_purl IN (SELECT purl FROM versions WHERE package_purl = ?) + `) + if err := db.Select(&purls, query, packagePURL); err != nil { + return nil, fmt.Errorf("listing fetched versions: %w", err) + } + fetched := make(map[string]bool, len(purls)) + for _, p := range purls { + fetched[p] = true + } + return fetched, nil +} + // Artifact queries func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { diff --git a/internal/server/api.go b/internal/server/api.go index b2043672..ca1e626b 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "strings" + "time" shared "github.com/git-pkgs/enrichment" "github.com/git-pkgs/proxy/internal/database" @@ -23,6 +24,8 @@ type APIHandler struct { enrichment *enrichment.Service ecosystems *shared.EcosystemsClient db DBSearcher + // cooldown is nil with cooldown off. + cooldown *cooldownView } // DBSearcher defines the interface for database search operations. @@ -46,6 +49,13 @@ func NewAPIHandler(svc *enrichment.Service, db DBSearcher) *APIHandler { return h } +// newAPIHandler returns the server's API handler, with its cooldown policy. +func (s *Server) newAPIHandler(svc *enrichment.Service) *APIHandler { + h := NewAPIHandler(svc, s.db) + h.cooldown = s.cooldown + return h +} + // PackageResponse contains enriched package metadata. type PackageResponse struct { Ecosystem string `json:"ecosystem"` @@ -57,6 +67,9 @@ type PackageResponse struct { Homepage string `json:"homepage,omitempty"` Repository string `json:"repository,omitempty"` RegistryURL string `json:"registry_url,omitempty"` + // Cooldown is the cooldown that applies to the package, in the + // configuration's syntax ("3d", "0"). + Cooldown string `json:"cooldown,omitempty"` } // VersionResponse contains enriched version metadata. @@ -66,11 +79,16 @@ type VersionResponse struct { Version string `json:"version"` License string `json:"license,omitempty"` PublishedAt string `json:"published_at,omitempty"` + // AvailableAt is when a version held back by cooldown leaves it. + AvailableAt string `json:"available_at,omitempty"` Integrity string `json:"integrity,omitempty"` Yanked bool `json:"yanked"` IsOutdated bool `json:"is_outdated"` } +// apiTimeFormat is the format of the API's timestamps. +const apiTimeFormat = "2006-01-02T15:04:05Z" + // VulnResponse contains vulnerability information. type VulnResponse struct { ID string `json:"id"` @@ -179,6 +197,7 @@ func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) { Homepage: info.Homepage, Repository: info.Repository, RegistryURL: info.RegistryURL, + Cooldown: h.packageCooldown(ecosystem, fullName), } writeJSON(w, resp) return @@ -212,6 +231,7 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste Homepage: info.Homepage, Repository: info.Repository, RegistryURL: info.RegistryURL, + Cooldown: h.packageCooldown(ecosystem, name), } writeJSON(w, resp) @@ -240,6 +260,7 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste Homepage: result.Package.Homepage, Repository: result.Package.Repository, RegistryURL: result.Package.RegistryURL, + Cooldown: h.packageCooldown(ecosystem, name), } } @@ -253,8 +274,11 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste Yanked: result.Version.Yanked, IsOutdated: result.IsOutdated, } - if !result.Version.PublishedAt.IsZero() { - resp.Version.PublishedAt = result.Version.PublishedAt.Format("2006-01-02T15:04:05Z") + if published := result.Version.PublishedAt; !published.IsZero() { + resp.Version.PublishedAt = published.UTC().Format(apiTimeFormat) + if at := h.cooldown.availableAt(ecosystem, name, published, time.Now()); !at.IsZero() { + resp.Version.AvailableAt = at.UTC().Format(apiTimeFormat) + } } } @@ -272,6 +296,19 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste writeJSON(w, resp) } +// packageCooldown returns the cooldown for a package in the configuration's +// syntax, or "" with cooldown off or not applied to the ecosystem. +func (h *APIHandler) packageCooldown(ecosystem, name string) string { + c := h.cooldown.forPackage(ecosystem, name) + switch { + case c == nil: + return "" + case c.Duration == cooldownOff: + return "0" + } + return c.Duration +} + // HandleVulnsPath dispatches /api/vulns/{ecosystem}/* to the vulns handler. // Supports both {name} and {name}/{version} paths with namespaced package names. func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { diff --git a/internal/server/cooldown_view.go b/internal/server/cooldown_view.go index f244c851..415d708f 100644 --- a/internal/server/cooldown_view.go +++ b/internal/server/cooldown_view.go @@ -7,6 +7,7 @@ import ( "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/packageurl" ) @@ -30,6 +31,14 @@ type CooldownOverride struct { Unsupported bool } +// PackageVersion is a version on the package page. +type PackageVersion struct { + database.Version + // AvailableAt is when the version leaves cooldown, zero when it is not + // held back. + AvailableAt time.Time +} + // PackageCooldown is the cooldown that applies to one package, and the kind // of rule it comes from. type PackageCooldown struct { @@ -104,6 +113,37 @@ func (v *cooldownView) forPackage(ecosystem, name string) *PackageCooldown { return &PackageCooldown{Duration: formatCooldown(v.policy.For(ecosystem, purl)), Rule: rule} } +// availableAt returns when a version published at publishedAt leaves +// cooldown, or zero when it is not held back at now. +func (v *cooldownView) availableAt(ecosystem, name string, publishedAt, now time.Time) time.Time { + if !v.enabled() || !handler.CooldownSupported(ecosystem) || publishedAt.IsZero() { + return time.Time{} + } + d := v.policy.Evaluate(ecosystem, packageurl.MakeString(ecosystem, name, ""), publishedAt, now) + if d.Allowed { + return time.Time{} + } + return d.AvailableAt +} + +// packageVersions returns the versions to list on a package page. A version +// without an artifact row was only seen in metadata, by a cooldown filter or +// a download check, and is listed only while it is held back. +func (v *cooldownView) packageVersions(pkg *database.Package, versions []database.Version, fetched map[string]bool, now time.Time) []PackageVersion { + rows := make([]PackageVersion, 0, len(versions)) + for _, ver := range versions { + var at time.Time + if ver.PublishedAt.Valid { + at = v.availableAt(pkg.Ecosystem, pkg.Name, ver.PublishedAt.Time, now) + } + if at.IsZero() && !fetched[ver.PURL] { + continue + } + rows = append(rows, PackageVersion{Version: ver, AvailableAt: at}) + } + return rows +} + func formatConfigDuration(s string) string { d, err := cooldown.ParseDuration(s) if err != nil { @@ -112,13 +152,16 @@ func formatConfigDuration(s string) string { return formatCooldown(d) } +// cooldownOff is how formatCooldown writes a zero cooldown. +const cooldownOff = "off" + // formatCooldown writes d as the configuration does: "3d", "12h", "30m", // and "off" for zero. func formatCooldown(d time.Duration) string { const day = 24 * time.Hour switch { case d <= 0: - return "off" + return cooldownOff case d%day == 0: return fmt.Sprintf("%dd", d/day) case d%time.Hour == 0: diff --git a/internal/server/cooldown_view_test.go b/internal/server/cooldown_view_test.go index 13e731cb..0a347a82 100644 --- a/internal/server/cooldown_view_test.go +++ b/internal/server/cooldown_view_test.go @@ -1,6 +1,7 @@ package server import ( + "database/sql" "net/http" "net/http/httptest" "slices" @@ -152,3 +153,106 @@ func TestCooldownPages(t *testing.T) { t.Error("package page doesn't show the 7d ecosystem override") } } + +func TestPackagePageWithheldVersions(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.cooldown = testCooldownView(t, config.CooldownConfig{Default: "3d"}) + + const pkg = "pkg:npm/lodash" + if err := ts.db.UpsertPackage(&database.Package{PURL: pkg, Ecosystem: "npm", Name: "lodash"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + now := time.Now() + held := now.Add(-time.Hour) + versions := map[string]time.Time{ + "1.0.0": now.Add(-30 * 24 * time.Hour), // fetched + "1.1.0": now.Add(-time.Hour), // fetched while held back + "2.0.0": now.Add(-20 * 24 * time.Hour), // only seen, left cooldown + "2.1.0": held, // only seen, held back + } + for v, at := range versions { + if err := ts.db.SetVersionPublishedAt(pkg+"@"+v, pkg, at); err != nil { + t.Fatalf("SetVersionPublishedAt: %v", err) + } + } + // No publish time: the download checks never stored one. + if err := ts.db.UpsertVersion(&database.Version{PURL: pkg + "@0.9.0", PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + for _, v := range []string{"1.0.0", "1.1.0"} { + if err := ts.db.UpsertArtifact(&database.Artifact{ + VersionPURL: pkg + "@" + v, Filename: "lodash-" + v + ".tgz", UpstreamURL: "https://example.com/" + v, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + } + + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/package/npm/lodash", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status %d", w.Code) + } + page := w.Body.String() + + for _, v := range []string{"1.0.0", "1.1.0", "2.1.0"} { + if !strings.Contains(page, "lodash@"+v) { + t.Errorf("page is missing %s", v) + } + } + for _, v := range []string{"2.0.0", "0.9.0"} { + if strings.Contains(page, "lodash@"+v) { + t.Errorf("page lists %s, which nobody fetched and cooldown no longer holds back", v) + } + } + until := held.Add(3 * 24 * time.Hour).UTC().Format("2006-01-02 15:04") + if got := strings.Count(page, "held back until"); got != 2 { + t.Errorf("page marks %d versions as held back, want 2 (1.1.0 and 2.1.0)", got) + } + if !strings.Contains(page, "held back until "+until+" UTC") { + t.Errorf("page is missing the availability of 2.1.0, %s", until) + } +} + +func TestPackageVersionsCooldownOff(t *testing.T) { + var off *cooldownView + pkg := &database.Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"} + recent := sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true} + versions := []database.Version{ + {PURL: "pkg:npm/lodash@1.0.0", PublishedAt: recent}, + {PURL: "pkg:npm/lodash@2.0.0", PublishedAt: recent}, + } + got := off.packageVersions(pkg, versions, map[string]bool{"pkg:npm/lodash@1.0.0": true}, time.Now()) + if len(got) != 1 || got[0].PURL != "pkg:npm/lodash@1.0.0" || !got[0].AvailableAt.IsZero() { + t.Errorf("packageVersions with cooldown off = %+v, want only the fetched 1.0.0", got) + } +} + +func TestAPIPackageCooldown(t *testing.T) { + h := &APIHandler{cooldown: testCooldownView(t, config.CooldownConfig{ + Default: "3d", + Packages: map[string]string{"pkg:npm/leftpad": "0"}, + })} + tests := []struct{ ecosystem, name, want string }{ + {"npm", "lodash", "3d"}, + {"npm", "leftpad", "0"}, + {"deb", "curl", ""}, + } + for _, tt := range tests { + if got := h.packageCooldown(tt.ecosystem, tt.name); got != tt.want { + t.Errorf("packageCooldown(%s, %s) = %q, want %q", tt.ecosystem, tt.name, got, tt.want) + } + } + if got := (&APIHandler{}).packageCooldown("npm", "lodash"); got != "" { + t.Errorf("packageCooldown with cooldown off = %q, want empty", got) + } + + now := time.Now() + published := now.Add(-time.Hour) + if got, want := h.cooldown.availableAt("npm", "lodash", published, now), published.Add(72*time.Hour); !got.Equal(want) { + t.Errorf("availableAt = %v, want %v", got, want) + } + if got := h.cooldown.availableAt("npm", "lodash", now.Add(-96*time.Hour), now); !got.IsZero() { + t.Errorf("availableAt after cooldown = %v, want zero", got) + } +} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index 7e14af6c..08cda58d 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -66,7 +66,7 @@ type RegistryConfig struct { type PackageShowData struct { Layout Package *database.Package - Versions []database.Version + Versions []PackageVersion Vulnerabilities []database.Vulnerability LicenseCategory string // Cooldown is nil with cooldown off. diff --git a/internal/server/server.go b/internal/server/server.go index 2f0bd68b..5ced4bc4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -372,7 +372,7 @@ func (s *Server) serve(listener net.Listener) error { // API endpoints for enrichment data enrichSvc := enrichment.New(s.logger) - apiHandler := NewAPIHandler(enrichSvc, s.db) + apiHandler := s.newAPIHandler(enrichSvc) r.Get("/api/package/{ecosystem}/*", apiHandler.HandlePackagePath) r.Get("/api/vulns/{ecosystem}/*", apiHandler.HandleVulnsPath) @@ -1163,6 +1163,15 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, s.logger.Error("failed to get versions", "error", err) versions = []database.Version{} } + fetched, err := s.db.GetFetchedVersionPURLs(pkg.PURL) + if err != nil { + // List every version rather than hide fetched ones. + s.logger.Error("failed to get fetched versions", "error", err) + fetched = make(map[string]bool, len(versions)) + for _, v := range versions { + fetched[v.PURL] = true + } + } vulns, err := s.db.GetVulnerabilitiesForPackage(ecosystem, name) if err != nil { @@ -1173,7 +1182,7 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, data := PackageShowData{ Layout: s.layoutFor(r), Package: pkg, - Versions: versions, + Versions: s.cooldown.packageVersions(pkg, versions, fetched, time.Now()), Vulnerabilities: vulns, LicenseCategory: categorizeLicense(pkg.License), Cooldown: s.cooldown.forPackage(pkg.Ecosystem, pkg.Name), diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 292643c1..aad90377 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1098,6 +1098,11 @@ func TestVersionShowPage_PlusInVersion(t *testing.T) { }); err != nil { t.Fatalf("failed to upsert version: %v", err) } + if err := ts.db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, Filename: "nmap.deb", UpstreamURL: "https://example.com/nmap.deb", + }); err != nil { + t.Fatalf("failed to upsert artifact: %v", err) + } // The package page must link to and display the decoded version. req := httptest.NewRequest("GET", "/ui/package/deb/nmap", nil) diff --git a/internal/server/templates/pages/package_show.html b/internal/server/templates/pages/package_show.html index b0dd29a6..977ff6e2 100644 --- a/internal/server/templates/pages/package_show.html +++ b/internal/server/templates/pages/package_show.html @@ -69,6 +69,7 @@

Versions ({{len .Versions}})

{{.DisplayPURL}} {{if .Yanked}}yanked{{end}} + {{if not .AvailableAt.IsZero}}held back until {{.AvailableAt.UTC.Format "2006-01-02 15:04"}} UTC{{end}} {{if .PublishedAt.Valid}}{{.PublishedAt.Time.Format "2006-01-02"}}{{end}} diff --git a/internal/server/templates_test.go b/internal/server/templates_test.go index 708f02c5..c6fa4927 100644 --- a/internal/server/templates_test.go +++ b/internal/server/templates_test.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" "testing" + "time" "github.com/git-pkgs/proxy/internal/database" ) @@ -122,8 +123,12 @@ func TestTemplatesRenderAllPages(t *testing.T) { LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, License: sql.NullString{String: "MIT", Valid: true}, }, - Versions: []database.Version{ - {PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}, + Versions: []PackageVersion{ + {Version: database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}}, + { + Version: database.Version{PURL: "pkg:npm/lodash@4.17.22", PackagePURL: "pkg:npm/lodash"}, + AvailableAt: time.Date(2026, 1, 2, 3, 4, 0, 0, time.UTC), + }, }, Vulnerabilities: []database.Vulnerability{}, LicenseCategory: "permissive", @@ -134,7 +139,7 @@ func TestTemplatesRenderAllPages(t *testing.T) { Ecosystem: "npm", Name: "minimal", }, - Versions: []database.Version{}, + Versions: []PackageVersion{}, Vulnerabilities: []database.Vulnerability{}, LicenseCategory: "unknown", }}, From 683455aecb386a8a8cb54616cb7db42268e30dd3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 3 Nov 2026 14:39:36 +0100 Subject: [PATCH 2/2] Test available_at through the API and drop FillVersionPublishedAt The API handler takes the enrichment calls it uses as an interface, so the test can answer them without reaching a registry. Remove FillVersionPublishedAt until the shared cooldown filter calls it. --- internal/database/database_test.go | 36 ------------- internal/database/queries.go | 35 ------------- internal/server/api.go | 14 +++-- internal/server/api_cooldown_test.go | 76 ++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 74 deletions(-) create mode 100644 internal/server/api_cooldown_test.go diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 6b49c32d..1374dd95 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1165,42 +1165,6 @@ func TestVersionPublishedAtPreserved(t *testing.T) { }) } -func TestFillVersionPublishedAt(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - const pkg, held, known = "pkg:npm/leftpad", "pkg:npm/leftpad@2.0.0", "pkg:npm/leftpad@1.0.0" - stored := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) - filtered := time.Date(2021, 6, 7, 8, 9, 10, 0, time.UTC) - - if err := db.SetVersionPublishedAt(known, pkg, stored); err != nil { - t.Fatalf("SetVersionPublishedAt: %v", err) - } - if err := db.UpsertVersion(&Version{PURL: held, PackagePURL: pkg}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - - for _, purl := range []string{known, held, "pkg:npm/leftpad@3.0.0"} { - if err := db.FillVersionPublishedAt(purl, pkg, filtered); err != nil { - t.Fatalf("FillVersionPublishedAt(%s): %v", purl, err) - } - } - - want := map[string]time.Time{ - known: stored, - held: filtered, - "pkg:npm/leftpad@3.0.0": filtered, - } - for purl, at := range want { - got, err := db.GetVersionByPURL(purl) - if err != nil || got == nil { - t.Fatalf("GetVersionByPURL(%s): %v", purl, err) - } - if !got.PublishedAt.Valid || !got.PublishedAt.Time.Equal(at) { - t.Errorf("%s PublishedAt = %v, want %v", purl, got.PublishedAt, at) - } - } - }) -} - func TestGetFetchedVersionPURLs(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { const pkg = "pkg:npm/leftpad" diff --git a/internal/database/queries.go b/internal/database/queries.go index 825841e1..5d8f5f3e 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -224,41 +224,6 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } -// FillVersionPublishedAt records a version's publish time like -// SetVersionPublishedAt, but keeps a time already stored. The cooldown -// filters call it for each version they hold back, on every uncached -// metadata request, so it writes without reading first. -func (db *DB) FillVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { - now := time.Now() - var query string - - if db.dialect == DialectPostgres { - query = ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT(purl) DO UPDATE SET - published_at = EXCLUDED.published_at, - updated_at = EXCLUDED.updated_at - WHERE versions.published_at IS NULL - ` - } else { - query = ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(purl) DO UPDATE SET - published_at = excluded.published_at, - updated_at = excluded.updated_at - WHERE versions.published_at IS NULL - ` - } - - _, err := db.Exec(query, versionPURL, packagePURL, publishedAt, now, now) - if err != nil { - return fmt.Errorf("filling version publish time: %w", err) - } - return nil -} - // GetFetchedVersionPURLs returns the versions of a package that have an // artifact row, cached now or evicted since. Versions without one were only // seen in metadata. diff --git a/internal/server/api.go b/internal/server/api.go index b08055d8..07080aa1 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -1,6 +1,7 @@ package server import ( + "context" "encoding/json" "errors" "net/http" @@ -22,13 +23,22 @@ const ( // APIHandler provides REST endpoints for package enrichment data. type APIHandler struct { - enrichment *enrichment.Service + enrichment enricher ecosystems *shared.EcosystemsClient db DBSearcher // cooldown is nil with cooldown off. cooldown *cooldownView } +// enricher is the part of enrichment.Service the API uses. +type enricher interface { + EnrichPackage(ctx context.Context, ecosystem, name string) (*enrichment.PackageInfo, error) + EnrichFull(ctx context.Context, ecosystem, name, version string) (*enrichment.EnrichmentResult, error) + CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]enrichment.VulnInfo, error) + GetLatestVersion(ctx context.Context, ecosystem, name string) (string, error) + BulkEnrichPackages(ctx context.Context, packages []struct{ Ecosystem, Name string }) map[string]*enrichment.PackageInfo +} + // DBSearcher defines the interface for database search operations. type DBSearcher interface { SearchPackages(query string, ecosystem string, limit int, offset int) ([]database.SearchResult, error) @@ -50,7 +60,6 @@ func NewAPIHandler(svc *enrichment.Service, db DBSearcher) *APIHandler { return h } -// newAPIHandler returns the server's API handler, with its cooldown policy. func (s *Server) newAPIHandler(svc *enrichment.Service) *APIHandler { h := NewAPIHandler(svc, s.db) h.cooldown = s.cooldown @@ -87,7 +96,6 @@ type VersionResponse struct { IsOutdated bool `json:"is_outdated"` } -// apiTimeFormat is the format of the API's timestamps. const apiTimeFormat = "2006-01-02T15:04:05Z" // VulnResponse contains vulnerability information. diff --git a/internal/server/api_cooldown_test.go b/internal/server/api_cooldown_test.go new file mode 100644 index 00000000..4e3e3c7d --- /dev/null +++ b/internal/server/api_cooldown_test.go @@ -0,0 +1,76 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/enrichment" + "github.com/go-chi/chi/v5" +) + +func TestAPIVersionAvailableAt(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + recent, old := now.Add(-time.Hour), now.Add(-96*time.Hour) + h := &APIHandler{ + enrichment: fakeEnricher{published: map[string]time.Time{"1.0.0": old, "2.0.0": recent}}, + cooldown: testCooldownView(t, config.CooldownConfig{Default: "3d"}), + } + r := chi.NewRouter() + r.Get("/api/package/{ecosystem}/*", h.HandlePackagePath) + + tests := []struct{ version, availableAt string }{ + {"2.0.0", recent.Add(72 * time.Hour).Format(apiTimeFormat)}, + {"1.0.0", ""}, + } + for _, tt := range tests { + t.Run(tt.version, func(t *testing.T) { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/package/npm/leftpad/"+tt.version, nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", w.Code, w.Body) + } + var resp EnrichmentResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Package == nil || resp.Package.Cooldown != "3d" { + t.Errorf("package = %+v, want cooldown 3d", resp.Package) + } + if resp.Version == nil || resp.Version.AvailableAt != tt.availableAt { + t.Errorf("version = %+v, want available_at %q", resp.Version, tt.availableAt) + } + }) + } +} + +// fakeEnricher answers version lookups with the given publish times and +// knows no packages by name alone. +type fakeEnricher struct{ published map[string]time.Time } + +func (fakeEnricher) EnrichPackage(context.Context, string, string) (*enrichment.PackageInfo, error) { + return nil, nil +} + +func (f fakeEnricher) EnrichFull(_ context.Context, ecosystem, name, version string) (*enrichment.EnrichmentResult, error) { + return &enrichment.EnrichmentResult{ + Package: &enrichment.PackageInfo{Ecosystem: ecosystem, Name: name}, + Version: &enrichment.VersionInfo{Number: version, PublishedAt: f.published[version]}, + }, nil +} + +func (fakeEnricher) CheckVulnerabilities(context.Context, string, string, string) ([]enrichment.VulnInfo, error) { + return nil, nil +} + +func (fakeEnricher) GetLatestVersion(context.Context, string, string) (string, error) { + return "", nil +} + +func (fakeEnricher) BulkEnrichPackages(context.Context, []struct{ Ecosystem, Name string }) map[string]*enrichment.PackageInfo { + return nil +}