From b79cd4d14ad2566ed9d7590ad3c57d4d7a901749 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 13:03:42 +0100 Subject: [PATCH 1/2] Show the cooldown configuration in the web UI The dashboard shows the default and the ecosystem overrides, and counts package overrides and patterns without naming them, since the UI has no auth. A package page shows the cooldown that applies to the package and the kind of rule it comes from. Also add FillVersionPublishedAt, which stores a publish time only where none is stored, for recording the versions cooldown holds back. --- docs/configuration.md | 2 + internal/database/database_test.go | 42 ++++++ internal/database/queries.go | 30 ++++ internal/server/cooldown_view.go | 122 ++++++++++++++++ internal/server/cooldown_view_test.go | 137 ++++++++++++++++++ internal/server/dashboard.go | 3 + internal/server/server.go | 6 + .../server/templates/pages/dashboard.html | 15 ++ .../server/templates/pages/package_show.html | 3 + 9 files changed, 360 insertions(+) create mode 100644 internal/server/cooldown_view.go create mode 100644 internal/server/cooldown_view_test.go diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b23..b0dee951 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1076,6 +1076,8 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from. + Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go and Maven. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a63..09f5a93d 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1112,6 +1112,48 @@ func BenchmarkMigrateSchemaFullyMigrated(b *testing.B) { } } +func TestFillVersionPublishedAt(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const purl, pkg = "pkg:npm/leftpad@1.0.0", "pkg:npm/leftpad" + first := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + publishedAt := func() sql.NullTime { + t.Helper() + got, err := db.GetVersionByPURL(purl) + if err != nil || got == nil { + t.Fatalf("GetVersionByPURL: %v, %v", got, err) + } + return got.PublishedAt + } + + // A row without a time, as the artifact cache writes it, is filled. + if err := db.UpsertVersion(&Version{PURL: purl, PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + if err := db.FillVersionPublishedAt(purl, pkg, first.In(time.FixedZone("CEST", 2*60*60))); err != nil { + t.Fatalf("FillVersionPublishedAt: %v", err) + } + if got := publishedAt(); !got.Valid || !got.Time.Equal(first) { + t.Fatalf("PublishedAt = %v, want %v", got, first) + } + + // A stored time is kept. + if err := db.FillVersionPublishedAt(purl, pkg, first.Add(time.Hour)); err != nil { + t.Fatalf("FillVersionPublishedAt again: %v", err) + } + if got := publishedAt(); !got.Time.Equal(first) { + t.Fatalf("PublishedAt after a second fill = %v, want %v kept", got.Time, first) + } + + // A version the proxy hasn't seen gets a row. + if err := db.FillVersionPublishedAt("pkg:npm/leftpad@2.0.0", pkg, first); err != nil { + t.Fatalf("FillVersionPublishedAt new: %v", err) + } + if got, err := db.GetVersionByPURL("pkg:npm/leftpad@2.0.0"); err != nil || got == nil || !got.PublishedAt.Time.Equal(first) { + t.Fatalf("new version = %+v, %v, want published %v", got, err, first) + } + }) +} + func TestVersionPublishedAtPreserved(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { publishedAt := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e80..0bd5e821 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -223,6 +223,36 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } +// FillVersionPublishedAt records a version's publish time, in UTC, only where +// none is stored yet, creating the versions row if needed. Callers that see +// the same version on every metadata request write it once without reading +// first. +func (db *DB) FillVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { + now := time.Now() + query := ` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO UPDATE SET + published_at = excluded.published_at, + updated_at = excluded.updated_at + WHERE versions.published_at IS NULL + ` + if db.dialect == DialectPostgres { + query = ` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(purl) DO UPDATE SET + published_at = EXCLUDED.published_at, + updated_at = EXCLUDED.updated_at + WHERE versions.published_at IS NULL + ` + } + if _, err := db.Exec(query, versionPURL, packagePURL, publishedAt.UTC(), now, now); err != nil { + return fmt.Errorf("filling version publish time: %w", err) + } + return nil +} + // Artifact queries func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { diff --git a/internal/server/cooldown_view.go b/internal/server/cooldown_view.go new file mode 100644 index 00000000..57a64ecb --- /dev/null +++ b/internal/server/cooldown_view.go @@ -0,0 +1,122 @@ +package server + +import ( + "fmt" + "sort" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/packageurl" +) + +// CooldownSummary describes the cooldown configuration on the dashboard. The +// UI has no auth, so package overrides and patterns are only counted: their +// names often list internal packages. +type CooldownSummary struct { + Enabled bool + Default string + Ecosystems []CooldownOverride + PackageOverrides int + PackagePatterns int +} + +// CooldownOverride is one ecosystem's cooldown. +type CooldownOverride struct { + Ecosystem string + Duration string +} + +// PackageCooldown is the cooldown that applies to one package, and the kind +// of rule it comes from. +type PackageCooldown struct { + Duration string + Rule string +} + +// cooldownView answers the UI's cooldown questions from the configuration. +type cooldownView struct { + cfg config.CooldownConfig + policy *cooldown.Policy + packages map[string]string + // patterns sets every package pattern to the same non-zero duration over + // a zero default, so a non-zero result means a pattern matched, with the + // policy's own matching. + patterns *cooldown.Policy +} + +func newCooldownView(cfg config.CooldownConfig, policy *cooldown.Policy) (*cooldownView, error) { + probe := make(map[string]string, len(cfg.PackagePatterns)) + for glob := range cfg.PackagePatterns { + probe[glob] = "1h" + } + patterns, err := cooldown.NewPolicy(nil, probe) + if err != nil { + return nil, fmt.Errorf("configuring cooldown package patterns: %w", err) + } + return &cooldownView{cfg: cfg, policy: policy, packages: cfg.NormalizedPackages(), patterns: patterns}, nil +} + +func (v *cooldownView) enabled() bool { + return v != nil && v.policy != nil && v.policy.Enabled() +} + +// summary returns the dashboard's view of the configuration. +func (v *cooldownView) summary() CooldownSummary { + if !v.enabled() { + return CooldownSummary{} + } + s := CooldownSummary{ + Enabled: true, + Default: formatConfigDuration(v.cfg.Default), + PackageOverrides: len(v.cfg.Packages), + PackagePatterns: len(v.cfg.PackagePatterns), + } + for eco, d := range v.cfg.Ecosystems { + s.Ecosystems = append(s.Ecosystems, CooldownOverride{Ecosystem: eco, Duration: formatConfigDuration(d)}) + } + sort.Slice(s.Ecosystems, func(i, j int) bool { return s.Ecosystems[i].Ecosystem < s.Ecosystems[j].Ecosystem }) + return s +} + +// forPackage returns the cooldown for a package, or nil with cooldown off. +func (v *cooldownView) forPackage(ecosystem, name string) *PackageCooldown { + if !v.enabled() { + return nil + } + purl := packageurl.MakeString(ecosystem, name, "") + rule := "default" + if _, ok := v.packages[purl]; ok { + rule = "package override" + } else if v.patterns.For(ecosystem, purl) != 0 { + rule = "package pattern" + } else if _, ok := v.cfg.Ecosystems[ecosystem]; ok { + rule = "ecosystem override" + } + return &PackageCooldown{Duration: formatCooldown(v.policy.For(ecosystem, purl)), Rule: rule} +} + +func formatConfigDuration(s string) string { + d, err := cooldown.ParseDuration(s) + if err != nil { + return s + } + return formatCooldown(d) +} + +// formatCooldown writes d as the configuration does: "3d", "12h", "30m", +// and "off" for zero. +func formatCooldown(d time.Duration) string { + const day = 24 * time.Hour + switch { + case d <= 0: + return "off" + case d%day == 0: + return fmt.Sprintf("%dd", d/day) + case d%time.Hour == 0: + return fmt.Sprintf("%dh", d/time.Hour) + case d%time.Minute == 0: + return fmt.Sprintf("%dm", d/time.Minute) + } + return d.String() +} diff --git a/internal/server/cooldown_view_test.go b/internal/server/cooldown_view_test.go new file mode 100644 index 00000000..41ff789b --- /dev/null +++ b/internal/server/cooldown_view_test.go @@ -0,0 +1,137 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" +) + +func testCooldownView(t *testing.T, cfg config.CooldownConfig) *cooldownView { + t.Helper() + policy, err := cooldown.NewPolicy(&cooldown.Config{ + Default: cfg.Default, + Ecosystems: cfg.Ecosystems, + Packages: cfg.NormalizedPackages(), + }, cfg.PackagePatterns) + if err != nil { + t.Fatalf("NewPolicy: %v", err) + } + v, err := newCooldownView(cfg, policy) + if err != nil { + t.Fatalf("newCooldownView: %v", err) + } + return v +} + +func TestCooldownViewForPackage(t *testing.T) { + v := testCooldownView(t, config.CooldownConfig{ + Default: "3d", + Ecosystems: map[string]string{"npm": "7d", "composer": "48h"}, + Packages: map[string]string{"pkg:composer/phpunit/phpunit": "0"}, + PackagePatterns: map[string]string{"pkg:npm/@corp/*": "0", "pkg:composer/acme/*": "12h"}, + }) + tests := []struct { + ecosystem, name string + want PackageCooldown + }{ + {"cargo", "serde", PackageCooldown{"3d", "default"}}, + {"npm", "lodash", PackageCooldown{"7d", "ecosystem override"}}, + {"composer", "monolog/monolog", PackageCooldown{"2d", "ecosystem override"}}, + {"composer", "phpunit/phpunit", PackageCooldown{"off", "package override"}}, + {"composer", "acme/tool", PackageCooldown{"12h", "package pattern"}}, + {"npm", "@corp/ui", PackageCooldown{"off", "package pattern"}}, + } + for _, tt := range tests { + got := v.forPackage(tt.ecosystem, tt.name) + if got == nil || *got != tt.want { + t.Errorf("forPackage(%s, %s) = %+v, want %+v", tt.ecosystem, tt.name, got, tt.want) + } + } +} + +func TestCooldownViewOff(t *testing.T) { + var unset *cooldownView + for _, v := range []*cooldownView{unset, testCooldownView(t, config.CooldownConfig{})} { + if got := v.forPackage("npm", "lodash"); got != nil { + t.Errorf("forPackage with cooldown off = %+v, want nil", got) + } + if got := v.summary(); got.Enabled { + t.Errorf("summary with cooldown off = %+v, want disabled", got) + } + } +} + +// TestCooldownViewOnlyPattern checks that a pattern alone turns cooldown on. +func TestCooldownViewOnlyPattern(t *testing.T) { + v := testCooldownView(t, config.CooldownConfig{PackagePatterns: map[string]string{"pkg:npm/*": "1d"}}) + if got := v.forPackage("npm", "lodash"); got == nil || *got != (PackageCooldown{"1d", "package pattern"}) { + t.Errorf("forPackage = %+v, want 1d from the pattern", got) + } + if got := v.forPackage("cargo", "serde"); got == nil || *got != (PackageCooldown{"off", "default"}) { + t.Errorf("forPackage outside the pattern = %+v, want off by default", got) + } +} + +func TestFormatCooldown(t *testing.T) { + for d, want := range map[time.Duration]string{ + 0: "off", + 72 * time.Hour: "3d", + 36 * time.Hour: "36h", + 90 * time.Minute: "90m", + 90*time.Minute + 30*time.Second: "1h30m30s", + } { + if got := formatCooldown(d); got != want { + t.Errorf("formatCooldown(%v) = %q, want %q", d, got, want) + } + } +} + +// TestCooldownPages checks that the dashboard shows the configuration +// without the names of package overrides and patterns, and the package page +// the cooldown that applies to the package. +func TestCooldownPages(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.cooldown = testCooldownView(t, config.CooldownConfig{ + Default: "3d", + Ecosystems: map[string]string{"npm": "7d"}, + Packages: map[string]string{"pkg:npm/internal-secret": "0"}, + PackagePatterns: map[string]string{"pkg:npm/@corp-secret/*": "0"}, + }) + if err := ts.db.UpsertPackage(&database.Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + + get := func(path string) string { + t.Helper() + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusOK { + t.Fatalf("GET %s: status %d", path, w.Code) + } + return w.Body.String() + } + + dashboard := get("/ui/") + for _, want := range []string{"Cooldown", "3d", "npm:", "7d", "Package overrides:", "Package patterns:"} { + if !strings.Contains(dashboard, want) { + t.Errorf("dashboard is missing %q", want) + } + } + for _, secret := range []string{"internal-secret", "corp-secret"} { + if strings.Contains(dashboard, secret) { + t.Errorf("dashboard shows %q", secret) + } + } + + page := get("/ui/package/npm/lodash") + if !strings.Contains(page, "7d") || !strings.Contains(page, "(ecosystem override)") { + t.Error("package page doesn't show the 7d ecosystem override") + } +} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index da866a6a..7e14af6c 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -13,6 +13,7 @@ type DashboardData struct { Layout Stats DashboardStats EnrichmentStats EnrichmentStatsView + Cooldown CooldownSummary RecentPackages []PackageInfo PopularPackages []PackageInfo } @@ -68,6 +69,8 @@ type PackageShowData struct { Versions []database.Version Vulnerabilities []database.Vulnerability LicenseCategory string + // Cooldown is nil with cooldown off. + Cooldown *PackageCooldown } // VersionShowData contains data for rendering the version show page. diff --git a/internal/server/server.go b/internal/server/server.go index 12619a6a..ea87e082 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -126,6 +126,7 @@ type Server struct { sources sourceTracker scheduler *mirror.Scheduler git *gitcache.Cache + cooldown *cooldownView // browseLimits overrides defaultBrowseLimits when set; tests use it. browseLimits archives.StreamOptions @@ -254,6 +255,9 @@ func (s *Server) serve(listener net.Listener) error { return fmt.Errorf("configuring cooldown policy: %w", err) } proxy.Cooldown = cooldownPolicy + if s.cooldown, err = newCooldownView(s.cfg.Cooldown, cooldownPolicy); err != nil { + return err + } policy, err := denylist.New(s.cfg.Denylist.Packages) if err != nil { return fmt.Errorf("configuring denylist: %w", err) @@ -832,6 +836,7 @@ func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) { TotalVersions: stats.TotalVersions, }, EnrichmentStats: enrichmentStatsView(enrichStats), + Cooldown: s.cooldown.summary(), } for _, p := range popular { @@ -1169,6 +1174,7 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, Versions: versions, Vulnerabilities: vulns, LicenseCategory: categorizeLicense(pkg.License), + Cooldown: s.cooldown.forPackage(pkg.Ecosystem, pkg.Name), } if err := s.templates.Render(w, "package_show", data); err != nil { diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index e48e205f..eb4c786a 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -25,6 +25,21 @@ {{template "security_overview" .EnrichmentStats}} {{end}} +{{with .Cooldown}}{{if .Enabled}} +
+

Cooldown

+

Versions published more recently than this are left out of package metadata.

+
+
Default:
{{.Default}}
+ {{range .Ecosystems}} +
{{.Ecosystem}}:
{{.Duration}}
+ {{end}} + {{if .PackageOverrides}}
Package overrides:
{{.PackageOverrides}}
{{end}} + {{if .PackagePatterns}}
Package patterns:
{{.PackagePatterns}}
{{end}} +
+
+{{end}}{{end}} +
diff --git a/internal/server/templates/pages/package_show.html b/internal/server/templates/pages/package_show.html index a2724a89..b0dd29a6 100644 --- a/internal/server/templates/pages/package_show.html +++ b/internal/server/templates/pages/package_show.html @@ -18,6 +18,9 @@

Package Info

{{if .Package.LatestVersion.Valid}}
Latest Version:
{{.Package.LatestVersion.String}}
{{end}} + {{with .Cooldown}} +
Cooldown:
{{.Duration}} ({{.Rule}})
+ {{end}} {{if .Package.License.Valid}}
License:
{{.Package.License.String}}
{{end}} From 16bd0e5a59371c13773f9871cb76a7a0b3b3da53 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Wed, 28 Oct 2026 12:50:57 +0000 Subject: [PATCH 2/2] Drop FillVersionPublishedAt until it has a caller --- internal/database/database_test.go | 42 ------------------------------ internal/database/queries.go | 30 --------------------- 2 files changed, 72 deletions(-) diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 09f5a93d..f8956a63 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1112,48 +1112,6 @@ func BenchmarkMigrateSchemaFullyMigrated(b *testing.B) { } } -func TestFillVersionPublishedAt(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - const purl, pkg = "pkg:npm/leftpad@1.0.0", "pkg:npm/leftpad" - first := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) - publishedAt := func() sql.NullTime { - t.Helper() - got, err := db.GetVersionByPURL(purl) - if err != nil || got == nil { - t.Fatalf("GetVersionByPURL: %v, %v", got, err) - } - return got.PublishedAt - } - - // A row without a time, as the artifact cache writes it, is filled. - if err := db.UpsertVersion(&Version{PURL: purl, PackagePURL: pkg}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - if err := db.FillVersionPublishedAt(purl, pkg, first.In(time.FixedZone("CEST", 2*60*60))); err != nil { - t.Fatalf("FillVersionPublishedAt: %v", err) - } - if got := publishedAt(); !got.Valid || !got.Time.Equal(first) { - t.Fatalf("PublishedAt = %v, want %v", got, first) - } - - // A stored time is kept. - if err := db.FillVersionPublishedAt(purl, pkg, first.Add(time.Hour)); err != nil { - t.Fatalf("FillVersionPublishedAt again: %v", err) - } - if got := publishedAt(); !got.Time.Equal(first) { - t.Fatalf("PublishedAt after a second fill = %v, want %v kept", got.Time, first) - } - - // A version the proxy hasn't seen gets a row. - if err := db.FillVersionPublishedAt("pkg:npm/leftpad@2.0.0", pkg, first); err != nil { - t.Fatalf("FillVersionPublishedAt new: %v", err) - } - if got, err := db.GetVersionByPURL("pkg:npm/leftpad@2.0.0"); err != nil || got == nil || !got.PublishedAt.Time.Equal(first) { - t.Fatalf("new version = %+v, %v, want published %v", got, err, first) - } - }) -} - func TestVersionPublishedAtPreserved(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { publishedAt := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) diff --git a/internal/database/queries.go b/internal/database/queries.go index 0bd5e821..d7150e80 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -223,36 +223,6 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } -// FillVersionPublishedAt records a version's publish time, in UTC, only where -// none is stored yet, creating the versions row if needed. Callers that see -// the same version on every metadata request write it once without reading -// first. -func (db *DB) FillVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { - now := time.Now() - query := ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(purl) DO UPDATE SET - published_at = excluded.published_at, - updated_at = excluded.updated_at - WHERE versions.published_at IS NULL - ` - if db.dialect == DialectPostgres { - query = ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT(purl) DO UPDATE SET - published_at = EXCLUDED.published_at, - updated_at = EXCLUDED.updated_at - WHERE versions.published_at IS NULL - ` - } - if _, err := db.Exec(query, versionPURL, packagePURL, publishedAt.UTC(), now, now); err != nil { - return fmt.Errorf("filling version publish time: %w", err) - } - return nil -} - // Artifact queries func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) {