diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b23..b0dee951 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1076,6 +1076,8 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from. + Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go and Maven. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. diff --git a/internal/server/cooldown_view.go b/internal/server/cooldown_view.go new file mode 100644 index 00000000..57a64ecb --- /dev/null +++ b/internal/server/cooldown_view.go @@ -0,0 +1,122 @@ +package server + +import ( + "fmt" + "sort" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/packageurl" +) + +// CooldownSummary describes the cooldown configuration on the dashboard. The +// UI has no auth, so package overrides and patterns are only counted: their +// names often list internal packages. +type CooldownSummary struct { + Enabled bool + Default string + Ecosystems []CooldownOverride + PackageOverrides int + PackagePatterns int +} + +// CooldownOverride is one ecosystem's cooldown. +type CooldownOverride struct { + Ecosystem string + Duration string +} + +// PackageCooldown is the cooldown that applies to one package, and the kind +// of rule it comes from. +type PackageCooldown struct { + Duration string + Rule string +} + +// cooldownView answers the UI's cooldown questions from the configuration. +type cooldownView struct { + cfg config.CooldownConfig + policy *cooldown.Policy + packages map[string]string + // patterns sets every package pattern to the same non-zero duration over + // a zero default, so a non-zero result means a pattern matched, with the + // policy's own matching. + patterns *cooldown.Policy +} + +func newCooldownView(cfg config.CooldownConfig, policy *cooldown.Policy) (*cooldownView, error) { + probe := make(map[string]string, len(cfg.PackagePatterns)) + for glob := range cfg.PackagePatterns { + probe[glob] = "1h" + } + patterns, err := cooldown.NewPolicy(nil, probe) + if err != nil { + return nil, fmt.Errorf("configuring cooldown package patterns: %w", err) + } + return &cooldownView{cfg: cfg, policy: policy, packages: cfg.NormalizedPackages(), patterns: patterns}, nil +} + +func (v *cooldownView) enabled() bool { + return v != nil && v.policy != nil && v.policy.Enabled() +} + +// summary returns the dashboard's view of the configuration. +func (v *cooldownView) summary() CooldownSummary { + if !v.enabled() { + return CooldownSummary{} + } + s := CooldownSummary{ + Enabled: true, + Default: formatConfigDuration(v.cfg.Default), + PackageOverrides: len(v.cfg.Packages), + PackagePatterns: len(v.cfg.PackagePatterns), + } + for eco, d := range v.cfg.Ecosystems { + s.Ecosystems = append(s.Ecosystems, CooldownOverride{Ecosystem: eco, Duration: formatConfigDuration(d)}) + } + sort.Slice(s.Ecosystems, func(i, j int) bool { return s.Ecosystems[i].Ecosystem < s.Ecosystems[j].Ecosystem }) + return s +} + +// forPackage returns the cooldown for a package, or nil with cooldown off. +func (v *cooldownView) forPackage(ecosystem, name string) *PackageCooldown { + if !v.enabled() { + return nil + } + purl := packageurl.MakeString(ecosystem, name, "") + rule := "default" + if _, ok := v.packages[purl]; ok { + rule = "package override" + } else if v.patterns.For(ecosystem, purl) != 0 { + rule = "package pattern" + } else if _, ok := v.cfg.Ecosystems[ecosystem]; ok { + rule = "ecosystem override" + } + return &PackageCooldown{Duration: formatCooldown(v.policy.For(ecosystem, purl)), Rule: rule} +} + +func formatConfigDuration(s string) string { + d, err := cooldown.ParseDuration(s) + if err != nil { + return s + } + return formatCooldown(d) +} + +// formatCooldown writes d as the configuration does: "3d", "12h", "30m", +// and "off" for zero. +func formatCooldown(d time.Duration) string { + const day = 24 * time.Hour + switch { + case d <= 0: + return "off" + case d%day == 0: + return fmt.Sprintf("%dd", d/day) + case d%time.Hour == 0: + return fmt.Sprintf("%dh", d/time.Hour) + case d%time.Minute == 0: + return fmt.Sprintf("%dm", d/time.Minute) + } + return d.String() +} diff --git a/internal/server/cooldown_view_test.go b/internal/server/cooldown_view_test.go new file mode 100644 index 00000000..41ff789b --- /dev/null +++ b/internal/server/cooldown_view_test.go @@ -0,0 +1,137 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" +) + +func testCooldownView(t *testing.T, cfg config.CooldownConfig) *cooldownView { + t.Helper() + policy, err := cooldown.NewPolicy(&cooldown.Config{ + Default: cfg.Default, + Ecosystems: cfg.Ecosystems, + Packages: cfg.NormalizedPackages(), + }, cfg.PackagePatterns) + if err != nil { + t.Fatalf("NewPolicy: %v", err) + } + v, err := newCooldownView(cfg, policy) + if err != nil { + t.Fatalf("newCooldownView: %v", err) + } + return v +} + +func TestCooldownViewForPackage(t *testing.T) { + v := testCooldownView(t, config.CooldownConfig{ + Default: "3d", + Ecosystems: map[string]string{"npm": "7d", "composer": "48h"}, + Packages: map[string]string{"pkg:composer/phpunit/phpunit": "0"}, + PackagePatterns: map[string]string{"pkg:npm/@corp/*": "0", "pkg:composer/acme/*": "12h"}, + }) + tests := []struct { + ecosystem, name string + want PackageCooldown + }{ + {"cargo", "serde", PackageCooldown{"3d", "default"}}, + {"npm", "lodash", PackageCooldown{"7d", "ecosystem override"}}, + {"composer", "monolog/monolog", PackageCooldown{"2d", "ecosystem override"}}, + {"composer", "phpunit/phpunit", PackageCooldown{"off", "package override"}}, + {"composer", "acme/tool", PackageCooldown{"12h", "package pattern"}}, + {"npm", "@corp/ui", PackageCooldown{"off", "package pattern"}}, + } + for _, tt := range tests { + got := v.forPackage(tt.ecosystem, tt.name) + if got == nil || *got != tt.want { + t.Errorf("forPackage(%s, %s) = %+v, want %+v", tt.ecosystem, tt.name, got, tt.want) + } + } +} + +func TestCooldownViewOff(t *testing.T) { + var unset *cooldownView + for _, v := range []*cooldownView{unset, testCooldownView(t, config.CooldownConfig{})} { + if got := v.forPackage("npm", "lodash"); got != nil { + t.Errorf("forPackage with cooldown off = %+v, want nil", got) + } + if got := v.summary(); got.Enabled { + t.Errorf("summary with cooldown off = %+v, want disabled", got) + } + } +} + +// TestCooldownViewOnlyPattern checks that a pattern alone turns cooldown on. +func TestCooldownViewOnlyPattern(t *testing.T) { + v := testCooldownView(t, config.CooldownConfig{PackagePatterns: map[string]string{"pkg:npm/*": "1d"}}) + if got := v.forPackage("npm", "lodash"); got == nil || *got != (PackageCooldown{"1d", "package pattern"}) { + t.Errorf("forPackage = %+v, want 1d from the pattern", got) + } + if got := v.forPackage("cargo", "serde"); got == nil || *got != (PackageCooldown{"off", "default"}) { + t.Errorf("forPackage outside the pattern = %+v, want off by default", got) + } +} + +func TestFormatCooldown(t *testing.T) { + for d, want := range map[time.Duration]string{ + 0: "off", + 72 * time.Hour: "3d", + 36 * time.Hour: "36h", + 90 * time.Minute: "90m", + 90*time.Minute + 30*time.Second: "1h30m30s", + } { + if got := formatCooldown(d); got != want { + t.Errorf("formatCooldown(%v) = %q, want %q", d, got, want) + } + } +} + +// TestCooldownPages checks that the dashboard shows the configuration +// without the names of package overrides and patterns, and the package page +// the cooldown that applies to the package. +func TestCooldownPages(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.cooldown = testCooldownView(t, config.CooldownConfig{ + Default: "3d", + Ecosystems: map[string]string{"npm": "7d"}, + Packages: map[string]string{"pkg:npm/internal-secret": "0"}, + PackagePatterns: map[string]string{"pkg:npm/@corp-secret/*": "0"}, + }) + if err := ts.db.UpsertPackage(&database.Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + + get := func(path string) string { + t.Helper() + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusOK { + t.Fatalf("GET %s: status %d", path, w.Code) + } + return w.Body.String() + } + + dashboard := get("/ui/") + for _, want := range []string{"Cooldown", "3d", "npm:", "7d", "Package overrides:", "Package patterns:"} { + if !strings.Contains(dashboard, want) { + t.Errorf("dashboard is missing %q", want) + } + } + for _, secret := range []string{"internal-secret", "corp-secret"} { + if strings.Contains(dashboard, secret) { + t.Errorf("dashboard shows %q", secret) + } + } + + page := get("/ui/package/npm/lodash") + if !strings.Contains(page, "7d") || !strings.Contains(page, "(ecosystem override)") { + t.Error("package page doesn't show the 7d ecosystem override") + } +} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index da866a6a..7e14af6c 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -13,6 +13,7 @@ type DashboardData struct { Layout Stats DashboardStats EnrichmentStats EnrichmentStatsView + Cooldown CooldownSummary RecentPackages []PackageInfo PopularPackages []PackageInfo } @@ -68,6 +69,8 @@ type PackageShowData struct { Versions []database.Version Vulnerabilities []database.Vulnerability LicenseCategory string + // Cooldown is nil with cooldown off. + Cooldown *PackageCooldown } // VersionShowData contains data for rendering the version show page. diff --git a/internal/server/server.go b/internal/server/server.go index 12619a6a..ea87e082 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -126,6 +126,7 @@ type Server struct { sources sourceTracker scheduler *mirror.Scheduler git *gitcache.Cache + cooldown *cooldownView // browseLimits overrides defaultBrowseLimits when set; tests use it. browseLimits archives.StreamOptions @@ -254,6 +255,9 @@ func (s *Server) serve(listener net.Listener) error { return fmt.Errorf("configuring cooldown policy: %w", err) } proxy.Cooldown = cooldownPolicy + if s.cooldown, err = newCooldownView(s.cfg.Cooldown, cooldownPolicy); err != nil { + return err + } policy, err := denylist.New(s.cfg.Denylist.Packages) if err != nil { return fmt.Errorf("configuring denylist: %w", err) @@ -832,6 +836,7 @@ func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) { TotalVersions: stats.TotalVersions, }, EnrichmentStats: enrichmentStatsView(enrichStats), + Cooldown: s.cooldown.summary(), } for _, p := range popular { @@ -1169,6 +1174,7 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, Versions: versions, Vulnerabilities: vulns, LicenseCategory: categorizeLicense(pkg.License), + Cooldown: s.cooldown.forPackage(pkg.Ecosystem, pkg.Name), } if err := s.templates.Render(w, "package_show", data); err != nil { diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index e48e205f..eb4c786a 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -25,6 +25,21 @@ {{template "security_overview" .EnrichmentStats}} {{end}} +{{with .Cooldown}}{{if .Enabled}} +
+

Cooldown

+

Versions published more recently than this are left out of package metadata.

+
+
Default:
{{.Default}}
+ {{range .Ecosystems}} +
{{.Ecosystem}}:
{{.Duration}}
+ {{end}} + {{if .PackageOverrides}}
Package overrides:
{{.PackageOverrides}}
{{end}} + {{if .PackagePatterns}}
Package patterns:
{{.PackagePatterns}}
{{end}} +
+
+{{end}}{{end}} +
diff --git a/internal/server/templates/pages/package_show.html b/internal/server/templates/pages/package_show.html index a2724a89..b0dd29a6 100644 --- a/internal/server/templates/pages/package_show.html +++ b/internal/server/templates/pages/package_show.html @@ -18,6 +18,9 @@

Package Info

{{if .Package.LatestVersion.Valid}}
Latest Version:
{{.Package.LatestVersion.String}}
{{end}} + {{with .Cooldown}} +
Cooldown:
{{.Duration}} ({{.Rule}})
+ {{end}} {{if .Package.License.Valid}}
License:
{{.Package.License.String}}
{{end}}