diff --git a/docs/configuration.md b/docs/configuration.md index e1a4f9ef..b20685f0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1119,6 +1119,8 @@ Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to Package PURL keys are normalized to canonical form before matching, so `pkg:npm/@babel/core` and `pkg:npm/%40babel/core` are equivalent, as are `pkg:pypi/Django` and `pkg:pypi/django`. If both forms configure the same package, the canonical entry wins. +Go module names in exact cooldown overrides are matched case-insensitively. If both lowercase and mixed-case keys are configured, the lowercase entry wins. Go package patterns match lowercase module names. + `package_patterns` matches canonical, versionless PURLs using `*` for zero or more characters and `?` for one character. Neither wildcard crosses `/` separators. Character classes (`[...]`) and backslash escapes are rejected at startup. For example, `"pkg:npm/@example/*"` matches packages under the `@example` npm scope. Patterns accept `@` as an alias for `%40`; other characters must use their canonical PURL form. Equivalent patterns with different durations are rejected at startup. Equal durations, such as `1d` and `24h`, are accepted. Exact `packages` entries take precedence over patterns. When several patterns match, longer patterns win after excluding `*` and `?` from the length. Ties use lexical order of the normalized patterns. @@ -1180,6 +1182,7 @@ Names and versions are compared the way each ecosystem treats them: `pkg:composer/symfony/console@6.1.7` also covers `v6.1.7`. - Go versions are matched as written, `pkg:golang/example.com/mod@v1.0.0-RC1`, though download URLs escape uppercase letters (`v1.0.0-!r!c1`). + Go module names in denylist entries are matched case-insensitively. - A Swift entry names the registry scope and package, ignoring case: `pkg:swift/apple/swift-nio@2.62.0`. The stored form, `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too. diff --git a/go.mod b/go.mod index 4a164c29..2ce65e22 100644 --- a/go.mod +++ b/go.mod @@ -12,12 +12,12 @@ require ( github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.0 - github.com/git-pkgs/enrichment v0.7.4 + github.com/git-pkgs/enrichment v0.7.5 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.6.0 - github.com/git-pkgs/purl v0.1.24 - github.com/git-pkgs/registries v0.12.0 + github.com/git-pkgs/purl v0.2.1 + github.com/git-pkgs/registries v0.13.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 983c87e1..c3221612 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= -github.com/git-pkgs/enrichment v0.7.4 h1:owiCKjITG8tNxBikcyUCDuDf1i3C/+Yoa5yJseuqIjA= -github.com/git-pkgs/enrichment v0.7.4/go.mod h1:ZN/cH4PnmzsapEVPvmAwREP7+ox+RX0yfR6KvoB8xKE= +github.com/git-pkgs/enrichment v0.7.5 h1:jbfFzf7+HSS7el8AuMri4vHXq2izIIZDVIU0uYI5LFM= +github.com/git-pkgs/enrichment v0.7.5/go.mod h1:TbxC/nGQN96YCbdoiDs8a9z3gk2zXmLVS1989/jvnsU= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZOxM= @@ -133,10 +133,10 @@ github.com/git-pkgs/magic v0.6.0 h1:fINPe67FEIaFRVlZzhOIDClMkm+gftBroBENa8bHSxQ= github.com/git-pkgs/magic v0.6.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= -github.com/git-pkgs/purl v0.1.24 h1:JMEB1uNWI+baEpl6AeVx+Kfq1c+xkou+THxa5KJ3MrU= -github.com/git-pkgs/purl v0.1.24/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.12.0 h1:fhxYaR5iR9Zlx6doLkjlXurVqPxkYcXmNFiIrFeV3/k= -github.com/git-pkgs/registries v0.12.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= +github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= +github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= +github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= diff --git a/internal/config/config.go b/internal/config/config.go index 5e901505..c09a1bae 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -77,6 +77,7 @@ import ( "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" "gopkg.in/yaml.v3" @@ -261,7 +262,7 @@ func (c *CooldownConfig) NormalizedPackages() map[string]string { for _, key := range keys { canonical := key if parsed, err := purl.Parse(key); err == nil { - canonical = parsed.String() + canonical = packageurl.PolicyPackageKey(parsed.String()) } if _, exists := normalized[canonical]; exists && key != canonical { continue diff --git a/internal/database/rekey_artifact.go b/internal/database/rekey_artifact.go new file mode 100644 index 00000000..68a8dbd4 --- /dev/null +++ b/internal/database/rekey_artifact.go @@ -0,0 +1,40 @@ +package database + +import ( + "fmt" + "time" +) + +// RekeyCachedArtifact moves one unchanged upstream record without replacing +// an existing destination. Access history and stored bytes are preserved. +func (db *DB) RekeyCachedArtifact(old *Artifact, pkg *Package, version *Version) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + now := time.Now() + _, err = tx.Exec(db.Rebind(`INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(purl) DO NOTHING`), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now) + if err != nil { + return false, fmt.Errorf("creating artifact package: %w", err) + } + _, err = tx.Exec(db.Rebind(`INSERT INTO versions (purl, package_purl, created_at, updated_at) + VALUES (?, ?, ?, ?) ON CONFLICT(purl) DO NOTHING`), version.PURL, version.PackagePURL, now, now) + if err != nil { + return false, fmt.Errorf("creating artifact version: %w", err) + } + result, err := tx.Exec(db.Rebind(`UPDATE artifacts SET version_purl = ?, updated_at = ? + WHERE id = ? AND version_purl = ? AND storage_path = ? AND upstream_url = ? AND source = ? + AND NOT EXISTS (SELECT 1 FROM artifacts WHERE version_purl = ? AND filename = ?)`), + version.PURL, now, old.ID, old.VersionPURL, old.StoragePath, old.UpstreamURL, SourceUpstream, + version.PURL, old.Filename) + if err != nil { + return false, fmt.Errorf("moving cached artifact: %w", err) + } + count, err := result.RowsAffected() + if err != nil || count == 0 { + return false, err + } + return true, tx.Commit() +} diff --git a/internal/database/rekey_artifact_test.go b/internal/database/rekey_artifact_test.go new file mode 100644 index 00000000..d5fd2704 --- /dev/null +++ b/internal/database/rekey_artifact_test.go @@ -0,0 +1,69 @@ +package database + +import ( + "database/sql" + "testing" +) + +func TestRekeyCachedArtifact(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + for _, scenario := range []string{"move", "destination exists", "source changed", "local source"} { + t.Run(scenario, func(t *testing.T) { + oldPackage := &Package{PURL: "pkg:golang/example.com/old", Ecosystem: "golang", Name: "example.com/Old"} + oldVersion := &Version{PURL: oldPackage.PURL + "@v1.0.0", PackagePURL: oldPackage.PURL} + old := &Artifact{VersionPURL: oldVersion.PURL, Filename: scenario + ".zip", + UpstreamURL: "https://proxy.example/example.com/!old/@v/v1.0.0.zip", + StoragePath: sql.NullString{String: scenario + "/old.zip", Valid: true}} + if err := db.RecordCachedArtifact(oldPackage, oldVersion, old); err != nil { + t.Fatal(err) + } + old, err := db.GetArtifact(oldVersion.PURL, old.Filename) + if err != nil || old == nil { + t.Fatalf("source artifact = %+v, %v", old, err) + } + pkg := &Package{PURL: "pkg:golang/example.com/Old", Ecosystem: "golang", Name: "example.com/Old"} + version := &Version{PURL: pkg.PURL + "@v1.0.0", PackagePURL: pkg.PURL} + prepareRekeyConflict(t, db, scenario, old, pkg, version) + moved, err := db.RekeyCachedArtifact(old, pkg, version) + if err != nil || moved != (scenario == "move") { + t.Fatalf("RekeyCachedArtifact = %v, %v", moved, err) + } + checkRekeyResult(t, db, scenario, old, version) + }) + } + }) +} + +func checkRekeyResult(t *testing.T, db *DB, scenario string, old *Artifact, version *Version) { + t.Helper() + current, err := db.GetArtifact(version.PURL, old.Filename) + if err != nil { + t.Fatal(err) + } + if scenario == "move" && (current == nil || current.ID != old.ID || current.StoragePath != old.StoragePath) { + t.Errorf("moved artifact = %+v, old = %+v", current, old) + } + if scenario == "destination exists" && (current == nil || current.StoragePath.String != "destination.zip") { + t.Errorf("destination replaced: %+v", current) + } + if paths := allPendingDeletes(t, db); len(paths) != 0 { + t.Errorf("rekey queued stored bytes for deletion: %v", paths) + } +} + +func prepareRekeyConflict(t *testing.T, db *DB, scenario string, old *Artifact, pkg *Package, version *Version) { + t.Helper() + var err error + switch scenario { + case "destination exists": + err = db.RecordCachedArtifact(pkg, version, &Artifact{VersionPURL: version.PURL, Filename: old.Filename, + StoragePath: sql.NullString{String: "destination.zip", Valid: true}}) + case "source changed": + _, err = db.Exec(db.Rebind(`UPDATE artifacts SET storage_path = ? WHERE id = ?`), "replacement.zip", old.ID) + case "local source": + _, err = db.Exec(db.Rebind(`UPDATE artifacts SET source = ? WHERE id = ?`), SourceLocal, old.ID) + } + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go index 15bb9e5e..76f2de5f 100644 --- a/internal/denylist/denylist.go +++ b/internal/denylist/denylist.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" ) @@ -138,5 +139,5 @@ func identify(pkg *purl.PURL) identity { channel = c.Qualifier("channel") } key := purl.New(c.Type, c.Namespace, c.Name, "", nil).String() - return identity{key: key, channel: channel, version: c.Version} + return identity{key: packageurl.PolicyPackageKey(key), channel: channel, version: c.Version} } diff --git a/internal/enrichment/enrichment.go b/internal/enrichment/enrichment.go index 4d847af5..2246e394 100644 --- a/internal/enrichment/enrichment.go +++ b/internal/enrichment/enrichment.go @@ -100,9 +100,6 @@ func (s *Service) BulkEnrichPackages(ctx context.Context, packages []struct{ Eco // CheckVulnerabilities queries for vulnerabilities affecting a package version. func (s *Service) CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]VulnInfo, error) { - if packageurl.Make(ecosystem, name, version) == nil { - return nil, nil - } return s.vulns.Check(ctx, ecosystem, name, version) } @@ -137,7 +134,8 @@ type EnrichmentResult struct { } // EnrichFull fetches package, version and vulnerability data for a package -// version concurrently. Failed lookups are logged and left empty. +// version concurrently. Unsupported vulnerability identities return an error; +// other failed lookups are logged and left empty. func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version string) (*EnrichmentResult, error) { result := &EnrichmentResult{} @@ -147,6 +145,9 @@ func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version strin wg.Go(func() { result.Version, verErr = s.EnrichVersion(ctx, ecosystem, name, version) }) wg.Go(func() { result.Vulnerabilities, vulnErr = s.CheckVulnerabilities(ctx, ecosystem, name, version) }) wg.Wait() + if errors.Is(vulnErr, shared.ErrUnsupportedVulnerabilityIdentity) { + return nil, vulnErr + } if pkgErr != nil { s.logger.Debug("failed to enrich package", "ecosystem", ecosystem, "name", name, "error", pkgErr) diff --git a/internal/enrichment/enrichment_test.go b/internal/enrichment/enrichment_test.go index a854e5a8..f01bb291 100644 --- a/internal/enrichment/enrichment_test.go +++ b/internal/enrichment/enrichment_test.go @@ -2,10 +2,12 @@ package enrichment import ( "context" + "errors" "log/slog" "os" "testing" + shared "github.com/git-pkgs/enrichment" "github.com/git-pkgs/registries" ) @@ -26,7 +28,7 @@ func TestNew(t *testing.T) { } } -func TestSwiftRegistryIdentitySkipsPURLDependentLookups(t *testing.T) { +func TestSwiftRegistryIdentityLookups(t *testing.T) { svc := New(slog.New(slog.NewTextHandler(os.Stdout, nil))) ctx := context.Background() @@ -41,8 +43,11 @@ func TestSwiftRegistryIdentitySkipsPURLDependentLookups(t *testing.T) { } vulnerabilities, err := svc.CheckVulnerabilities(ctx, "swift", "apple/example", "1.2.3") - if err != nil || vulnerabilities != nil { - t.Errorf("CheckVulnerabilities() = %#v, %v; want nil, nil", vulnerabilities, err) + if !errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) || vulnerabilities != nil { + t.Errorf("CheckVulnerabilities() = %#v, %v; want unsupported identity", vulnerabilities, err) + } + if _, err := svc.EnrichFull(ctx, "swift", "apple/example", "1.2.3"); !errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + t.Errorf("EnrichFull error = %v, want unsupported identity", err) } latest, err := svc.GetLatestVersion(ctx, "swift", "apple/example") diff --git a/internal/handler/cooldown_expiry.go b/internal/handler/cooldown_expiry.go index 1163bde9..c31ace8e 100644 --- a/internal/handler/cooldown_expiry.go +++ b/internal/handler/cooldown_expiry.go @@ -5,6 +5,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/packageurl" ) // A rewrite made with cooldown on stays right until a version it held back @@ -127,6 +128,7 @@ type recordingCooldown struct { } func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { + packagePURL = packageurl.PolicyPackageKey(packagePURL) if r.p.Cooldown.IsAllowed(ecosystem, packagePURL, publishedAt) { return true } @@ -135,6 +137,7 @@ func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt } func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision { + packagePURL = packageurl.PolicyPackageKey(packagePURL) d := r.p.Cooldown.Evaluate(ecosystem, packagePURL, publishedAt, evaluatedAt) if !d.Allowed { r.p.withheld(d.AvailableAt) @@ -143,7 +146,7 @@ func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, } func (r recordingCooldown) For(ecosystem, packagePURL string) time.Duration { - return r.p.Cooldown.For(ecosystem, packagePURL) + return r.p.Cooldown.For(ecosystem, packageurl.PolicyPackageKey(packagePURL)) } func (r recordingCooldown) Enabled() bool { @@ -156,7 +159,7 @@ func (r recordingCooldown) availableAt(ecosystem, packagePURL string, publishedA if publishedAt.IsZero() { return time.Time{} } - d := r.p.Cooldown.For(ecosystem, packagePURL) + d := r.For(ecosystem, packagePURL) if d <= 0 { return time.Time{} } diff --git a/internal/handler/cooldown_fail_closed.go b/internal/handler/cooldown_fail_closed.go index da600c1e..0dd2c360 100644 --- a/internal/handler/cooldown_fail_closed.go +++ b/internal/handler/cooldown_fail_closed.go @@ -5,7 +5,7 @@ import "net/http" // cooldownApplies reports whether cooldown withholds new versions of the // package, taking package overrides and exemptions into account. func (p *Proxy) cooldownApplies(ecosystem, name string) bool { - return p.cooldownEnabled() && p.Cooldown.For(ecosystem, canonicalPackagePURL(ecosystem, name)) > 0 + return p.cooldownEnabled() && p.cooldown().For(ecosystem, canonicalPackagePURL(ecosystem, name)) > 0 } // mustFilterMetadata reports whether the package's metadata may have diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go index bb41e38a..663d0c3d 100644 --- a/internal/handler/denylist_test.go +++ b/internal/handler/denylist_test.go @@ -288,6 +288,12 @@ func TestDenylistMatchesEcosystemSpellings(t *testing.T) { {"go case-escaped version", "pkg:golang/github.com/example/mod@v1.0.0-RC1", "/github.com/example/mod/@v/v1.0.0-!r!c1.zip", "golang", "github.com/example/mod", "v1.0.0-!r!c1", "mod@v1.0.0-!r!c1.zip", func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, + {"go lowercase rule", "pkg:golang/github.com/example/mod@v1.0.0", "/github.com/!example/mod/@v/v1.0.0.zip", + "golang", "github.com/Example/mod", "v1.0.0", goZipFilename("github.com/Example/mod", "v1.0.0"), + func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, + {"go mixed-case rule", "pkg:golang/github.com/Example/mod@v1.0.0", "/github.com/example/mod/@v/v1.0.0.zip", + "golang", "github.com/example/mod", "v1.0.0", "mod@v1.0.0.zip", + func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, {"oci repository path", "pkg:oci/library/alpine@" + ociTestDigest, "/library/alpine/blobs/" + ociTestDigest, "oci", "library/alpine", ociTestDigest, ociTestDigest, func(p *Proxy) http.Handler { diff --git a/internal/handler/go.go b/internal/handler/go.go index 1af00edc..08dae575 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -4,11 +4,15 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" "net/http" + "net/url" "strings" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" gomodule "golang.org/x/mod/module" ) @@ -18,15 +22,13 @@ const ( asciiCaseOffset = 32 // difference between lowercase and uppercase ASCII letters ) -// Module zips are stored under pkg:golang/{module path}, which the PURL -// rules lowercase, so a package rule covers a module whatever case either -// is written in. A major version suffix (/v2) is part of the module path -// and so a module of its own. A key in the escaped form proxy URLs and the -// module cache use (github.com/!azure) does not survive the default -// mapping's round trip and is refused; the handler stores decoded paths, -// so such a key could never match. func init() { - retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: retention.DefaultCanonical("golang")}) + retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: golangRetentionKey}) +} + +func golangRetentionKey(p *purl.PURL) (string, bool) { + key, ok := retention.DefaultCanonical("golang")(p) + return packageurl.PolicyPackageKey(key), ok } // GoHandler handles Go module proxy protocol requests. @@ -200,13 +202,7 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul ServeArtifactRequest(w, r, result) } -// goZipFilename is the name a module zip is cached under. Module paths are -// case-sensitive, but the zip is stored under the module's PURL, which -// lowercases golang names, so github.com/Sirupsen/logrus and -// github.com/sirupsen/logrus share one package. A path with uppercase -// letters therefore gets a hash of its exact spelling in the filename. An -// all-lowercase path keeps the plain name, so existing cache entries stay -// valid. +// Keep the module spelling hash used by caches with lowercased PURL keys. func goZipFilename(module, version string) string { base := lastComponent(module) + "@" + version if module == strings.ToLower(module) { @@ -220,6 +216,40 @@ func goZipFilename(module, version string) string { // filename carries: enough to tell apart paths that differ only in case. const goModuleHashBytes = 8 +// An old cache key may combine case-distinct modules. The upstream path +// identifies which module owns an individual archive. +func (p *Proxy) rekeyLegacyGoArtifact(pkgPURL, versionPURL, filename string) (bool, error) { + if !strings.HasPrefix(versionPURL, "pkg:golang/") { + return false, nil + } + parsed, err := purl.Parse(versionPURL) + if err != nil || parsed.FullName() == strings.ToLower(parsed.FullName()) { + return false, nil + } + name := parsed.FullName() + _, legacyVersion, err := packagePURLStrings("golang", strings.ToLower(name), parsed.Version) + if err != nil { + return false, err + } + record, err := p.DB.GetArtifact(legacyVersion, filename) + if err != nil { + return false, fmt.Errorf("reading legacy Go artifact: %w", err) + } + if record == nil || !record.StoragePath.Valid || record.Source != database.SourceUpstream { + return false, nil + } + escaped, err := gomodule.EscapePath(name) + if err != nil { + return false, nil + } + upstream, err := url.Parse(record.UpstreamURL) + if err != nil || !strings.HasSuffix(upstream.Path, "/"+escaped+"/@v/"+parsed.Version+".zip") { + return false, nil + } + return p.DB.RekeyCachedArtifact(record, &database.Package{PURL: pkgPURL, Ecosystem: "golang", Name: name}, + &database.Version{PURL: versionPURL, PackagePURL: pkgPURL}) +} + // discardOtherModulesZip drops a cached zip that another module stored // under filename. Before module paths kept their case, a module with // uppercase letters was cached under the plain filename of its lowercase diff --git a/internal/handler/go_case_test.go b/internal/handler/go_case_test.go index 9451caa8..91bac444 100644 --- a/internal/handler/go_case_test.go +++ b/internal/handler/go_case_test.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "testing" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/registries/fetch" ) @@ -115,6 +116,74 @@ func TestGoZipFromAnotherUpstreamHostIsKept(t *testing.T) { } } +func TestGoLegacyCacheKeepsCaseDistinctModules(t *testing.T) { + upstream, counts := goCaseUpstream(t) + h, proxy := newGoCaseHandler(t, upstream) + seedGoZip(t, proxy, upstream.URL+lowerLogrusPath, "zip of "+lowerLogrusPath) + old := seedLegacyUpperGoZip(t, proxy, "https://old-proxy.example"+upperLogrusPath, "zip of "+upperLogrusPath) + upstream.Close() + + for range 2 { + for _, path := range []string{upperLogrusPath, lowerLogrusPath} { + if w := getGo(h, path); w.Code != http.StatusOK || w.Body.String() != "zip of "+path { + t.Fatalf("GET %s: status = %d, body = %q", path, w.Code, w.Body) + } + } + } + current, err := proxy.DB.GetArtifact("pkg:golang/github.com/Sirupsen/logrus@v1.0.6", old.Filename) + if err != nil || current == nil { + t.Fatalf("moved artifact = %+v, %v", current, err) + } + if current.ID != old.ID || current.StoragePath != old.StoragePath || current.FetchedAt != old.FetchedAt || current.HitCount != old.HitCount+2 { + t.Errorf("artifact history changed: before=%+v, after=%+v", old, current) + } + legacy, err := proxy.DB.GetArtifact(old.VersionPURL, old.Filename) + if err != nil || legacy != nil { + t.Fatalf("legacy artifact = %+v, %v", legacy, err) + } + for path, count := range counts { + if count.Load() != 0 { + t.Errorf("unexpected upstream request: %s", path) + } + } +} + +func TestGoLegacyCacheRequiresExactModuleOrigin(t *testing.T) { + for _, origin := range []string{"", "https://proxy.example" + lowerLogrusPath} { + t.Run(origin, func(t *testing.T) { + upstream, counts := goCaseUpstream(t) + h, proxy := newGoCaseHandler(t, upstream) + seedLegacyUpperGoZip(t, proxy, origin, "another module") + if w := getGo(h, upperLogrusPath); w.Code != http.StatusOK || w.Body.String() != "zip of "+upperLogrusPath { + t.Fatalf("status = %d, body = %q", w.Code, w.Body) + } + if counts[upperLogrusPath].Load() != 1 { + t.Fatal("archive with unproven identity was reused") + } + }) + } +} + +func seedLegacyUpperGoZip(t *testing.T, proxy *Proxy, upstreamURL, content string) *database.Artifact { + t.Helper() + const packagePURL = "pkg:golang/github.com/sirupsen/logrus" + const versionPURL = packagePURL + "@v1.0.6" + const storagePath = "golang/logrus/v1.0.6/legacy/upper.zip" + filename := goZipFilename("github.com/Sirupsen/logrus", "v1.0.6") + if _, _, err := proxy.Storage.Store(context.Background(), storagePath, strings.NewReader(content)); err != nil { + t.Fatal(err) + } + artifact := testArtifact(content, versionPURL, filename, "application/zip") + if err := proxy.updateCacheDB("golang", "github.com/Sirupsen/logrus", packagePURL, upstreamURL, storagePath, artifact, ""); err != nil { + t.Fatal(err) + } + old, err := proxy.DB.GetArtifact(versionPURL, filename) + if err != nil || old == nil { + t.Fatalf("seeded artifact = %+v, %v", old, err) + } + return old +} + // seedGoZip caches content as github.com/sirupsen/logrus v1.0.6 under the // plain filename, recorded as fetched from upstreamURL. func seedGoZip(t *testing.T, proxy *Proxy, upstreamURL, content string) { diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index 892e77b3..efcc1e19 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -71,12 +71,8 @@ func (h *GoHandler) fetchGoMetadata(ctx context.Context, escapedModule, module, return body, err } -// goTimeKey returns the versions-table PURLs a module's times are stored -// under. golang PURLs lowercase the name, so the key uses the escaped module -// path (github.com/!azure for github.com/Azure), which keeps modules that -// differ only in case apart and is the plain path for lowercase modules. -// Only canonical versions are stored: a query such as a branch name can -// resolve to another commit later. +// Keep escaped module paths in publish-time keys for cache compatibility. +// Only canonical versions are stored; branch queries can resolve differently. func (h *GoHandler) goTimeKey(module, version string) (versionPURL, packagePURL string, ok bool) { if h.proxy.DB == nil || gomodule.CanonicalVersion(version) != version { return "", "", false diff --git a/internal/handler/go_policy_test.go b/internal/handler/go_policy_test.go new file mode 100644 index 00000000..0ce981f1 --- /dev/null +++ b/internal/handler/go_policy_test.go @@ -0,0 +1,47 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" +) + +func TestGoCooldownOverridesIgnoreModuleCase(t *testing.T) { + for _, tc := range []struct { + name, rule, fallback, override string + status int + }{ + {"lowercase restriction", "example.com/module", "0", "7d", http.StatusForbidden}, + {"mixed-case restriction", "example.com/Module", "0", "7d", http.StatusForbidden}, + {"lowercase exemption", "example.com/module", "7d", "0", http.StatusOK}, + {"mixed-case exemption", "example.com/Module", "7d", "0", http.StatusOK}, + } { + t.Run(tc.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".info") { + _ = json.NewEncoder(w).Encode(goInfo{Version: "v1.0.0", Time: time.Now()}) + return + } + _, _ = w.Write([]byte("module zip")) + })) + defer upstream.Close() + h, proxy := newGoCaseHandler(t, upstream) + cfg := config.CooldownConfig{Packages: map[string]string{"pkg:golang/" + tc.rule: tc.override}} + proxy.Cooldown = &cooldown.Config{Default: tc.fallback, Packages: cfg.NormalizedPackages()} + for _, path := range []string{ + "/example.com/!module/@v/v1.0.0.zip", "/example.com/module/@v/v1.0.0.zip", + "/example.com/!module/@v/v1.0.0.info", "/example.com/module/@v/v1.0.0.info", + } { + if response := getGo(h, path); response.Code != tc.status { + t.Errorf("GET %s = %d %s, want %d", path, response.Code, response.Body, tc.status) + } + } + }) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 39510164..4b7ebe72 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -498,6 +498,12 @@ func (p *Proxy) lookupCache(ctx context.Context, pkgPURL, versionPURL, filename if p.StreamArtifacts { return nil, nil } + if cached, err := p.readStored(ctx, pkgPURL, versionPURL, filename); err != nil || cached != nil { + return cached, err + } + if moved, err := p.rekeyLegacyGoArtifact(pkgPURL, versionPURL, filename); err != nil || !moved { + return nil, err + } return p.readStored(ctx, pkgPURL, versionPURL, filename) } diff --git a/internal/packageurl/packageurl.go b/internal/packageurl/packageurl.go index 71109426..e01f4b2a 100644 --- a/internal/packageurl/packageurl.go +++ b/internal/packageurl/packageurl.go @@ -18,6 +18,15 @@ func MakeString(ecosystem, name, version string) string { return purl.MakePURLString(ecosystem, name, version) } +// PolicyPackageKey maps a canonical versionless PURL to its policy key. +// Go policies match module names case-insensitively for compatibility. +func PolicyPackageKey(packagePURL string) string { + if strings.HasPrefix(packagePURL, "pkg:golang/") { + return strings.ToLower(packagePURL) + } + return packagePURL +} + // WithVersionString returns a package PURL with its version replaced. It // returns an empty string when packagePURL is invalid. func WithVersionString(packagePURL, version string) string { @@ -29,25 +38,19 @@ func WithVersionString(packagePURL, version string) string { } // MakeCacheStrings returns package and version PURLs suitable for artifact -// cache records. Swift registry identities use an explicit generic PURL until -// their source repository has been resolved. The result is independent of the -// configured upstream so cache entries survive an upstream.swift change, -// matching every other ecosystem. +// cache records. Swift registry identities keep their generic storage PURLs. +// Cache keys are independent of the configured upstream. func MakeCacheStrings(ecosystem, name, version string) (packagePURL, versionPURL string) { + if purl.NormalizeEcosystem(ecosystem) == "swift" { + if identity, ok := swiftRegistryIdentity(name); ok { + pkg := purl.New("generic", "swift-registry", identity, "", nil) + return pkg.String(), pkg.WithVersion(version).String() + } + } if pkg := Make(ecosystem, name, ""); pkg != nil { return pkg.String(), pkg.WithVersion(version).String() } - if purl.NormalizeEcosystem(ecosystem) != "swift" { - return "", "" - } - - identity, ok := swiftRegistryIdentity(name) - if !ok { - return "", "" - } - - pkg := purl.New("generic", "swift-registry", identity, "", nil) - return pkg.String(), pkg.WithVersion(version).String() + return "", "" } func swiftRegistryIdentity(name string) (string, bool) { diff --git a/internal/packageurl/packageurl_test.go b/internal/packageurl/packageurl_test.go index 8982288c..5a46776c 100644 --- a/internal/packageurl/packageurl_test.go +++ b/internal/packageurl/packageurl_test.go @@ -2,17 +2,12 @@ package packageurl import "testing" -func TestMakeSwiftRegistryIdentityUnsupported(t *testing.T) { - identities := []string{"apple.swift-argument-parser", "apple/swift-argument-parser"} - for _, identity := range identities { - t.Run(identity, func(t *testing.T) { - if got := Make("swift", identity, "1.8.2"); got != nil { - t.Errorf("Make() = %q, want nil", got.String()) - } - if got := MakeString("swift", identity, "1.8.2"); got != "" { - t.Errorf("MakeString() = %q, want empty string", got) - } - }) +func TestMakeSwiftRegistryPathUnsupported(t *testing.T) { + if got := Make("swift", "apple/swift-argument-parser", "1.8.2"); got != nil { + t.Errorf("Make() = %q, want nil", got.String()) + } + if got := MakeString("swift", "apple/swift-argument-parser", "1.8.2"); got != "" { + t.Errorf("MakeString() = %q, want empty string", got) } } @@ -49,9 +44,11 @@ func TestMakeCacheStringsSwiftRegistryIdentity(t *testing.T) { t.Errorf("version PURL = %q, want %q", versionPURL, wantVersion) } - dottedPackage, dottedVersion := MakeCacheStrings("swift", "apple.example", "1.2.3") - if dottedPackage != packagePURL || dottedVersion != versionPURL { - t.Errorf("dotted identity cache PURLs = %q, %q; want %q, %q", dottedPackage, dottedVersion, packagePURL, versionPURL) + for _, identity := range []string{"apple.example", "APPLE.EXAMPLE", "apple/example"} { + dottedPackage, dottedVersion := MakeCacheStrings("Swift", identity, "1.2.3") + if dottedPackage != packagePURL || dottedVersion != versionPURL { + t.Errorf("%q cache PURLs = %q, %q; want %q, %q", identity, dottedPackage, dottedVersion, packagePURL, versionPURL) + } } } diff --git a/internal/retention/retention.go b/internal/retention/retention.go index a8ff8359..df8e5383 100644 --- a/internal/retention/retention.go +++ b/internal/retention/retention.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" ) @@ -41,8 +42,8 @@ type Spec struct { // derived from Key that belong to this ecosystem. ExtraDBEcosystems []string - // CanonicalPackage turns a package PURL from the config into the - // package PURL the handler stores in the cache, reporting false when the + // CanonicalPackage turns a package PURL from the config into a + // policy key, reporting false when the // PURL does not belong to this ecosystem. Nil means the ecosystem accepts // no package overrides. CanonicalPackage func(p *purl.PURL) (string, bool) @@ -150,8 +151,7 @@ func (r *Registry) KeyForDBEcosystem(ecosystem string) (string, bool) { return key, ok } -// Canonical returns the stored package PURL a configured package PURL -// stands for, asking the registered specs in key order. +// Canonical returns the policy key for a configured package PURL. func (r *Registry) Canonical(p *purl.PURL) (string, bool) { for _, key := range r.Keys() { s, _ := r.Lookup(key) @@ -173,7 +173,7 @@ type Rules struct { Default time.Duration // Ecosystems holds rules keyed by registered ecosystem key. Ecosystems map[string]time.Duration - // Packages holds rules keyed by the package PURL stored in the cache. + // Packages holds rules keyed by package PURL, with Go keys lowercased. Packages map[string]time.Duration } @@ -181,6 +181,7 @@ type Rules struct { // stored package PURL: the package rule, else the ecosystem rule, else the // default. Generic artifacts require an explicit package or ecosystem rule. func (r Rules) For(key, packagePURL string) time.Duration { + packagePURL = packageurl.PolicyPackageKey(packagePURL) if d, ok := r.Packages[packagePURL]; ok { return d } diff --git a/internal/server/api.go b/internal/server/api.go index b2043672..22bfb8a1 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -2,6 +2,7 @@ package server import ( "encoding/json" + "errors" "net/http" "strings" @@ -219,6 +220,10 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) { result, err := h.enrichment.EnrichFull(r.Context(), ecosystem, name, version) + if errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + badRequest(w, "package identity is not supported for vulnerability lookups") + return + } if err != nil { writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to enrich version") return @@ -303,6 +308,10 @@ func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { } vulns, err := h.enrichment.CheckVulnerabilities(r.Context(), ecosystem, name, version) + if errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + badRequest(w, "package identity is not supported for vulnerability lookups") + return + } if err != nil { writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to check vulnerabilities") return diff --git a/internal/server/retention_golang_test.go b/internal/server/retention_golang_test.go index f74eaf41..8ff03579 100644 --- a/internal/server/retention_golang_test.go +++ b/internal/server/retention_golang_test.go @@ -188,8 +188,8 @@ func TestGolangRetentionPackageRules(t *testing.T) { if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), zip.VersionPURL); err != nil { t.Fatalf("reading stored package PURL: %v", err) } - if stored != "pkg:golang/github.com/burntsushi/toml" { - t.Errorf("stored package PURL = %q, want pkg:golang/github.com/burntsushi/toml", stored) + if stored != "pkg:golang/github.com/BurntSushi/toml" { + t.Errorf("stored package PURL = %q, want pkg:golang/github.com/BurntSushi/toml", stored) } for _, keys := range []map[string]string{ diff --git a/internal/server/swift_vulns_test.go b/internal/server/swift_vulns_test.go new file mode 100644 index 00000000..f28caaf3 --- /dev/null +++ b/internal/server/swift_vulns_test.go @@ -0,0 +1,57 @@ +package server + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/enrichment" + "github.com/go-chi/chi/v5" +) + +func TestSwiftRegistryVulnerabilityErrors(t *testing.T) { + h := NewAPIHandler(enrichment.New(slog.Default()), nil) + r := chi.NewRouter() + r.Get("/api/vulns/{ecosystem}/*", h.HandleVulnsPath) + r.Get("/api/package/{ecosystem}/*", h.HandlePackagePath) + transport := &swiftVulnsTransport{} + original := http.DefaultTransport + http.DefaultTransport = transport + t.Cleanup(func() { http.DefaultTransport = original }) + + for _, path := range []string{ + "/api/vulns/swift/apple.swift-nio", + "/api/vulns/swift/apple.swift-nio/2.0.0", + "/api/vulns/Swift/Apple.Swift-NIO/2.0.0", + "/api/package/swift/apple.swift-nio/2.0.0", + "/api/package/Swift/Apple.Swift-NIO/2.0.0", + } { + t.Run(path, func(t *testing.T) { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, body = %s", w.Code, w.Body) + } + var response ErrorResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.Code != ErrCodeBadRequest || response.Message != "package identity is not supported for vulnerability lookups" { + t.Fatalf("unexpected response: %+v", response) + } + }) + } + if transport.requests != 0 { + t.Errorf("made %d upstream requests for unsupported identities", transport.requests) + } +} + +type swiftVulnsTransport struct{ requests int } + +func (s *swiftVulnsTransport) RoundTrip(r *http.Request) (*http.Response, error) { + s.requests++ + return nil, fmt.Errorf("unexpected upstream request: %s", r.URL) +}