From 29863e858a62d5c313e97eec627721475023dcd8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 13:53:37 +0100 Subject: [PATCH 01/15] Refresh the most used packages in the background mirror.refresh mirrors, for the top packages per ecosystem by recent downloads, the newest release of each major line a version is cached on. Releases within cooldown or on the denylist are skipped, and PyPI and conda get only the kinds of files clients fetched before. max_versions and max_bytes cap one run. Instances sharing a database take turns as for schedules. --- go.mod | 2 +- internal/config/config.go | 2 + internal/config/mirror.go | 99 ++++++++ internal/config/mirror_test.go | 81 ++++++ internal/database/most_used.go | 78 ++++++ internal/database/most_used_test.go | 94 +++++++ internal/handler/release_allowed.go | 19 ++ internal/handler/release_allowed_test.go | 44 ++++ internal/mirror/files.go | 54 ++++ internal/mirror/refresh.go | 138 ++++++++++ internal/mirror/refresh_test.go | 306 +++++++++++++++++++++++ internal/mirror/refresh_versions.go | 67 +++++ internal/mirror/refresh_versions_test.go | 79 ++++++ internal/mirror/schedule.go | 15 +- internal/mirror/schedule_test.go | 33 +++ internal/mirror/source.go | 3 + internal/server/mirror_schedule.go | 31 ++- internal/server/mirror_schedule_test.go | 22 ++ 18 files changed, 1158 insertions(+), 9 deletions(-) create mode 100644 internal/database/most_used.go create mode 100644 internal/database/most_used_test.go create mode 100644 internal/handler/release_allowed.go create mode 100644 internal/handler/release_allowed_test.go create mode 100644 internal/mirror/refresh.go create mode 100644 internal/mirror/refresh_test.go create mode 100644 internal/mirror/refresh_versions.go create mode 100644 internal/mirror/refresh_versions_test.go diff --git a/go.mod b/go.mod index 2ce65e22..2b1ee11a 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/git-pkgs/registries v0.13.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 + github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 github.com/klauspost/compress v1.20.3 @@ -76,7 +77,6 @@ require ( github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/pom v0.1.8 // indirect - github.com/git-pkgs/vers v0.7.2 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index c09a1bae..8f5ca1fc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1614,6 +1614,8 @@ func (c *Config) validateCacheArtifacts() error { return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served") case len(c.Mirror.Schedules) > 0: return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror.schedules: mirrored artifacts would never be served") + case c.Mirror.Refresh != nil: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror.refresh: mirrored artifacts would never be served") case c.Hosted.Enabled: return fmt.Errorf("storage.cache_artifacts: false cannot be combined with hosted.enabled: pushed packages would never be served") } diff --git a/internal/config/mirror.go b/internal/config/mirror.go index 7f6c668a..473b7440 100644 --- a/internal/config/mirror.go +++ b/internal/config/mirror.go @@ -15,6 +15,102 @@ type MirrorConfig struct { // Schedules re-mirror container image tags at a fixed interval, so a // moved tag is picked up even when nobody pulls it. Schedules []MirrorSchedule `json:"schedules" yaml:"schedules"` + + // Refresh, when set, keeps the most used packages up to date in the + // background. + Refresh *MirrorRefresh `json:"refresh" yaml:"refresh"` +} + +const ( + defaultMirrorRefreshTop = 50 + defaultMirrorRefreshSince = 14 * 24 * time.Hour +) + +// MirrorRefresh mirrors new releases of the packages clients use most, so +// they are cached before the first client asks for them. +type MirrorRefresh struct { + // Interval is how often the refresh runs, at least one minute (e.g. + // "24h"). + Interval string `json:"interval" yaml:"interval"` + + // Top is how many packages per ecosystem are refreshed, the ones with + // the most downloads among those used within Since. Default: 50. + Top int `json:"top" yaml:"top"` + + // Since is how recently a package must have been downloaded to be + // refreshed (e.g. "14d"). Default: 14d. + Since string `json:"since" yaml:"since"` + + // MaxVersions caps how many new versions one run mirrors. 0 means no + // cap. + MaxVersions int `json:"max_versions" yaml:"max_versions"` + + // MaxBytes caps how much one run downloads (e.g. "2GB"). Empty means no + // cap. + MaxBytes string `json:"max_bytes" yaml:"max_bytes"` + + interval time.Duration + since time.Duration + maxBytes int64 +} + +// IntervalDuration returns the parsed Interval. Validate must run first. +func (r *MirrorRefresh) IntervalDuration() time.Duration { return r.interval } + +// TopPackages returns Top, or its default. +func (r *MirrorRefresh) TopPackages() int { + if r.Top == 0 { + return defaultMirrorRefreshTop + } + return r.Top +} + +// SinceDuration returns the parsed Since, or its default. Validate must run +// first. +func (r *MirrorRefresh) SinceDuration() time.Duration { return r.since } + +// MaxVersionsPerRun returns MaxVersions; 0 means no cap. +func (r *MirrorRefresh) MaxVersionsPerRun() int { return r.MaxVersions } + +// MaxBytesPerRun returns the parsed MaxBytes; 0 means no cap. Validate must +// run first. +func (r *MirrorRefresh) MaxBytesPerRun() int64 { return r.maxBytes } + +func (r *MirrorRefresh) validate() error { + d, err := time.ParseDuration(r.Interval) + if err != nil { + return fmt.Errorf("invalid mirror.refresh.interval %q: %w", r.Interval, err) + } + if d < minMirrorScheduleInterval { + return fmt.Errorf("invalid mirror.refresh.interval %q: must be at least %s", r.Interval, minMirrorScheduleInterval) + } + r.interval = d + if r.Top < 0 { + return fmt.Errorf("invalid mirror.refresh.top %d: must not be negative", r.Top) + } + r.since = defaultMirrorRefreshSince + if r.Since != "" { + since, err := parseDayDuration(r.Since) + if err != nil { + return fmt.Errorf("invalid mirror.refresh.since %q: %w", r.Since, err) + } + if since <= 0 { + return fmt.Errorf("invalid mirror.refresh.since %q: must be positive", r.Since) + } + r.since = since + } + if r.MaxVersions < 0 { + return fmt.Errorf("invalid mirror.refresh.max_versions %d: must not be negative", r.MaxVersions) + } + r.maxBytes = 0 + if r.MaxBytes != "" { + n, err := ParseSize(r.MaxBytes) + if err != nil { + return fmt.Errorf("invalid mirror.refresh.max_bytes %q: %w", r.MaxBytes, err) + } + r.maxBytes = n + } + return nil } // MirrorSchedule is a set of image PURLs mirrored every Interval. @@ -45,6 +141,9 @@ func (m *MirrorConfig) Validate() error { return err } } + if m.Refresh != nil { + return m.Refresh.validate() + } return nil } diff --git a/internal/config/mirror_test.go b/internal/config/mirror_test.go index ae1220dc..2a535851 100644 --- a/internal/config/mirror_test.go +++ b/internal/config/mirror_test.go @@ -87,3 +87,84 @@ mirror: t.Error("Validate() accepted a 30s interval in the second schedule") } } + +func TestMirrorRefreshValidate(t *testing.T) { + tests := []struct { + name string + refresh *MirrorRefresh + wantErr bool + }{ + {name: "defaults", refresh: &MirrorRefresh{Interval: "24h"}}, + {name: "all fields", refresh: &MirrorRefresh{Interval: "12h", Top: 20, Since: "14d", MaxVersions: 100, MaxBytes: "2GB"}}, + {name: "missing interval", refresh: &MirrorRefresh{}, wantErr: true}, + {name: "interval under a minute", refresh: &MirrorRefresh{Interval: "10s"}, wantErr: true}, + {name: "negative top", refresh: &MirrorRefresh{Interval: "1h", Top: -1}, wantErr: true}, + {name: "bad since", refresh: &MirrorRefresh{Interval: "1h", Since: "recently"}, wantErr: true}, + {name: "negative since", refresh: &MirrorRefresh{Interval: "1h", Since: "-1d"}, wantErr: true}, + {name: "negative max_versions", refresh: &MirrorRefresh{Interval: "1h", MaxVersions: -1}, wantErr: true}, + {name: "bad max_bytes", refresh: &MirrorRefresh{Interval: "1h", MaxBytes: "lots"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := MirrorConfig{Refresh: tt.refresh} + err := m.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestMirrorRefreshDefaults(t *testing.T) { + m := MirrorConfig{Refresh: &MirrorRefresh{Interval: "24h"}} + if err := m.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + r := m.Refresh + if r.IntervalDuration() != 24*time.Hour || r.TopPackages() != 50 || r.SinceDuration() != 14*24*time.Hour || + r.MaxVersionsPerRun() != 0 || r.MaxBytesPerRun() != 0 { + t.Errorf("got interval %v top %d since %v max_versions %d max_bytes %d, want 24h 50 336h 0 0", + r.IntervalDuration(), r.TopPackages(), r.SinceDuration(), r.MaxVersionsPerRun(), r.MaxBytesPerRun()) + } +} + +func TestValidateRejectsMirrorRefreshWithoutCachedArtifacts(t *testing.T) { + cfg := Default() + cfg.Mirror.Refresh = &MirrorRefresh{Interval: "24h"} + cfg.Storage.CacheArtifacts = false + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted mirror.refresh with storage.cache_artifacts: false") + } +} + +func TestLoadYAMLMirrorRefresh(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + data := ` +mirror: + refresh: + interval: 12h + top: 10 + since: 7d + max_versions: 200 + max_bytes: 1GB +` + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + r := cfg.Mirror.Refresh + if r == nil { + t.Fatal("mirror.refresh not loaded") + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if r.IntervalDuration() != 12*time.Hour || r.TopPackages() != 10 || r.SinceDuration() != 7*24*time.Hour || + r.MaxVersionsPerRun() != 200 || r.MaxBytesPerRun() != 1<<30 { + t.Errorf("got interval %v top %d since %v max_versions %d max_bytes %d", + r.IntervalDuration(), r.TopPackages(), r.SinceDuration(), r.MaxVersionsPerRun(), r.MaxBytesPerRun()) + } +} diff --git a/internal/database/most_used.go b/internal/database/most_used.go new file mode 100644 index 00000000..c68e8335 --- /dev/null +++ b/internal/database/most_used.go @@ -0,0 +1,78 @@ +package database + +import ( + "time" +) + +// UsedPackage is a package clients download from the cache, with the files +// cached for it. +type UsedPackage struct { + Ecosystem string + Name string + PURL string + // Hits counts the downloads of the files downloaded since the cutoff. + Hits int64 + Files []UsedFile +} + +// UsedFile is a cached file of a UsedPackage. +type UsedFile struct { + Version string + Filename string + UpstreamURL string +} + +// MostUsedPackages returns, for each ecosystem, the top packages with the most +// downloads among the cached files downloaded since cutoff, ordered by +// ecosystem and then by hits. Each lists every file cached for it. Files +// from local sources, which have no upstream, are left out. +func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) { + var rows []struct { + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + PURL string `db:"purl"` + Hits int64 `db:"hits"` + } + err := db.Select(&rows, db.Rebind(` + SELECT ecosystem, name, purl, hits FROM ( + SELECT p.ecosystem, p.name, p.purl, SUM(a.hit_count) AS hits, + ROW_NUMBER() OVER (PARTITION BY p.ecosystem ORDER BY SUM(a.hit_count) DESC, p.name) AS rank + FROM artifacts a + JOIN versions v ON v.purl = a.version_purl + JOIN packages p ON p.purl = v.package_purl + WHERE a.storage_path IS NOT NULL AND a.source = ? + AND COALESCE(a.last_accessed_at, a.fetched_at) >= ? + GROUP BY p.ecosystem, p.name, p.purl + ) ranked + WHERE rank <= ? + ORDER BY ecosystem, hits DESC, name + `), SourceUpstream, cutoff.UTC(), top) + if err != nil { + return nil, err + } + packages := make([]UsedPackage, len(rows)) + for i, row := range rows { + packages[i] = UsedPackage{Ecosystem: row.Ecosystem, Name: row.Name, PURL: row.PURL, Hits: row.Hits} + var files []struct { + VersionPURL string `db:"version_purl"` + Filename string `db:"filename"` + UpstreamURL string `db:"upstream_url"` + } + err := db.Select(&files, db.Rebind(` + SELECT a.version_purl, a.filename, a.upstream_url + FROM artifacts a + JOIN versions v ON v.purl = a.version_purl + WHERE v.package_purl = ? AND a.storage_path IS NOT NULL AND a.source = ? + ORDER BY a.version_purl, a.filename + `), row.PURL, SourceUpstream) + if err != nil { + return nil, err + } + for _, f := range files { + packages[i].Files = append(packages[i].Files, UsedFile{ + Version: VersionFromPURL(f.VersionPURL), Filename: f.Filename, UpstreamURL: f.UpstreamURL, + }) + } + } + return packages, nil +} diff --git a/internal/database/most_used_test.go b/internal/database/most_used_test.go new file mode 100644 index 00000000..8704f904 --- /dev/null +++ b/internal/database/most_used_test.go @@ -0,0 +1,94 @@ +package database + +import ( + "reflect" + "testing" + "time" +) + +// seedUsedArtifact stores a cached artifact with hits, last downloaded at +// accessed. +func seedUsedArtifact(t *testing.T, db *DB, ecosystem, name, version, filename string, hits int64, accessed time.Time) { + t.Helper() + packagePURL := "pkg:" + ecosystem + "/" + name + versionPURL := packagePURL + "@" + version + if err := db.UpsertPackage(&Package{PURL: packagePURL, Ecosystem: ecosystem, Name: name}); err != nil { + t.Fatal(err) + } + if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: packagePURL}); err != nil { + t.Fatal(err) + } + if err := db.UpsertArtifact(&Artifact{VersionPURL: versionPURL, Filename: filename, UpstreamURL: "https://example.test/" + filename}); err != nil { + t.Fatal(err) + } + if err := db.MarkArtifactCached(versionPURL, filename, ecosystem+"/"+filename, "sha256:00", 1, "application/octet-stream"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET hit_count = ?, last_accessed_at = ? WHERE version_purl = ? AND filename = ?"), + hits, accessed.UTC(), versionPURL, filename); err != nil { + t.Fatal(err) + } +} + +func TestMostUsedPackages(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seedUsedArtifact(t, db, "npm", "lodash", "4.17.20", "lodash-4.17.20.tgz", 5, now.Add(-time.Hour)) + seedUsedArtifact(t, db, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", 4, now.Add(-20*24*time.Hour)) + seedUsedArtifact(t, db, "npm", "react", "18.3.1", "react-18.3.1.tgz", 10, now.Add(-time.Hour)) + seedUsedArtifact(t, db, "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", 100, now.Add(-30*24*time.Hour)) + seedUsedArtifact(t, db, "cargo", "serde", "1.0.200", "serde-1.0.200.crate", 1, now.Add(-time.Hour)) + + // Not cached: never counts. + if err := db.UpsertArtifact(&Artifact{VersionPURL: "pkg:npm/react@19.0.0", Filename: "react-19.0.0.tgz", UpstreamURL: "https://example.test/x"}); err != nil { + t.Fatal(err) + } + + got, err := db.MostUsedPackages(now.Add(-14*24*time.Hour), 2) + if err != nil { + t.Fatal(err) + } + want := []UsedPackage{ + {Ecosystem: "cargo", Name: "serde", PURL: "pkg:cargo/serde", Hits: 1, Files: []UsedFile{ + {Version: "1.0.200", Filename: "serde-1.0.200.crate", UpstreamURL: "https://example.test/serde-1.0.200.crate"}, + }}, + // react has more hits than lodash. Every cached file of lodash is + // listed, but only recent downloads count. + {Ecosystem: "npm", Name: "react", PURL: "pkg:npm/react", Hits: 10, Files: []UsedFile{ + {Version: "18.3.1", Filename: "react-18.3.1.tgz", UpstreamURL: "https://example.test/react-18.3.1.tgz"}, + }}, + {Ecosystem: "npm", Name: "lodash", PURL: "pkg:npm/lodash", Hits: 5, Files: []UsedFile{ + {Version: "4.17.20", Filename: "lodash-4.17.20.tgz", UpstreamURL: "https://example.test/lodash-4.17.20.tgz"}, + {Version: "4.17.21", Filename: "lodash-4.17.21.tgz", UpstreamURL: "https://example.test/lodash-4.17.21.tgz"}, + }}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("MostUsedPackages =\n%+v\nwant\n%+v", got, want) + } + + top1, err := db.MostUsedPackages(now.Add(-14*24*time.Hour), 1) + if err != nil { + t.Fatal(err) + } + if len(top1) != 2 || top1[0].Name != "serde" || top1[1].Name != "react" { + t.Errorf("top 1 per ecosystem = %+v, want serde and react", top1) + } + }) +} + +func TestMostUsedPackagesSkipsLocalArtifacts(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seedUsedArtifact(t, db, "npm", "internal-lib", "1.0.0", "internal-lib-1.0.0.tgz", 50, now) + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET source = ? WHERE filename = ?"), SourceLocal, "internal-lib-1.0.0.tgz"); err != nil { + t.Fatal(err) + } + got, err := db.MostUsedPackages(now.Add(-time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Errorf("MostUsedPackages = %+v, want none: local packages have no upstream to refresh from", got) + } + }) +} diff --git a/internal/handler/release_allowed.go b/internal/handler/release_allowed.go new file mode 100644 index 00000000..4c70214c --- /dev/null +++ b/internal/handler/release_allowed.go @@ -0,0 +1,19 @@ +package handler + +import ( + "time" +) + +// ReleaseAllowed reports whether clients would be served a version published +// at publishedAt: the denylist doesn't name it, and cooldown, where the +// ecosystem applies it, doesn't withhold it. Without a publish time a version +// counts as withheld wherever cooldown applies. +func (p *Proxy) ReleaseAllowed(ecosystem, name, version string, publishedAt time.Time) bool { + if p.versionDenied(ecosystem, name, version) { + return false + } + if !CooldownSupported(ecosystem) || !p.cooldownApplies(ecosystem, name) { + return true + } + return !publishedAt.IsZero() && p.cooldown().IsAllowed(ecosystem, canonicalPackagePURL(ecosystem, name), publishedAt) +} diff --git a/internal/handler/release_allowed_test.go b/internal/handler/release_allowed_test.go new file mode 100644 index 00000000..0d8dd88e --- /dev/null +++ b/internal/handler/release_allowed_test.go @@ -0,0 +1,44 @@ +package handler + +import ( + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/denylist" +) + +func TestReleaseAllowed(t *testing.T) { + deny, err := denylist.New([]string{"pkg:npm/lodash@4.17.22"}) + if err != nil { + t.Fatal(err) + } + p := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}, Denylist: deny} + old := time.Now().Add(-10 * 24 * time.Hour) + fresh := time.Now().Add(-time.Hour) + + tests := []struct { + name string + ecosystem, pkg, version string + published time.Time + want bool + }{ + {"old enough", "npm", "lodash", "4.17.21", old, true}, + {"within cooldown", "npm", "lodash", "4.17.23", fresh, false}, + {"denied", "npm", "lodash", "4.17.22", old, false}, + {"no publish time under cooldown", "npm", "lodash", "4.17.24", time.Time{}, false}, + // deb has no publish times, so cooldown never applies to it. + {"ecosystem without cooldown", "deb", "curl", "8.5.0-2", time.Time{}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := p.ReleaseAllowed(tt.ecosystem, tt.pkg, tt.version, tt.published); got != tt.want { + t.Errorf("ReleaseAllowed = %v, want %v", got, tt.want) + } + }) + } + + if !(&Proxy{}).ReleaseAllowed("npm", "lodash", "4.17.23", fresh) { + t.Error("a proxy without cooldown or denylist refused a release") + } +} diff --git a/internal/mirror/files.go b/internal/mirror/files.go index 8cf61fe7..87ce8956 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -2,6 +2,8 @@ package mirror import ( "fmt" + "path" + "regexp" "slices" "strings" @@ -22,6 +24,12 @@ type fileEcosystem struct { // check refuses a package the ecosystem can't mirror, before its files // are listed. check func(pv PackageVersion) error + // variant names the kind of a file, such as a wheel's tags or a conda + // package's platform, so a refresh mirrors the same kinds of files of a + // new version as were cached of an old one. cachedVariant names it for + // the filename cacheKey returns. + variant func(file registries.Artifact) string + cachedVariant func(cacheFilename string) string } var fileEcosystems = map[string]fileEcosystem{ @@ -37,6 +45,8 @@ var fileEcosystems = map[string]fileEcosystem{ } return name, version, file.Filename, true }, + variant: func(file registries.Artifact) string { return pypiVariant(file.Filename) }, + cachedVariant: pypiVariant, }, "conda": { qualifiers: []string{"subdir", "build", "type"}, @@ -60,9 +70,47 @@ var fileEcosystems = map[string]fileEcosystem{ } return nil }, + variant: func(file registries.Artifact) string { + subdir, _ := file.Metadata["subdir"].(string) + return condaVariant(subdir, file.Filename) + }, + cachedVariant: func(cacheFilename string) string { + subdir, filename := path.Split(cacheFilename) + return condaVariant(strings.TrimSuffix(subdir, "/"), filename) + }, }, } +// pypiVariant names a wheel by its tags, python-abi-platform; every sdist +// is one variant. +func pypiVariant(filename string) string { + stem, wheel := strings.CutSuffix(filename, ".whl") + if !wheel { + return "sdist" + } + parts := strings.Split(stem, "-") + if len(parts) < 5 { //nolint:mnd // name, version and three tags + return stem + } + return strings.Join(parts[len(parts)-3:], "-") +} + +// condaBuildHash matches the hash and build number a conda build string +// ends in, such as h8753938_0 in py312h8753938_0. +var condaBuildHash = regexp.MustCompile(`h[0-9a-f]{7}_\d+$`) + +// condaVariant names a conda package file by its subdir, the part of its +// build string before the hash, such as py312, and its type. +func condaVariant(subdir, filename string) string { + fileType := condaFileType(filename) + stem := strings.TrimSuffix(strings.TrimSuffix(filename, ".conda"), ".tar.bz2") + build := stem[strings.LastIndex(stem, "-")+1:] + if loc := condaBuildHash.FindStringIndex(build); loc != nil { + build = build[:loc[0]] + } + return subdir + "/" + build + "/" + fileType +} + // condaFileType returns the PURL type qualifier of a conda package file. func condaFileType(filename string) string { switch { @@ -78,6 +126,9 @@ func condaFileType(filename string) string { // selection returns the qualifiers of pv that select files, as // "key=value" pairs, for error messages. func (e fileEcosystem) selection(pv PackageVersion) string { + if pv.variants != nil { + return "the kinds of files cached of earlier versions" + } var pairs []string for _, key := range e.qualifiers { if value := pv.Qualifiers[key]; value != "" { @@ -89,6 +140,9 @@ func (e fileEcosystem) selection(pv PackageVersion) string { // selects reports whether file matches every selecting qualifier of pv. func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool { + if pv.variants != nil && !pv.variants[e.variant(file)] { + return false + } attrs := e.attrs(file) for _, key := range e.qualifiers { if want := pv.Qualifiers[key]; want != "" && attrs[key] != want { diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go new file mode 100644 index 00000000..bdc4c29f --- /dev/null +++ b/internal/mirror/refresh.go @@ -0,0 +1,138 @@ +package mirror + +import ( + "context" + "fmt" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries" +) + +// Refresh mirrors new releases of the packages clients download most, so +// they are cached before the first client asks for them. +type Refresh struct { + // Top is how many packages per ecosystem are refreshed. + Top int + // Since is how recently a package must have been downloaded. + Since time.Duration + // MaxVersions stops a run after mirroring this many versions. 0 means + // no cap. + MaxVersions int + // MaxBytes stops a run once it has downloaded this much. It is checked + // before each version, so the last one may go over it. 0 means no cap. + MaxBytes int64 +} + +func (r Refresh) spent(tracker *progressTracker) bool { + return (r.MaxVersions > 0 && tracker.completed.Load() >= int64(r.MaxVersions)) || + (r.MaxBytes > 0 && tracker.bytes.Load() >= r.MaxBytes) +} + +// ReleaseLister lists every release of a package, with its publish time and +// status, for refreshes. +type ReleaseLister interface { + ListReleases(ctx context.Context, ecosystem, name string) ([]registries.Version, error) +} + +// ListReleases implements ReleaseLister. +func (r RegistryVersions) ListReleases(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { + return r.fetchVersions(ctx, ecosystem, name) +} + +// releaseLister lists releases where Versions lists versions, so both come +// from the configured upstream. +func (m *Mirror) releaseLister() ReleaseLister { //nolint:ireturn // any lister will do + if lister, ok := m.Versions.(ReleaseLister); ok { + return lister + } + return RegistryVersions{} +} + +// Refresh mirrors, for each of the most used packages, the newest release +// of each major line it has a version cached on, as clients would be served +// it: releases within cooldown or on the denylist are left out. Packages it +// can't list are counted as failed. +func (m *Mirror) Refresh(ctx context.Context, r Refresh) (*Progress, error) { + packages, err := m.db.MostUsedPackages(time.Now().Add(-r.Since), r.Top) + if err != nil { + return nil, fmt.Errorf("listing the most used packages: %w", err) + } + tracker := newProgressTracker() + tracker.phase.Store("downloading") +refresh: + for _, pkg := range packages { + if ctx.Err() != nil { + break + } + items, err := m.refreshItems(ctx, pkg) + if err != nil { + m.recordFailure(tracker, PackageVersion{Ecosystem: pkg.Ecosystem, Name: pkg.Name}, err) + continue + } + for _, item := range items { + if r.spent(tracker) { + m.logger.Info("mirror refresh reached its cap for one run", + "mirrored", tracker.completed.Load(), "bytes", tracker.bytes.Load()) + break refresh + } + tracker.total.Add(1) + m.mirrorOne(ctx, item, tracker) + } + } + tracker.phase.Store("complete") + p := tracker.snapshot() + return &p, nil +} + +// refreshItems returns the versions to mirror for a used package. +func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([]PackageVersion, error) { + p, err := purl.Parse(pkg.PURL) + if err != nil { + return nil, err + } + ecosystem, name, _ := mirrorTarget(p) + if !refreshable(ecosystem) { + return nil, nil + } + releases, err := m.releaseLister().ListReleases(ctx, ecosystem, name) + if err != nil { + return nil, fmt.Errorf("listing releases: %w", err) + } + + var cached []string + seen := map[string]bool{} + for _, f := range pkg.Files { + if !seen[f.Version] { + seen[f.Version] = true + cached = append(cached, f.Version) + } + } + allowed := func(v registries.Version) bool { + return m.proxy == nil || m.proxy.ReleaseAllowed(pkg.Ecosystem, pkg.Name, v.Number, v.PublishedAt) + } + + var items []PackageVersion + for _, version := range newestPerCachedLine(ecosystem, cached, releases, allowed) { + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: version} + if eco, ok := fileEcosystems[ecosystem]; ok { + pv.variants = map[string]bool{} + for _, f := range pkg.Files { + pv.variants[eco.cachedVariant(f.Filename)] = true + } + } + items = append(items, pv) + } + return items, nil +} + +// refreshable reports whether a refresh can list the releases of an +// ecosystem's packages. +func refreshable(ecosystem string) bool { + if isImageEcosystem(ecosystem) { + return false + } + _, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", nil) + return err == nil +} diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go new file mode 100644 index 00000000..684b44c8 --- /dev/null +++ b/internal/mirror/refresh_test.go @@ -0,0 +1,306 @@ +package mirror + +import ( + "context" + "fmt" + "io" + "net/http" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries" + "github.com/git-pkgs/registries/fetch" +) + +// servingFetcher answers every fetch with body and records the URLs. +type servingFetcher struct { + body string + mu sync.Mutex + urls []string +} + +func (f *servingFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { + return f.FetchWithHeaders(ctx, url, nil) +} + +func (f *servingFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) { + f.mu.Lock() + f.urls = append(f.urls, url) + f.mu.Unlock() + return &fetch.Artifact{Body: io.NopCloser(strings.NewReader(f.body)), Size: int64(len(f.body)), ContentType: "application/octet-stream"}, nil +} + +func (f *servingFetcher) Head(context.Context, string) (int64, string, error) { + return int64(len(f.body)), "application/octet-stream", nil +} + +func (f *servingFetcher) fetched() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.urls...) +} + +func (f *servingFetcher) reset() { + f.mu.Lock() + defer f.mu.Unlock() + f.urls = nil +} + +// stubReleases lists releases by "ecosystem/name". +type stubReleases map[string][]registries.Version + +func (s stubReleases) ListReleases(_ context.Context, ecosystem, name string) ([]registries.Version, error) { + r, ok := s[ecosystem+"/"+name] + if !ok { + return nil, fmt.Errorf("%w: %s/%s", registries.ErrNotFound, ecosystem, name) + } + return r, nil +} + +func (s stubReleases) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) { + r, err := s.ListReleases(ctx, ecosystem, name) + numbers := make([]string, len(r)) + for i, v := range r { + numbers[i] = v.Number + } + return numbers, err +} + +func (s stubReleases) FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) { + r, err := s.ListReleases(ctx, ecosystem, name) + if err != nil { + return registries.Version{}, err + } + for _, v := range r { + if v.Number == version { + return v, nil + } + } + return registries.Version{}, registries.ErrNotFound +} + +func oldReleases(numbers ...string) []registries.Version { + r := releases(numbers...) + for i := range r { + r[i].PublishedAt = time.Now().Add(-time.Duration(len(r)-i) * 30 * 24 * time.Hour) + } + return r +} + +func setupRefreshMirror(t *testing.T, body string) (*Mirror, *servingFetcher) { + t.Helper() + m := setupTestMirror(t, 1) + fetcher := &servingFetcher{body: body} + m.proxy.Fetcher = fetcher + return m, fetcher +} + +// cacheNPM downloads a version the way a client would, so it counts as used. +func cacheNPM(t *testing.T, m *Mirror, name, version string) { + t.Helper() + result, err := m.proxy.GetOrFetchArtifact(context.Background(), "npm", name, version, "") + if err != nil { + t.Fatalf("caching %s@%s: %v", name, version, err) + } + _ = result.Reader.Close() +} + +func fetchedMatching(urls []string, part string) []string { + var out []string + for _, u := range urls { + if strings.Contains(u, part) { + out = append(out, u) + } + } + return out +} + +func TestRefreshMirrorsNewerVersionOnCachedLine(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21", "5.0.0")} + fetcher.reset() + + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Completed != 1 || progress.Failed != 0 { + t.Errorf("progress = %+v, want one mirrored", progress) + } + urls := fetcher.fetched() + if len(fetchedMatching(urls, "lodash-4.17.21.tgz")) != 1 || len(fetchedMatching(urls, "5.0.0")) != 0 { + t.Errorf("fetched %v, want lodash 4.17.21 only", urls) + } + + // The new version is cached now, so the next run has nothing to do. + progress, err = m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("second Refresh: %v", err) + } + if progress.Completed != 0 || progress.Failed != 0 { + t.Errorf("second run progress = %+v, want nothing to do", progress) + } +} + +func TestRefreshKeepsCooldown(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + r := oldReleases("4.17.20", "4.17.21") + r[1].PublishedAt = time.Now().Add(-time.Hour) + m.Versions = stubReleases{"npm/lodash": r} + m.proxy.Cooldown = &cooldown.Config{Default: "3d"} + fetcher.reset() + + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want a version within cooldown left alone", progress, fetcher.fetched()) + } +} + +func TestRefreshOnlyTopPackagesRecentlyUsed(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + cacheNPM(t, m, "react", "18.3.0") + cacheNPM(t, m, "react", "18.3.0") // a second download: react is used most + m.Versions = stubReleases{ + "npm/lodash": oldReleases("4.17.20", "4.17.21"), + "npm/react": oldReleases("18.3.0", "18.3.1"), + } + if err := m.db.RecordArtifactHit("pkg:npm/react@18.3.0", "react-18.3.0.tgz"); err != nil { + t.Fatal(err) + } + fetcher.reset() + + progress, err := m.Refresh(context.Background(), Refresh{Top: 1, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + urls := fetcher.fetched() + if progress.Completed != 1 || len(fetchedMatching(urls, "react-18.3.1.tgz")) != 1 || len(fetchedMatching(urls, "lodash")) != 0 { + t.Errorf("progress = %+v, fetched %v; want only react refreshed", progress, urls) + } +} + +func TestRefreshStopsAtMaxVersions(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + cacheNPM(t, m, "react", "18.3.0") + m.Versions = stubReleases{ + "npm/lodash": oldReleases("4.17.20", "4.17.21"), + "npm/react": oldReleases("18.3.0", "18.3.1"), + } + fetcher.reset() + + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour, MaxVersions: 1}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Completed != 1 || len(fetcher.fetched()) != 1 { + t.Errorf("progress = %+v, fetched %v; want one version", progress, fetcher.fetched()) + } +} + +func TestRefreshStopsAtMaxBytes(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "0123456789") + cacheNPM(t, m, "lodash", "4.17.20") + cacheNPM(t, m, "react", "18.3.0") + m.Versions = stubReleases{ + "npm/lodash": oldReleases("4.17.20", "4.17.21"), + "npm/react": oldReleases("18.3.0", "18.3.1"), + } + fetcher.reset() + + // The cap is checked before each download, so the first one goes over it. + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour, MaxBytes: 5}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Completed != 1 || progress.Bytes != 10 { + t.Errorf("progress = %+v, want one version of 10 bytes", progress) + } +} + +func TestRefreshCountsListingFailure(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{} + fetcher.reset() + + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Failed != 1 || len(progress.Errors) != 1 || progress.Errors[0].Name != "lodash" { + t.Errorf("progress = %+v, want lodash failed", progress) + } +} + +func TestRefreshMirrorsTheFileVariantsClientsFetched(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "wheel") + ctx := context.Background() + for _, f := range []string{"demo-1.0.0-py3-none-any.whl", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.whl"} { + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "pypi", "demo", "1.0.0", f, "https://files.example/"+f, "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + } + files := func(version string, names ...string) registries.Version { + v := registries.Version{Number: version, PublishedAt: time.Now().Add(-30 * 24 * time.Hour)} + for _, n := range names { + v.Artifacts = append(v.Artifacts, registries.Artifact{Filename: n, URL: "https://files.example/" + n}) + } + return v + } + m.Versions = stubReleases{"pypi/demo": { + files("1.0.0", "demo-1.0.0-py3-none-any.whl", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.whl"), + files("1.1.0", + "demo-1.1.0.tar.gz", + "demo-1.1.0-py3-none-any.whl", + "demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.whl", + "demo-1.1.0-cp313-cp313-manylinux_2_17_x86_64.whl", + "demo-1.1.0-cp312-cp312-win_amd64.whl"), + }} + fetcher.reset() + + progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + got := fetcher.fetched() + slices.Sort(got) + want := []string{ + "https://files.example/demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.whl", + "https://files.example/demo-1.1.0-py3-none-any.whl", + } + if progress.Completed != 1 || !slices.Equal(got, want) { + t.Errorf("progress = %+v, fetched %v; want %v", progress, got, want) + } +} + +func TestFileVariants(t *testing.T) { + tests := []struct { + ecosystem, cacheFilename, want string + }{ + {"pypi", "demo-1.0.0-py3-none-any.whl", "py3-none-any"}, + {"pypi", "demo-1.0.0-1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64"}, + {"pypi", "demo-1.0.0.tar.gz", "sdist"}, + {"pypi", "demo-1.0.0.zip", "sdist"}, + {"conda", "linux-64/numpy-1.26.4-py312h8753938_0.conda", "linux-64/py312/conda"}, + {"conda", "noarch/requests-2.32.3-pyhd8ed1ab_0.conda", "noarch/py/conda"}, + {"conda", "osx-arm64/zlib-1.3.1-h8359307_2.tar.bz2", "osx-arm64//tar.bz2"}, + } + for _, tt := range tests { + if got := fileEcosystems[tt.ecosystem].cachedVariant(tt.cacheFilename); got != tt.want { + t.Errorf("%s variant of %q = %q, want %q", tt.ecosystem, tt.cacheFilename, got, tt.want) + } + } +} diff --git a/internal/mirror/refresh_versions.go b/internal/mirror/refresh_versions.go new file mode 100644 index 00000000..b489170d --- /dev/null +++ b/internal/mirror/refresh_versions.go @@ -0,0 +1,67 @@ +package mirror + +import ( + "strconv" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries" + "github.com/git-pkgs/vers" +) + +// newestPerCachedLine returns, for each major line a cached version is on, +// the newest release of that line that allowed accepts, when it is newer +// than every cached version of the line. A line is the major version, or +// 0.minor below 1.0, as caret ranges read them. Prereleases and yanked, +// retracted or deprecated releases are skipped. +func newestPerCachedLine(ecosystem string, cached []string, releases []registries.Version, allowed func(registries.Version) bool) []string { + scheme := purl.EcosystemToPURLType(ecosystem) + newestCached := map[string]string{} + var lines []string + for _, v := range cached { + line, ok := versionLine(v) + if !ok { + continue + } + current, seen := newestCached[line] + if !seen { + lines = append(lines, line) + } + if !seen || vers.CompareWithScheme(v, current, scheme) > 0 { + newestCached[line] = v + } + } + + newest := map[string]string{} + for _, r := range releases { + line, ok := versionLine(r.Number) + if !ok || r.Status != registries.StatusNone || vers.IsPrereleaseWithScheme(r.Number, scheme) { + continue + } + if _, cachedLine := newestCached[line]; !cachedLine || !allowed(r) { + continue + } + if current, seen := newest[line]; !seen || vers.CompareWithScheme(r.Number, current, scheme) > 0 { + newest[line] = r.Number + } + } + + var out []string + for _, line := range lines { + if v, ok := newest[line]; ok && vers.CompareWithScheme(v, newestCached[line], scheme) > 0 { + out = append(out, v) + } + } + return out +} + +// versionLine returns the major line of a version. +func versionLine(v string) (string, bool) { + info, err := vers.ParseVersion(v) + if err != nil { + return "", false + } + if info.Major == 0 { + return "0." + strconv.Itoa(info.Minor), true + } + return strconv.Itoa(info.Major), true +} diff --git a/internal/mirror/refresh_versions_test.go b/internal/mirror/refresh_versions_test.go new file mode 100644 index 00000000..6e1e2628 --- /dev/null +++ b/internal/mirror/refresh_versions_test.go @@ -0,0 +1,79 @@ +package mirror + +import ( + "reflect" + "testing" + "time" + + "github.com/git-pkgs/registries" +) + +func releases(numbers ...string) []registries.Version { + out := make([]registries.Version, len(numbers)) + for i, n := range numbers { + out[i] = registries.Version{Number: n, PublishedAt: time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC)} + } + return out +} + +func TestNewestPerCachedLine(t *testing.T) { + allowAll := func(registries.Version) bool { return true } + yanked := releases("4.17.20", "4.17.21", "4.17.22") + yanked[2].Status = registries.StatusYanked + + tests := []struct { + name string + cached []string + releases []registries.Version + allowed func(registries.Version) bool + want []string + }{ + {name: "newer patch on the cached line", cached: []string{"4.17.20"}, + releases: releases("4.17.20", "4.17.21", "5.0.0"), want: []string{"4.17.21"}}, + {name: "newest already cached", cached: []string{"4.17.21"}, + releases: releases("4.17.20", "4.17.21", "5.0.0")}, + {name: "one per cached major line", cached: []string{"1.2.0", "2.0.0"}, + releases: releases("1.2.0", "1.3.0", "2.0.0", "2.1.0", "3.0.0"), want: []string{"1.3.0", "2.1.0"}}, + {name: "zero major lines are minor lines", cached: []string{"0.3.1"}, + releases: releases("0.3.1", "0.3.2", "0.4.0"), want: []string{"0.3.2"}}, + {name: "prereleases skipped", cached: []string{"2.0.0"}, + releases: releases("2.0.0", "2.1.0-beta.1")}, + {name: "yanked skipped", cached: []string{"4.17.20"}, + releases: yanked, want: []string{"4.17.21"}}, + {name: "newest withheld, older newer one taken", cached: []string{"4.17.20"}, + releases: releases("4.17.20", "4.17.21", "4.17.22"), + allowed: func(v registries.Version) bool { return v.Number != "4.17.22" }, + want: []string{"4.17.21"}}, + {name: "nothing allowed newer than cached", cached: []string{"4.17.20"}, + releases: releases("4.17.20", "4.17.21"), + allowed: func(v registries.Version) bool { return v.Number == "4.17.20" }}, + {name: "cached newer than every listed one", cached: []string{"4.17.25"}, + releases: releases("4.17.20", "4.17.21")}, + {name: "only the newest cached of a line counts", cached: []string{"4.17.19", "4.17.21"}, + releases: releases("4.17.19", "4.17.20", "4.17.21")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + allowed := tt.allowed + if allowed == nil { + allowed = allowAll + } + got := newestPerCachedLine("npm", tt.cached, tt.releases, allowed) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("newestPerCachedLine = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewestPerCachedLineUsesEcosystemOrdering(t *testing.T) { + // Maven orders 1.0.10 after 1.0.9, and a gem's 1.0.0.rc1 is a prerelease. + got := newestPerCachedLine("maven", []string{"1.0.9"}, releases("1.0.9", "1.0.10"), func(registries.Version) bool { return true }) + if !reflect.DeepEqual(got, []string{"1.0.10"}) { + t.Errorf("maven: got %v, want [1.0.10]", got) + } + got = newestPerCachedLine("gem", []string{"1.0.0"}, releases("1.0.0", "1.0.1.rc1"), func(registries.Version) bool { return true }) + if got != nil { + t.Errorf("gem: got %v, want nothing", got) + } +} diff --git a/internal/mirror/schedule.go b/internal/mirror/schedule.go index 1df9a750..244fd89b 100644 --- a/internal/mirror/schedule.go +++ b/internal/mirror/schedule.go @@ -17,10 +17,12 @@ const ( scheduleJitter = 10 ) -// Schedule mirrors a set of image PURLs every Interval. +// Schedule mirrors a set of image PURLs every Interval, or refreshes the +// most used packages when Refresh is set. type Schedule struct { PURLs []string Platforms []string + Refresh *Refresh Interval time.Duration } @@ -102,7 +104,13 @@ func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) { } } started := time.Now() - progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms}) + var progress *Progress + var err error + if schedule.Refresh != nil { + progress, err = s.mirror.Refresh(ctx, *schedule.Refresh) + } else { + progress, err = s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms}) + } if ctx.Err() != nil { if s.Claim != nil && s.Release != nil { if err := s.Release(key); err != nil { @@ -140,6 +148,9 @@ func scheduleDelay(interval time.Duration, first bool, randN func(int64) int64) // does not inherit the old one's last run. func scheduleKey(schedule Schedule) string { h := sha256.New() + if r := schedule.Refresh; r != nil { + _, _ = fmt.Fprintf(h, "refresh top %d since %s max_versions %d max_bytes %d\n", r.Top, r.Since, r.MaxVersions, r.MaxBytes) + } for _, purl := range schedule.PURLs { _, _ = fmt.Fprintf(h, "purl %s\n", purl) } diff --git a/internal/mirror/schedule_test.go b/internal/mirror/schedule_test.go index b5b0143a..0a6edc31 100644 --- a/internal/mirror/schedule_test.go +++ b/internal/mirror/schedule_test.go @@ -279,3 +279,36 @@ func TestScheduleKey(t *testing.T) { } } } + +func TestSchedulerRunsRefresh(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21")} + fetcher.reset() + + schedule := Schedule{Refresh: &Refresh{Top: 10, Since: time.Hour}, Interval: time.Hour} + s := NewScheduler(m, []Schedule{schedule}, m.logger) + s.run(context.Background(), 0, schedule) + + if urls := fetcher.fetched(); len(urls) != 1 || len(fetchedMatching(urls, "lodash-4.17.21.tgz")) != 1 { + t.Errorf("fetched %v, want lodash 4.17.21", urls) + } +} + +func TestScheduleKeyRefresh(t *testing.T) { + base := Schedule{Refresh: &Refresh{Top: 50, Since: time.Hour}, Interval: time.Hour} + same := Schedule{Refresh: &Refresh{Top: 50, Since: time.Hour}, Interval: time.Hour} + if scheduleKey(base) != scheduleKey(same) { + t.Error("same refresh gave different keys") + } + for name, other := range map[string]Schedule{ + "no refresh": {Interval: time.Hour}, + "top": {Refresh: &Refresh{Top: 10, Since: time.Hour}, Interval: time.Hour}, + "since": {Refresh: &Refresh{Top: 50, Since: 2 * time.Hour}, Interval: time.Hour}, + "interval": {Refresh: &Refresh{Top: 50, Since: time.Hour}, Interval: 2 * time.Hour}, + } { + if scheduleKey(other) == scheduleKey(base) { + t.Errorf("changing %s kept the key", name) + } + } +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index ecc3a0c5..30c4e0b7 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -22,6 +22,9 @@ type PackageVersion struct { Qualifiers map[string]string // Platforms limit which platforms of an image index are mirrored. Platforms []string + // variants, when set, limit the files of a version that are mirrored to + // those of these variants (see fileEcosystem.variant). + variants map[string]bool } func (pv PackageVersion) String() string { diff --git a/internal/server/mirror_schedule.go b/internal/server/mirror_schedule.go index 172c2de2..78315d44 100644 --- a/internal/server/mirror_schedule.go +++ b/internal/server/mirror_schedule.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/mirror" ) @@ -13,20 +14,38 @@ import ( const mirrorScheduleShutdownWait = 30 * time.Second func (s *Server) startMirrorSchedules(ctx context.Context, proxy *handler.Proxy) { - if len(s.cfg.Mirror.Schedules) == 0 { + schedules := mirrorSchedules(s.cfg.Mirror) + if len(schedules) == 0 { return } // One worker, so a schedule never fans out against the registry. m := s.newMirror(proxy, 1) - schedules := make([]mirror.Schedule, len(s.cfg.Mirror.Schedules)) - for i, schedule := range s.cfg.Mirror.Schedules { - schedules[i] = mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()} - } s.scheduler = mirror.NewScheduler(m, schedules, s.logger) s.scheduler.Claim = s.db.ClaimJobRun s.scheduler.Release = s.db.ReleaseJobRun s.scheduler.Start(ctx) - s.logger.Info("mirror schedules enabled", "schedules", len(schedules)) + s.logger.Info("mirror schedules enabled", "schedules", len(schedules), "refresh", s.cfg.Mirror.Refresh != nil) +} + +// mirrorSchedules returns the schedules of a validated mirror configuration, +// the refresh last. +func mirrorSchedules(cfg config.MirrorConfig) []mirror.Schedule { + var schedules []mirror.Schedule + for _, schedule := range cfg.Schedules { + schedules = append(schedules, mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()}) + } + if r := cfg.Refresh; r != nil { + schedules = append(schedules, mirror.Schedule{ + Refresh: &mirror.Refresh{ + Top: r.TopPackages(), + Since: r.SinceDuration(), + MaxVersions: r.MaxVersionsPerRun(), + MaxBytes: r.MaxBytesPerRun(), + }, + Interval: r.IntervalDuration(), + }) + } + return schedules } func (s *Server) waitForMirrorSchedules(ctx context.Context) { diff --git a/internal/server/mirror_schedule_test.go b/internal/server/mirror_schedule_test.go index d2440e65..1c19559f 100644 --- a/internal/server/mirror_schedule_test.go +++ b/internal/server/mirror_schedule_test.go @@ -4,10 +4,12 @@ import ( "context" "io" "log/slog" + "reflect" "sync/atomic" "testing" "time" + "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/mirror" ) @@ -96,3 +98,23 @@ func TestWaitForMirrorSchedulesWithoutSchedules(t *testing.T) { s := &Server{} s.waitForMirrorSchedules(context.Background()) } + +func TestMirrorSchedulesFromConfig(t *testing.T) { + cfg := config.MirrorConfig{ + Schedules: []config.MirrorSchedule{{Images: []string{"pkg:docker/alpine@3"}, Interval: "6h", Platforms: []string{"linux/amd64"}}}, + Refresh: &config.MirrorRefresh{Interval: "24h", Top: 20, Since: "7d", MaxVersions: 100, MaxBytes: "1GB"}, + } + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + want := []mirror.Schedule{ + {PURLs: []string{"pkg:docker/alpine@3"}, Platforms: []string{"linux/amd64"}, Interval: 6 * time.Hour}, + {Refresh: &mirror.Refresh{Top: 20, Since: 7 * 24 * time.Hour, MaxVersions: 100, MaxBytes: 1 << 30}, Interval: 24 * time.Hour}, + } + if got := mirrorSchedules(cfg); !reflect.DeepEqual(got, want) { + t.Errorf("mirrorSchedules =\n%+v\nwant\n%+v", got, want) + } + if got := mirrorSchedules(config.MirrorConfig{}); len(got) != 0 { + t.Errorf("mirrorSchedules of an empty config = %+v, want none", got) + } +} From 3e4491287d3634eb2f6e1e6fa6f6d236246ec09a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 13:57:49 +0100 Subject: [PATCH 02/15] Record the upstream URL of cached metadata The refresh fetches Debian, Alpine and RPM indexes again from the URLs clients fetched them from, which the hashed cache keys don't keep. Rows cached before get their URL on the next revalidation, even on a 304. --- .../database/metadata_upstream_url_test.go | 90 +++++++++++++++++++ internal/database/queries.go | 35 ++++++-- internal/database/schema.go | 19 ++++ internal/database/types.go | 15 ++-- internal/handler/handler.go | 11 ++- internal/handler/handler_test.go | 52 +++++++++++ 6 files changed, 209 insertions(+), 13 deletions(-) create mode 100644 internal/database/metadata_upstream_url_test.go diff --git a/internal/database/metadata_upstream_url_test.go b/internal/database/metadata_upstream_url_test.go new file mode 100644 index 00000000..ef588639 --- /dev/null +++ b/internal/database/metadata_upstream_url_test.go @@ -0,0 +1,90 @@ +package database + +import ( + "database/sql" + "path/filepath" + "reflect" + "testing" + "time" +) + +func TestMigrationAddsMetadataUpstreamURL(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "existing.db") + sqlDB, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + if _, err := sqlDB.Exec(schemaSQLite); err != nil { + t.Fatalf("create schema: %v", err) + } + if _, err := sqlDB.Exec(`ALTER TABLE metadata_cache DROP COLUMN upstream_url`); err != nil { + t.Fatalf("drop upstream_url column: %v", err) + } + if _, err := sqlDB.Exec(`DELETE FROM migrations WHERE name = '020_add_metadata_upstream_url'`); err != nil { + t.Fatalf("forget migration: %v", err) + } + if err := sqlDB.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + db, err := Open(dbPath) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer func() { _ = db.Close() }() + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + has, err := db.HasColumn("metadata_cache", "upstream_url") + if err != nil || !has { + t.Errorf("metadata_cache.upstream_url after migration = %v, %v", has, err) + } +} + +func upsertMetadataURL(t *testing.T, db *DB, ecosystem, name, url string) { + t.Helper() + entry := &MetadataCacheEntry{Ecosystem: ecosystem, Name: name, StoragePath: "_metadata/" + name} + if url != "" { + entry.UpstreamURL = sql.NullString{String: url, Valid: true} + } + if err := db.UpsertMetadataCache(entry); err != nil { + t.Fatal(err) + } +} + +func TestUpsertMetadataCacheKeepsUpstreamURL(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertMetadataURL(t, db, "apk", "k1", "https://dl-cdn.alpinelinux.org/alpine/v3.20/main/aarch64/APKINDEX.tar.gz") + // A writer that doesn't know the URL leaves it as it was. + upsertMetadataURL(t, db, "apk", "k1", "") + got, err := db.GetMetadataCache("apk", "k1") + if err != nil || got == nil { + t.Fatalf("GetMetadataCache: %+v, %v", got, err) + } + if got.UpstreamURL.String != "https://dl-cdn.alpinelinux.org/alpine/v3.20/main/aarch64/APKINDEX.tar.gz" { + t.Errorf("upstream_url = %q", got.UpstreamURL.String) + } + }) +} + +func TestRecentMetadataURLs(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertMetadataURL(t, db, "apk", "k1", "https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz") + upsertMetadataURL(t, db, "apk", "k2", "https://a.test/v3.19/main/aarch64/APKINDEX.tar.gz") + upsertMetadataURL(t, db, "apk", "k3", "") + upsertMetadataURL(t, db, "debian", "k4", "https://d.test/dists/bookworm/InRelease") + old := time.Now().Add(-30 * 24 * time.Hour).UTC() + if _, err := db.Exec(db.Rebind("UPDATE metadata_cache SET last_accessed_at = ?, updated_at = ? WHERE name = ?"), old, old, "k2"); err != nil { + t.Fatal(err) + } + + got, err := db.RecentMetadataURLs("apk", time.Now().Add(-14*24*time.Hour)) + if err != nil { + t.Fatal(err) + } + want := []string{"https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz"} + if !reflect.DeepEqual(got, want) { + t.Errorf("RecentMetadataURLs = %v, want %v", got, want) + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e80..3f74a6d8 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -1299,7 +1299,7 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err var entry MetadataCacheEntry query := db.Rebind(` SELECT id, ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at + content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at FROM metadata_cache WHERE ecosystem = ? AND name = ? `) err := db.Get(&entry, query, ecosystem, name) @@ -1312,6 +1312,19 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err return &entry, nil } +// RecentMetadataURLs returns the upstream URLs of the documents of an +// ecosystem cached or served since cutoff, as far as they were recorded. +func (db *DB) RecentMetadataURLs(ecosystem string, cutoff time.Time) ([]string, error) { + var urls []string + err := db.Select(&urls, db.Rebind(` + SELECT DISTINCT upstream_url FROM metadata_cache + WHERE ecosystem = ? AND upstream_url IS NOT NULL + AND COALESCE(last_accessed_at, updated_at) >= ? + ORDER BY upstream_url + `), ecosystem, cutoff.UTC()) + return urls, err +} + // RefreshMetadataCache sets fetched_at on the metadata cache row for ecosystem // and name, provided the row still records digest, and reports whether it did. // It is for an upstream 304, which leaves the stored document as it is. @@ -1328,6 +1341,14 @@ func (db *DB) RefreshMetadataCache(ecosystem, name, digest string, fetchedAt tim return n > 0, err } +// SetMetadataUpstreamURL records the upstream URL of the metadata cache row +// for ecosystem and name, for rows cached before URLs were recorded. +func (db *DB) SetMetadataUpstreamURL(ecosystem, name, upstreamURL string) error { + _, err := db.Exec(db.Rebind(`UPDATE metadata_cache SET upstream_url = ? WHERE ecosystem = ? AND name = ?`), + upstreamURL, ecosystem, name) + return err +} + // MoveMetadataCache points the metadata cache row for ecosystem and name at // to, provided it is still at from and records digest, and reports whether // it did. The caller has copied the document from from to to. @@ -1373,8 +1394,8 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { if db.dialect == DialectPostgres { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = EXCLUDED.storage_path, etag = EXCLUDED.etag, @@ -1382,6 +1403,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { content_type = EXCLUDED.content_type, content_encoding = EXCLUDED.content_encoding, content_digest = EXCLUDED.content_digest, + upstream_url = COALESCE(EXCLUDED.upstream_url, metadata_cache.upstream_url), size = EXCLUDED.size, last_modified = EXCLUDED.last_modified, fetched_at = EXCLUDED.fetched_at, @@ -1391,8 +1413,8 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { } else { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = excluded.storage_path, etag = excluded.etag, @@ -1400,6 +1422,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { content_type = excluded.content_type, content_encoding = excluded.content_encoding, content_digest = excluded.content_digest, + upstream_url = COALESCE(excluded.upstream_url, metadata_cache.upstream_url), size = excluded.size, last_modified = excluded.last_modified, fetched_at = excluded.fetched_at, @@ -1410,7 +1433,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { _, err := db.Exec(query, entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag, entry.Link, - entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.Size, entry.LastModified, fetchedAt, now.UTC(), now, now, + entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.UpstreamURL, entry.Size, entry.LastModified, fetchedAt, now.UTC(), now, now, ) if err != nil { return fmt.Errorf("upserting metadata cache: %w", err) diff --git a/internal/database/schema.go b/internal/database/schema.go index f0ec6516..d2ad5f53 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -128,6 +128,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( content_type TEXT, content_encoding TEXT, content_digest TEXT, + upstream_url TEXT, size INTEGER, last_modified DATETIME, fetched_at DATETIME, @@ -271,6 +272,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( content_type TEXT, content_encoding TEXT, content_digest TEXT, + upstream_url TEXT, size BIGINT, last_modified TIMESTAMP, fetched_at TIMESTAMP, @@ -468,6 +470,7 @@ var migrations = []migration{ {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, + {"020_add_metadata_upstream_url", migrateAddMetadataUpstreamURL}, } // isTableNotFound returns true if the error indicates a missing table. @@ -728,6 +731,22 @@ func migrateAddMetadataLink(db *DB) error { return nil } +// migrateAddMetadataUpstreamURL adds the URL a metadata document was +// fetched from, which the mirror refresh fetches it from again. +func migrateAddMetadataUpstreamURL(db *DB) error { + hasColumn, err := db.HasColumn("metadata_cache", "upstream_url") + if err != nil { + return fmt.Errorf("checking metadata_cache upstream_url column: %w", err) + } + if hasColumn { + return nil + } + if _, err := db.Exec("ALTER TABLE metadata_cache ADD COLUMN upstream_url TEXT"); err != nil { + return fmt.Errorf("adding metadata_cache upstream_url column: %w", err) + } + return nil +} + func migrateAddMetadataContentEncoding(db *DB) error { hasColumn, err := db.HasColumn("metadata_cache", "content_encoding") if err != nil { diff --git a/internal/database/types.go b/internal/database/types.go index fb49a1cd..b7593968 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -184,12 +184,15 @@ type MetadataCacheEntry struct { ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"` ContentEncoding sql.NullString `db:"content_encoding" json:"content_encoding,omitempty"` ContentDigest sql.NullString `db:"content_digest" json:"content_digest,omitempty"` - Size sql.NullInt64 `db:"size" json:"size,omitempty"` - LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` - FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` - LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + // UpstreamURL is where the document was fetched from. Rows written + // before it was recorded, and by writers that don't know it, have none. + UpstreamURL sql.NullString `db:"upstream_url" json:"upstream_url,omitempty"` + Size sql.NullInt64 `db:"size" json:"size,omitempty"` + LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` + FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` + LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } // Vulnerability represents a cached vulnerability record. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4b7ebe72..60a7ae44 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1943,6 +1943,8 @@ type upstreamMetadata struct { contentEncoding string etag string lastModified time.Time + // url is the upstream URL the document was fetched from. + url string // notModified is the cache row upstream answered 304 for, whose stored // bytes body holds. It is nil for a document upstream sent. notModified *database.MetadataCacheEntry @@ -1987,7 +1989,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e if readErr != nil { return nil, errStale304 } - meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, notModified: entry} + meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, url: upstreamURL, notModified: entry} if entry.ContentType.Valid { meta.contentType = entry.ContentType.String } @@ -2020,6 +2022,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e contentType: resp.Header.Get(headerContentType), contentEncoding: resp.Header.Get(headerContentEncoding), etag: resp.Header.Get(headerETag), + url: upstreamURL, } if meta.contentType == "" { meta.contentType = contentTypeJSON @@ -2049,6 +2052,11 @@ func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMe if !refreshed { return "" } + if !entry.UpstreamURL.Valid && meta.url != "" { + if err := p.DB.SetMetadataUpstreamURL(ecosystem, cacheKey, meta.url); err != nil { + p.Logger.Warn("failed to record metadata upstream URL", "ecosystem", ecosystem, "key", cacheKey, "error", err) + } + } p.recordMetadataRead(entry) return entry.ContentDigest.String } @@ -2081,6 +2089,7 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor // The digest identifies the stored bytes, so a rewrite cached for them // can be found without reading them back (see storedRewrite). ContentDigest: sql.NullString{String: digest, Valid: digest != ""}, + UpstreamURL: sql.NullString{String: meta.url, Valid: meta.url != ""}, Size: sql.NullInt64{Int64: size, Valid: true}, LastModified: sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 8ce636c9..ed56b631 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -13,6 +13,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -1584,3 +1585,54 @@ func TestCanonicalPackagePURLMatchesConfig(t *testing.T) { }) } } + +func TestProxyCachedRecordsUpstreamURL(t *testing.T) { + proxy, upstream := setupCachedProxy(t, `"abc123"`, "") + req := httptest.NewRequest(http.MethodGet, "/test", nil) + proxy.ProxyCached(httptest.NewRecorder(), req, upstream.URL+"/test", "test-eco", "url-key") + + entry, err := proxy.DB.GetMetadataCache("test-eco", "url-key") + if err != nil || entry == nil { + t.Fatalf("GetMetadataCache: %+v, %v", entry, err) + } + if entry.UpstreamURL.String != upstream.URL+"/test" { + t.Errorf("upstream_url = %q, want %q", entry.UpstreamURL.String, upstream.URL+"/test") + } +} + +// A row cached before upstream URLs were recorded gets its URL on the next +// revalidation, even when upstream answers 304. +func TestProxyCachedRecordsUpstreamURLOnNotModified(t *testing.T) { + var notModified atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"v1"`) + if r.Header.Get("If-None-Match") == `"v1"` { + notModified.Add(1) + w.WriteHeader(http.StatusNotModified) + return + } + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.HTTPClient = upstream.Client() + + proxy.ProxyCached(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/test", nil), upstream.URL+"/test", "test-eco", "url-304-key") + if _, err := proxy.DB.Exec("UPDATE metadata_cache SET upstream_url = NULL"); err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + proxy.ProxyCached(w, httptest.NewRequest(http.MethodGet, "/test", nil), upstream.URL+"/test", "test-eco", "url-304-key") + if w.Code != http.StatusOK || notModified.Load() != 1 { + t.Fatalf("status = %d, upstream 304s = %d; want 200 after one 304", w.Code, notModified.Load()) + } + entry, err := proxy.DB.GetMetadataCache("test-eco", "url-304-key") + if err != nil || entry == nil { + t.Fatalf("GetMetadataCache: %+v, %v", entry, err) + } + if entry.UpstreamURL.String != upstream.URL+"/test" { + t.Errorf("upstream_url after a 304 = %q, want %q", entry.UpstreamURL.String, upstream.URL+"/test") + } +} From 2f2d01a414b3867eab3544ebdd31bc0aee3b790c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 14:05:05 +0100 Subject: [PATCH 03/15] Refresh Debian, Alpine and RPM packages from their indexes These clients find new versions in a repository index. The refresh fetches the InRelease, APKINDEX and repomd.xml files clients fetched again, reads the Packages and primary indexes they point at for the repositories the most used packages came from, and mirrors the newer file per package and architecture, checked against the index's SHA-256 where it has one. --- go.mod | 2 +- .../database/metadata_upstream_url_test.go | 8 +- internal/database/queries.go | 23 +- internal/handler/os_package_identity.go | 17 ++ internal/handler/os_package_identity_test.go | 23 ++ internal/mirror/mirror.go | 9 + internal/mirror/refresh.go | 61 ++++- internal/mirror/refresh_index.go | 254 ++++++++++++++++++ internal/mirror/refresh_index_parse.go | 238 ++++++++++++++++ internal/mirror/refresh_index_parse_test.go | 188 +++++++++++++ internal/mirror/refresh_index_test.go | 231 ++++++++++++++++ internal/mirror/source.go | 4 + 12 files changed, 1032 insertions(+), 26 deletions(-) create mode 100644 internal/handler/os_package_identity.go create mode 100644 internal/handler/os_package_identity_test.go create mode 100644 internal/mirror/refresh_index.go create mode 100644 internal/mirror/refresh_index_parse.go create mode 100644 internal/mirror/refresh_index_parse_test.go create mode 100644 internal/mirror/refresh_index_test.go diff --git a/go.mod b/go.mod index 2b1ee11a..69031aca 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/prometheus/client_golang v1.25.0 github.com/prometheus/client_model v0.6.3 github.com/swaggo/swag v1.16.6 + github.com/ulikunitz/xz v0.5.17 github.com/vmihailenco/msgpack/v5 v5.4.1 gocloud.dev v0.47.0 golang.org/x/mod v0.42.0 @@ -108,7 +109,6 @@ require ( github.com/rogpeppe/go-internal v1.16.0 // indirect github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect - github.com/ulikunitz/xz v0.5.17 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.46.0 // indirect diff --git a/internal/database/metadata_upstream_url_test.go b/internal/database/metadata_upstream_url_test.go index ef588639..cb5bafc5 100644 --- a/internal/database/metadata_upstream_url_test.go +++ b/internal/database/metadata_upstream_url_test.go @@ -67,7 +67,7 @@ func TestUpsertMetadataCacheKeepsUpstreamURL(t *testing.T) { }) } -func TestRecentMetadataURLs(t *testing.T) { +func TestRecentMetadataDocuments(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { upsertMetadataURL(t, db, "apk", "k1", "https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz") upsertMetadataURL(t, db, "apk", "k2", "https://a.test/v3.19/main/aarch64/APKINDEX.tar.gz") @@ -78,13 +78,13 @@ func TestRecentMetadataURLs(t *testing.T) { t.Fatal(err) } - got, err := db.RecentMetadataURLs("apk", time.Now().Add(-14*24*time.Hour)) + got, err := db.RecentMetadataDocuments("apk", time.Now().Add(-14*24*time.Hour)) if err != nil { t.Fatal(err) } - want := []string{"https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz"} + want := []MetadataDocument{{Name: "k1", UpstreamURL: "https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz"}} if !reflect.DeepEqual(got, want) { - t.Errorf("RecentMetadataURLs = %v, want %v", got, want) + t.Errorf("RecentMetadataDocuments = %+v, want %+v", got, want) } }) } diff --git a/internal/database/queries.go b/internal/database/queries.go index 3f74a6d8..20771e1b 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -1312,17 +1312,24 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err return &entry, nil } -// RecentMetadataURLs returns the upstream URLs of the documents of an -// ecosystem cached or served since cutoff, as far as they were recorded. -func (db *DB) RecentMetadataURLs(ecosystem string, cutoff time.Time) ([]string, error) { - var urls []string - err := db.Select(&urls, db.Rebind(` - SELECT DISTINCT upstream_url FROM metadata_cache +// MetadataDocument names a cached metadata document and where it was +// fetched from. +type MetadataDocument struct { + Name string `db:"name"` + UpstreamURL string `db:"upstream_url"` +} + +// RecentMetadataDocuments returns the documents of an ecosystem cached or +// served since cutoff whose upstream URL was recorded. +func (db *DB) RecentMetadataDocuments(ecosystem string, cutoff time.Time) ([]MetadataDocument, error) { + var docs []MetadataDocument + err := db.Select(&docs, db.Rebind(` + SELECT name, upstream_url FROM metadata_cache WHERE ecosystem = ? AND upstream_url IS NOT NULL AND COALESCE(last_accessed_at, updated_at) >= ? - ORDER BY upstream_url + ORDER BY upstream_url, name `), ecosystem, cutoff.UTC()) - return urls, err + return docs, err } // RefreshMetadataCache sets fetched_at on the metadata cache row for ecosystem diff --git a/internal/handler/os_package_identity.go b/internal/handler/os_package_identity.go new file mode 100644 index 00000000..0e0400be --- /dev/null +++ b/internal/handler/os_package_identity.go @@ -0,0 +1,17 @@ +package handler + +// OSPackageIdentity returns the name, version and architecture the deb, +// alpine and rpm download handlers cache a package file under, read from its +// path below the repository the way they read it. It returns empty strings +// for other ecosystems and for paths they would not cache. +func OSPackageIdentity(ecosystem, path string) (name, version, arch string) { + switch ecosystem { + case "deb": + return (*DebianHandler)(nil).parsePoolPath("/" + path) + case apkEcosystem: + return (*APKHandler)(nil).parseAPKPath(path) + case "rpm": + return (*RPMHandler)(nil).parseRPMPath(path) + } + return "", "", "" +} diff --git a/internal/handler/os_package_identity_test.go b/internal/handler/os_package_identity_test.go new file mode 100644 index 00000000..3d487bec --- /dev/null +++ b/internal/handler/os_package_identity_test.go @@ -0,0 +1,23 @@ +package handler + +import "testing" + +func TestOSPackageIdentity(t *testing.T) { + tests := []struct { + ecosystem, path string + name, version, arch string + }{ + {"deb", "pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb", "curl", "7.88.1-10+deb12u8", "amd64"}, + {"alpine", "v3.20/main/aarch64/curl-8.9.1-r0.apk", "curl", "8.9.1-r0", "aarch64"}, + {"rpm", "Packages/c/curl-8.6.0-10.fc40.x86_64.rpm", "curl", "8.6.0-10.fc40", "x86_64"}, + {"deb", "pool/main/c/curl/README", "", "", ""}, + {"npm", "lodash-4.17.21.tgz", "", "", ""}, + } + for _, tt := range tests { + name, version, arch := OSPackageIdentity(tt.ecosystem, tt.path) + if name != tt.name || version != tt.version || arch != tt.arch { + t.Errorf("OSPackageIdentity(%s, %s) = %q %q %q, want %q %q %q", + tt.ecosystem, tt.path, name, version, arch, tt.name, tt.version, tt.arch) + } + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index ac20a4ee..20686446 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -246,6 +246,15 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages) return } + if pv.fileURL != "" { + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, pv.Name, pv.Version, pv.cacheFilename, pv.fileURL, pv.digest) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + m.recordResult(tracker, pv, result) + return + } if eco, ok := fileEcosystems[pv.Ecosystem]; ok { m.mirrorFiles(ctx, pv, eco, tracker) return diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index bdc4c29f..a20ba8be 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -55,37 +55,72 @@ func (m *Mirror) releaseLister() ReleaseLister { //nolint:ireturn // any lister // it: releases within cooldown or on the denylist are left out. Packages it // can't list are counted as failed. func (m *Mirror) Refresh(ctx context.Context, r Refresh) (*Progress, error) { - packages, err := m.db.MostUsedPackages(time.Now().Add(-r.Since), r.Top) + cutoff := time.Now().Add(-r.Since) + packages, err := m.db.MostUsedPackages(cutoff, r.Top) if err != nil { return nil, fmt.Errorf("listing the most used packages: %w", err) } tracker := newProgressTracker() tracker.phase.Store("downloading") -refresh: - for _, pkg := range packages { - if ctx.Err() != nil { - break - } - items, err := m.refreshItems(ctx, pkg) - if err != nil { - m.recordFailure(tracker, PackageVersion{Ecosystem: pkg.Ecosystem, Name: pkg.Name}, err) - continue - } + run := func(items []PackageVersion) bool { for _, item := range items { + if ctx.Err() != nil { + return false + } if r.spent(tracker) { m.logger.Info("mirror refresh reached its cap for one run", "mirrored", tracker.completed.Load(), "bytes", tracker.bytes.Load()) - break refresh + return false } tracker.total.Add(1) m.mirrorOne(ctx, item, tracker) } + return true + } + for _, group := range byEcosystem(packages) { + if ix, ok := osIndexes[group[0].Ecosystem]; ok { + items, errs := m.indexRefreshItems(ctx, ix, group, cutoff) + for _, err := range errs { + m.recordFailure(tracker, PackageVersion{Ecosystem: ix.ecosystem}, err) + } + if !run(items) { + break + } + continue + } + done := true + for _, pkg := range group { + items, err := m.refreshItems(ctx, pkg) + if err != nil { + m.recordFailure(tracker, PackageVersion{Ecosystem: pkg.Ecosystem, Name: pkg.Name}, err) + continue + } + if done = run(items); !done { + break + } + } + if !done { + break + } } tracker.phase.Store("complete") p := tracker.snapshot() return &p, nil } +// byEcosystem splits packages, ordered by ecosystem, into one group per +// ecosystem. +func byEcosystem(packages []database.UsedPackage) [][]database.UsedPackage { + var groups [][]database.UsedPackage + for i, pkg := range packages { + if i == 0 || pkg.Ecosystem != packages[i-1].Ecosystem { + groups = append(groups, nil) + } + groups[len(groups)-1] = append(groups[len(groups)-1], pkg) + } + return groups +} + // refreshItems returns the versions to mirror for a used package. func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([]PackageVersion, error) { p, err := purl.Parse(pkg.PURL) @@ -130,7 +165,7 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] // refreshable reports whether a refresh can list the releases of an // ecosystem's packages. func refreshable(ecosystem string) bool { - if isImageEcosystem(ecosystem) { + if isImageEcosystem(ecosystem) || osIndexes[ecosystem].ecosystem != "" { return false } _, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", nil) diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go new file mode 100644 index 00000000..dc7818a0 --- /dev/null +++ b/internal/mirror/refresh_index.go @@ -0,0 +1,254 @@ +package mirror + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "path" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/vers" +) + +// A Debian, Alpine or RPM client finds new versions in a repository index +// rather than in metadata per package. A refresh reads the indexes clients +// fetched again, from the URLs the metadata cache recorded, and mirrors the +// newer files they list of the packages downloaded most. + +// osIndex describes the repository index of one ecosystem. +type osIndex struct { + // ecosystem is what package files are cached under, metaEcosystem what + // the index documents are cached under. + ecosystem, metaEcosystem string + // scheme orders versions. + scheme string + // base returns the repository base package paths are relative to, for + // the URL of an index document a refresh starts from, or false for + // other documents. + base func(indexURL string) (string, bool) + // files lists the package files of the index document body fetched + // from indexURL. cached are the files of the most used packages under + // the repository base, for indexes that list their files in parts. + files func(ctx context.Context, m *Mirror, indexURL string, body []byte, cached []cachedOSFile) ([]indexFile, error) +} + +var osIndexes = map[string]osIndex{ + "alpine": { + ecosystem: "alpine", metaEcosystem: "alpine", scheme: "alpine", + base: func(u string) (string, bool) { + return strings.CutSuffix(u, "/APKINDEX.tar.gz") + }, + files: func(_ context.Context, _ *Mirror, _ string, body []byte, _ []cachedOSFile) ([]indexFile, error) { + return parseAPKIndex(bytes.NewReader(body)) + }, + }, + "deb": { + ecosystem: "deb", metaEcosystem: "debian", scheme: "deb", + base: func(u string) (string, bool) { + if !strings.HasSuffix(u, "/InRelease") && !strings.HasSuffix(u, "/Release") { + return "", false + } + base, _, ok := strings.Cut(u, "/dists/") + return base, ok + }, + files: debIndexFiles, + }, + "rpm": { + ecosystem: "rpm", metaEcosystem: "rpm", scheme: "rpm", + base: func(u string) (string, bool) { + return strings.CutSuffix(u, "/repodata/repomd.xml") + }, + files: rpmIndexFiles, + }, +} + +// cachedOSFile is a cached file of a most used package, by its path below a +// repository base. +type cachedOSFile struct { + file database.UsedFile + rel string + name, version, arch string +} + +func (f cachedOSFile) key() string { return f.name + "\x00" + f.arch } + +// indexRefreshItems returns the newer files the recently fetched indexes of +// an ecosystem list for packages, and an error per index it couldn't read. +func (m *Mirror) indexRefreshItems(ctx context.Context, ix osIndex, packages []database.UsedPackage, cutoff time.Time) ([]PackageVersion, []error) { + docs, err := m.db.RecentMetadataDocuments(ix.metaEcosystem, cutoff) + if err != nil { + return nil, []error{fmt.Errorf("listing %s indexes: %w", ix.metaEcosystem, err)} + } + var items []PackageVersion + var errs []error + seenIndex := map[string]bool{} + seenFile := map[string]bool{} + for _, doc := range docs { + base, ok := ix.base(doc.UpstreamURL) + if !ok || seenIndex[urlDir(doc.UpstreamURL)] { + continue + } + cached := cachedUnder(ix.ecosystem, base, packages) + if len(cached) == 0 { + continue + } + seenIndex[urlDir(doc.UpstreamURL)] = true + body, _, err := m.proxy.FetchOrCacheMetadata(ctx, ix.metaEcosystem, doc.Name, doc.UpstreamURL, "*/*") + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", doc.UpstreamURL, err)) + continue + } + files, err := ix.files(ctx, m, doc.UpstreamURL, body, cached) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", doc.UpstreamURL, err)) + continue + } + for _, item := range m.newerIndexFiles(ix, base, cached, files) { + if !seenFile[item.fileURL] { + seenFile[item.fileURL] = true + items = append(items, item) + } + } + } + return items, errs +} + +// cachedUnder returns the cached files of packages below base. +func cachedUnder(ecosystem, base string, packages []database.UsedPackage) []cachedOSFile { + var cached []cachedOSFile + for _, pkg := range packages { + for _, f := range pkg.Files { + rel, ok := strings.CutPrefix(f.UpstreamURL, base+"/") + if !ok { + continue + } + name, version, arch := handler.OSPackageIdentity(ecosystem, rel) + if name != "" { + cached = append(cached, cachedOSFile{file: f, rel: rel, name: name, version: version, arch: arch}) + } + } + } + return cached +} + +// newerIndexFiles returns, per package and architecture cached below base, +// the newest file the index lists when it is newer than every cached one. +func (m *Mirror) newerIndexFiles(ix osIndex, base string, cached []cachedOSFile, files []indexFile) []PackageVersion { + newestCached := map[string]cachedOSFile{} + for _, c := range cached { + if current, ok := newestCached[c.key()]; !ok || vers.CompareWithScheme(c.version, current.version, ix.scheme) > 0 { + newestCached[c.key()] = c + } + } + newest := map[string]PackageVersion{} + var order []string + for _, f := range files { + name, version, arch := handler.OSPackageIdentity(ix.ecosystem, f.Path) + key := name + "\x00" + arch + c, ok := newestCached[key] + if name == "" || !ok || vers.CompareWithScheme(version, c.version, ix.scheme) <= 0 { + continue + } + if m.proxy != nil && !m.proxy.ReleaseAllowed(ix.ecosystem, name, version, time.Time{}) { + continue + } + if current, seen := newest[key]; seen && vers.CompareWithScheme(version, current.Version, ix.scheme) <= 0 { + continue + } else if !seen { + order = append(order, key) + } + cacheFilename := path.Base(f.Path) + if dir := path.Dir(c.file.Filename); dir != "." { + cacheFilename = dir + "/" + cacheFilename + } + newest[key] = PackageVersion{ + Ecosystem: ix.ecosystem, Name: name, Version: version, + fileURL: base + "/" + f.Path, cacheFilename: cacheFilename, digest: f.Digest, + } + } + items := make([]PackageVersion, 0, len(order)) + for _, key := range order { + items = append(items, newest[key]) + } + return items +} + +// debIndexFiles reads the Packages indexes an InRelease or Release file +// lists for the components and architectures of the cached files. +func debIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, cached []cachedOSFile) ([]indexFile, error) { + indexes := debPackagesIndexes(body) + wanted := map[string]bool{} + for _, c := range cached { + rest, ok := strings.CutPrefix(c.rel, "pool/") + if !ok { + continue + } + component, _, _ := strings.Cut(rest, "/") + if dir := component + "/binary-" + c.arch; indexes[dir] != "" { + wanted[dir] = true + continue + } + // Files for every architecture are listed in each binary-* index. + for dir := range indexes { + if strings.HasPrefix(dir, component+"/binary-") { + wanted[dir] = true + break + } + } + } + suite := urlDir(indexURL) + var files []indexFile + for dir := range wanted { + listed, err := m.readIndex(ctx, suite+"/"+indexes[dir], parseDebPackages) + if err != nil { + return nil, err + } + files = append(files, listed...) + } + return files, nil +} + +// rpmIndexFiles reads the primary metadata a repomd.xml points at. +func rpmIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, _ []cachedOSFile) ([]indexFile, error) { + primary, err := repomdPrimary(body) + if err != nil { + return nil, err + } + base := strings.TrimSuffix(indexURL, "/repodata/repomd.xml") + return m.readIndex(ctx, base+"/"+primary, parseRPMPrimary) +} + +// readIndex streams an index from upstream through parse, decompressed as +// its name says. It is read directly rather than through the metadata +// cache, since it can be far larger than a metadata document may be. +func (m *Mirror) readIndex(ctx context.Context, indexURL string, parse func(io.Reader) ([]indexFile, error)) ([]indexFile, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil) + if err != nil { + return nil, err + } + resp, err := m.proxy.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s: upstream returned %d", indexURL, resp.StatusCode) + } + r, err := decompressByName(indexURL, resp.Body) + if err != nil { + return nil, err + } + defer func() { _ = r.Close() }() + return parse(r) +} + +// urlDir returns a URL without its last path segment. path.Dir would fold +// the slashes after the scheme. +func urlDir(u string) string { + return u[:strings.LastIndex(u, "/")] +} diff --git a/internal/mirror/refresh_index_parse.go b/internal/mirror/refresh_index_parse.go new file mode 100644 index 00000000..4ae98576 --- /dev/null +++ b/internal/mirror/refresh_index_parse.go @@ -0,0 +1,238 @@ +package mirror + +import ( + "archive/tar" + "bufio" + "compress/bzip2" + "compress/gzip" + "encoding/xml" + "errors" + "fmt" + "io" + "strings" + + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +// indexFile is a package file a repository index lists: its path below the +// repository base and, where the index gives one, its SHA-256 digest. +type indexFile struct { + Path string + Digest string +} + +// maxIndexLine bounds one line of a Debian or Alpine index. +const maxIndexLine = 1 << 20 + +// parseAPKIndex reads the files an APKINDEX.tar.gz lists. Its checksums +// cover only the control section of a package, so no digest is returned. +func parseAPKIndex(r io.Reader) ([]indexFile, error) { + gz, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer func() { _ = gz.Close() }() + // An APKINDEX.tar.gz may be several gzip streams: the signature, then the + // index. Reading on across them finds the index whichever comes first. + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil, errors.New("APKINDEX.tar.gz has no APKINDEX") + } + if err != nil { + return nil, err + } + if hdr.Name == "APKINDEX" { + return parseAPKIndexText(tr) + } + } +} + +func parseAPKIndexText(r io.Reader) ([]indexFile, error) { + var files []indexFile + var name, version string + flush := func() { + if name != "" && version != "" { + files = append(files, indexFile{Path: name + "-" + version + ".apk"}) + } + name, version = "", "" + } + scanner := bufio.NewScanner(r) + scanner.Buffer(nil, maxIndexLine) + for scanner.Scan() { + line := scanner.Text() + switch { + case line == "": + flush() + case strings.HasPrefix(line, "P:"): + name = line[2:] + case strings.HasPrefix(line, "V:"): + version = line[2:] + } + } + flush() + return files, scanner.Err() +} + +// debIndexPreference ranks the compressions of a Packages index, best first. +var debIndexPreference = []string{".xz", ".gz", ".bz2", ""} + +// debPackagesIndexes returns, per component/binary-arch directory, the path +// of the Packages index a Release or InRelease file lists, in the best +// compression it offers. +func debPackagesIndexes(release []byte) map[string]string { + listed := map[string]bool{} + inSHA256 := false + for _, line := range strings.Split(string(release), "\n") { + line = strings.TrimRight(line, "\r") + if !strings.HasPrefix(line, " ") { + inSHA256 = line == "SHA256:" + continue + } + if !inSHA256 { + continue + } + fields := strings.Fields(line) + if len(fields) == 3 { //nolint:mnd // checksum, size, path + listed[fields[2]] = true + } + } + indexes := map[string]string{} + for path := range listed { + dir, file, ok := strings.Cut(path, "/Packages") + if !ok || !strings.Contains(dir, "/binary-") || strings.Contains(file, "/") { + continue + } + for _, ext := range debIndexPreference { + if listed[dir+"/Packages"+ext] { + indexes[dir] = dir + "/Packages" + ext + break + } + } + } + return indexes +} + +// parseDebPackages reads the files a Packages index lists. +func parseDebPackages(r io.Reader) ([]indexFile, error) { + var files []indexFile + var current indexFile + flush := func() { + if current.Path != "" { + files = append(files, current) + } + current = indexFile{} + } + scanner := bufio.NewScanner(r) + scanner.Buffer(nil, maxIndexLine) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + flush() + continue + } + key, value, ok := strings.Cut(line, ": ") + if !ok || strings.HasPrefix(line, " ") { + continue + } + switch key { + case "Filename": + current.Path = value + case "SHA256": + current.Digest = "sha256:" + value + } + } + flush() + return files, scanner.Err() +} + +// repomdPrimary returns the location of the primary metadata a repomd.xml +// points at. +func repomdPrimary(body []byte) (string, error) { + var doc struct { + Data []struct { + Type string `xml:"type,attr"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"location"` + } `xml:"data"` + } + if err := xml.Unmarshal(body, &doc); err != nil { + return "", err + } + for _, d := range doc.Data { + if d.Type == "primary" && d.Location.Href != "" { + return d.Location.Href, nil + } + } + return "", errors.New("repomd.xml lists no primary metadata") +} + +// parseRPMPrimary reads the binary packages a primary.xml lists, one at a +// time. Source packages are left out. +func parseRPMPrimary(r io.Reader) ([]indexFile, error) { + dec := xml.NewDecoder(r) + var files []indexFile + for { + tok, err := dec.Token() + if errors.Is(err, io.EOF) { + return files, nil + } + if err != nil { + return nil, err + } + start, ok := tok.(xml.StartElement) + if !ok || start.Name.Local != "package" { + continue + } + var pkg struct { + Arch string `xml:"arch"` + Checksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` + } `xml:"checksum"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"location"` + } + if err := dec.DecodeElement(&pkg, &start); err != nil { + return nil, err + } + if pkg.Arch == "src" || pkg.Location.Href == "" { + continue + } + file := indexFile{Path: pkg.Location.Href} + if pkg.Checksum.Type == "sha256" { + file.Digest = "sha256:" + strings.TrimSpace(pkg.Checksum.Value) + } + files = append(files, file) + } +} + +// decompressByName returns r decompressed as the extension of name says. +func decompressByName(name string, r io.Reader) (io.ReadCloser, error) { + switch { + case strings.HasSuffix(name, ".gz"): + return gzip.NewReader(r) + case strings.HasSuffix(name, ".xz"): + x, err := xz.NewReader(r) + if err != nil { + return nil, err + } + return io.NopCloser(x), nil + case strings.HasSuffix(name, ".zst"): + z, err := zstd.NewReader(r) + if err != nil { + return nil, err + } + return z.IOReadCloser(), nil + case strings.HasSuffix(name, ".bz2"): + return io.NopCloser(bzip2.NewReader(r)), nil + case strings.HasSuffix(name, ".xml"), !strings.Contains(name[strings.LastIndex(name, "/")+1:], "."): + return io.NopCloser(r), nil + default: + return nil, fmt.Errorf("unsupported compression: %s", name) + } +} diff --git a/internal/mirror/refresh_index_parse_test.go b/internal/mirror/refresh_index_parse_test.go new file mode 100644 index 00000000..3b68805f --- /dev/null +++ b/internal/mirror/refresh_index_parse_test.go @@ -0,0 +1,188 @@ +package mirror + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "reflect" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" +) + +// apkIndexArchive packs an APKINDEX into the tar.gz Alpine serves. +func apkIndexArchive(t *testing.T, index string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for name, body := range map[string]string{"DESCRIPTION": "v3.20.3-1-g0000000", "APKINDEX": index} { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestParseAPKIndex(t *testing.T) { + index := "C:Q1abc=\nP:curl\nV:8.9.1-r0\nA:aarch64\nS:100\n\nC:Q1def=\nP:busybox\nV:1.36.1-r29\nA:aarch64\n\n" + got, err := parseAPKIndex(bytes.NewReader(apkIndexArchive(t, index))) + if err != nil { + t.Fatal(err) + } + want := []indexFile{{Path: "curl-8.9.1-r0.apk"}, {Path: "busybox-1.36.1-r29.apk"}} + if !reflect.DeepEqual(got, want) { + t.Errorf("parseAPKIndex = %+v, want %+v", got, want) + } +} + +const debRelease = `-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA256 + +Origin: Debian +Suite: stable +Codename: bookworm +Architectures: all amd64 arm64 +Components: main contrib +MD5Sum: + 0123 100 main/binary-amd64/Packages.xz +SHA256: + aaaa 1000 main/binary-amd64/Packages + bbbb 200 main/binary-amd64/Packages.xz + cccc 300 main/binary-amd64/Packages.gz + dddd 400 main/binary-arm64/Packages.gz + eeee 500 contrib/binary-amd64/Packages.xz + ffff 600 main/i18n/Translation-en.bz2 +-----BEGIN PGP SIGNATURE----- +xyz +-----END PGP SIGNATURE----- +` + +func TestDebPackagesIndexes(t *testing.T) { + got := debPackagesIndexes([]byte(debRelease)) + want := map[string]string{ + "main/binary-amd64": "main/binary-amd64/Packages.xz", + "main/binary-arm64": "main/binary-arm64/Packages.gz", + "contrib/binary-amd64": "contrib/binary-amd64/Packages.xz", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("debPackagesIndexes = %v, want %v", got, want) + } +} + +func TestParseDebPackages(t *testing.T) { + packages := `Package: curl +Version: 7.88.1-10+deb12u8 +Architecture: amd64 +Description: a tool + with a long description +Filename: pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb +SHA256: 1111 + +Package: zlib1g +Version: 1:1.2.13.dfsg-1 +Architecture: amd64 +Filename: pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb +SHA256: 2222 +` + got, err := parseDebPackages(strings.NewReader(packages)) + if err != nil { + t.Fatal(err) + } + want := []indexFile{ + {Path: "pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb", Digest: "sha256:1111"}, + {Path: "pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb", Digest: "sha256:2222"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("parseDebPackages = %+v, want %+v", got, want) + } +} + +const repomd = ` + + 1700000000 + + + +` + +func TestRepomdPrimary(t *testing.T) { + got, err := repomdPrimary([]byte(repomd)) + if err != nil || got != "repodata/bbbb-primary.xml.zst" { + t.Errorf("repomdPrimary = %q, %v", got, err) + } + if _, err := repomdPrimary([]byte(``)); err == nil { + t.Error("repomdPrimary accepted a repomd without primary") + } +} + +const rpmPrimary = ` + + + curl + x86_64 + + 3333 + + + + curl + src + + 4444 + + + + tzdata + noarch + + 5555 + + + +` + +func TestParseRPMPrimary(t *testing.T) { + got, err := parseRPMPrimary(strings.NewReader(rpmPrimary)) + if err != nil { + t.Fatal(err) + } + want := []indexFile{ + {Path: "Packages/c/curl-8.6.0-10.fc40.x86_64.rpm", Digest: "sha256:3333"}, + {Path: "Packages/t/tzdata-2024a-5.fc40.noarch.rpm"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("parseRPMPrimary = %+v, want %+v", got, want) + } +} + +func TestDecompressByName(t *testing.T) { + var gz bytes.Buffer + w := gzip.NewWriter(&gz) + _, _ = w.Write([]byte("hello")) + _ = w.Close() + enc, _ := zstd.NewWriter(nil) + zst := enc.EncodeAll([]byte("hello"), nil) + + for name, data := range map[string][]byte{"Packages": []byte("hello"), "Packages.gz": gz.Bytes(), "primary.xml.zst": zst} { + r, err := decompressByName(name, bytes.NewReader(data)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + var out bytes.Buffer + if _, err := out.ReadFrom(r); err != nil || out.String() != "hello" { + t.Errorf("%s: got %q, %v", name, out.String(), err) + } + _ = r.Close() + } +} diff --git a/internal/mirror/refresh_index_test.go b/internal/mirror/refresh_index_test.go new file mode 100644 index 00000000..1f66c388 --- /dev/null +++ b/internal/mirror/refresh_index_test.go @@ -0,0 +1,231 @@ +package mirror + +import ( + "bytes" + "compress/gzip" + "context" + "net/http" + "net/http/httptest" + "slices" + "sync" + "testing" + "time" +) + +// indexServer serves repository index documents that a test can change. +type indexServer struct { + *httptest.Server + mu sync.Mutex + docs map[string][]byte +} + +func newIndexServer(t *testing.T) *indexServer { + t.Helper() + s := &indexServer{docs: map[string][]byte{}} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + body, ok := s.docs[r.URL.Path] + s.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(s.Close) + return s +} + +func (s *indexServer) set(path string, body []byte) { + s.mu.Lock() + defer s.mu.Unlock() + s.docs[path] = body +} + +func gzipped(t *testing.T, s string) []byte { + t.Helper() + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + if _, err := w.Write([]byte(s)); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// setupIndexRefresh returns a mirror whose package files come from a +// servingFetcher and whose indexes come from an indexServer. +func setupIndexRefresh(t *testing.T) (*Mirror, *servingFetcher, *indexServer) { + t.Helper() + m, fetcher := setupRefreshMirror(t, "package") + idx := newIndexServer(t) + m.proxy.HTTPClient = idx.Client() + m.proxy.CacheMetadata = true + return m, fetcher, idx +} + +// clientFetchesCurl caches an index document and a curl package file the +// way the handlers do for a client. +func clientFetchesCurl(t *testing.T, m *Mirror, metaEcosystem, indexURL, ecosystem, version, cacheFilename, fileURL string) { + t.Helper() + ctx := context.Background() + if _, _, err := m.proxy.FetchOrCacheMetadata(ctx, metaEcosystem, "key:"+indexURL, indexURL, "*/*"); err != nil { + t.Fatalf("fetching %s: %v", indexURL, err) + } + result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, ecosystem, "curl", version, cacheFilename, fileURL) + if err != nil { + t.Fatalf("fetching %s: %v", fileURL, err) + } + _ = result.Reader.Close() +} + +func refreshOnce(t *testing.T, m *Mirror) *Progress { + t.Helper() + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + return progress +} + +func cachedNames(t *testing.T, m *Mirror, ecosystem, filename string) []string { + t.Helper() + names, err := m.db.CachedArtifactNames(ecosystem, filename) + if err != nil { + t.Fatal(err) + } + return names +} + +func TestRefreshAlpineIndex(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + dir := "/alpine/v3.20/main/aarch64" + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.0-r0\nA:aarch64\n\nP:busybox\nV:1.36.1-r29\n\n")) + clientFetchesCurl(t, m, "alpine", idx.URL+dir+"/APKINDEX.tar.gz", + "alpine", "8.9.0-r0", "edge/v3.20/main/aarch64/curl-8.9.0-r0.apk", idx.URL+dir+"/curl-8.9.0-r0.apk") + + // A new release of curl; busybox was never downloaded and stays out. + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.1-r0\nA:aarch64\n\nP:busybox\nV:1.36.1-r30\n\n")) + fetcher.reset() + progress := refreshOnce(t, m) + + if want := []string{idx.URL + dir + "/curl-8.9.1-r0.apk"}; !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 { + t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) + } + // Cached where the APK handler looks for it. + if names := cachedNames(t, m, "alpine", "edge/v3.20/main/aarch64/curl-8.9.1-r0.apk"); !slices.Equal(names, []string{"curl"}) { + t.Errorf("cached names = %v, want [curl]", names) + } + + fetcher.reset() + if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("second run: progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } +} + +func TestRefreshIndexNeverDowngrades(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + dir := "/alpine/v3.20/main/aarch64" + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.1-r0\n\n")) + clientFetchesCurl(t, m, "alpine", idx.URL+dir+"/APKINDEX.tar.gz", + "alpine", "8.9.1-r0", "edge/v3.20/main/aarch64/curl-8.9.1-r0.apk", idx.URL+dir+"/curl-8.9.1-r0.apk") + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.0-r0\n\n")) + fetcher.reset() + + if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } +} + +func TestRefreshDebianIndex(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + release := []byte("Suite: stable\nSHA256:\n 0000 1 main/binary-amd64/Packages.gz\n 0000 1 main/binary-arm64/Packages.gz\n") + packages := func(curl string) []byte { + return gzipped(t, "Package: curl\nVersion: "+curl+"\nArchitecture: amd64\nFilename: pool/main/c/curl/curl_"+curl+"_amd64.deb\nSHA256: "+ + "bc4a71180870f7945155fbb02f4b0a2e3faa2a62d6d31b7039013055ed19869a\n\n"+ + "Package: zlib1g\nVersion: 1:1.2.13.dfsg-1\nArchitecture: amd64\nFilename: pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb\n\n") + } + idx.set("/debian/dists/bookworm/InRelease", release) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("7.88.1-10")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease", + "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", idx.URL+"/debian/pool/main/c/curl/curl_7.88.1-10_amd64.deb") + + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("7.88.1-10+deb12u8")) + fetcher.reset() + progress := refreshOnce(t, m) + + // Only the amd64 index is read: nothing was downloaded for arm64. The + // file is checked against the index's SHA-256 of "package". + want := []string{idx.URL + "/debian/pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb"} + if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) + } + if names := cachedNames(t, m, "deb", "curl_7.88.1-10+deb12u8_amd64.deb"); !slices.Equal(names, []string{"curl"}) { + t.Errorf("cached names = %v, want [curl]", names) + } +} + +func TestRefreshDebianIndexChecksDigest(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + idx.set("/debian/dists/bookworm/InRelease", []byte("SHA256:\n 0000 1 main/binary-amd64/Packages\n")) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages", []byte("Package: curl\nFilename: pool/main/c/curl/curl_7.88.1-10_amd64.deb\n\n")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease", + "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", idx.URL+"/debian/pool/main/c/curl/curl_7.88.1-10_amd64.deb") + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages", []byte( + "Package: curl\nFilename: pool/main/c/curl/curl_7.88.1-11_amd64.deb\nSHA256: 0000000000000000000000000000000000000000000000000000000000000000\n\n")) + fetcher.reset() + + progress := refreshOnce(t, m) + if progress.Failed != 1 || progress.Completed != 0 { + t.Errorf("progress = %+v, want the file failing its digest", progress) + } + if names := cachedNames(t, m, "deb", "curl_7.88.1-11_amd64.deb"); len(names) != 0 { + t.Errorf("a file failing its digest was cached as %v", names) + } +} + +func TestRefreshRPMIndex(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + base := "/fedora/updates/40/Everything/x86_64" + primary := func(rel string) []byte { + return gzipped(t, `curlx86_64`+ + ``) + } + repomdFor := func(href string) []byte { + return []byte(``) + } + idx.set(base+"/repodata/repomd.xml", repomdFor("repodata/aaaa-primary.xml.gz")) + idx.set(base+"/repodata/aaaa-primary.xml.gz", primary("10")) + clientFetchesCurl(t, m, "rpm", idx.URL+base+"/repodata/repomd.xml", + "rpm", "8.6.0-10.fc40", "curl-8.6.0-10.fc40.x86_64.rpm", idx.URL+base+"/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm") + + idx.set(base+"/repodata/repomd.xml", repomdFor("repodata/bbbb-primary.xml.gz")) + idx.set(base+"/repodata/bbbb-primary.xml.gz", primary("11")) + fetcher.reset() + progress := refreshOnce(t, m) + + want := []string{idx.URL + base + "/Packages/c/curl-8.6.0-11.fc40.x86_64.rpm"} + if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 { + t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) + } + if names := cachedNames(t, m, "rpm", "curl-8.6.0-11.fc40.x86_64.rpm"); !slices.Equal(names, []string{"curl"}) { + t.Errorf("cached names = %v, want [curl]", names) + } +} + +func TestRefreshIndexWithoutRecordedURLDoesNothing(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + result, err := m.proxy.GetOrFetchArtifactFromURL(context.Background(), "alpine", "curl", "8.9.0-r0", + "edge/v3.20/main/aarch64/curl-8.9.0-r0.apk", idx.URL+"/alpine/v3.20/main/aarch64/curl-8.9.0-r0.apk") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + fetcher.reset() + + if progress := refreshOnce(t, m); progress.Completed != 0 || progress.Failed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 30c4e0b7..8ad47687 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -25,6 +25,10 @@ type PackageVersion struct { // variants, when set, limit the files of a version that are mirrored to // those of these variants (see fileEcosystem.variant). variants map[string]bool + // fileURL, when set, is the one file to mirror, cached as cacheFilename + // and checked against digest when there is one. A refresh finds it in + // a repository index. + fileURL, cacheFilename, digest string } func (pv PackageVersion) String() string { From f80c826e735791fa38a85d365e7fe469785662e1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 14:09:02 +0100 Subject: [PATCH 04/15] Document mirror.refresh --- config.example.yaml | 14 ++++++++++++-- docs/configuration.md | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index d3b6da49..0ee2601e 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -120,8 +120,8 @@ storage: # Store fetched artifacts. Set to false to stream every download from # upstream without storing it; metadata filtering, cooldown and the # denylist still apply. Useful when another cache sits in front of the - # proxy. false is incompatible with direct_serve, scanning, mirror_api and - # mirror.schedules. + # proxy. false is incompatible with direct_serve, scanning, mirror_api, + # mirror.schedules and mirror.refresh. cache_artifacts: true # Redirect cached artifact downloads to presigned storage URLs (HTTP 302) @@ -449,6 +449,16 @@ gradle: # - images: ["pkg:docker/library/alpine@3.20", "pkg:docker/library/nginx@stable"] # interval: "6h" # platforms: ["linux/amd64"] +# +# Mirror new releases of the most used packages in the background, so the +# first install after a release doesn't wait on the upstream. Cooldown and +# the denylist apply. +# refresh: +# interval: "24h" +# top: 50 # packages per ecosystem, by recent downloads +# since: "14d" +# max_versions: 200 # per run; default no cap +# max_bytes: "2GB" # per run; default no cap # Health endpoint configuration. health: diff --git a/docs/configuration.md b/docs/configuration.md index 3e94531a..d2cad093 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -91,7 +91,7 @@ storage: Every download is a fresh upstream fetch, and concurrent requests for the same artifact are not combined. Artifacts with a digest known up front (OCI blobs, Swift archives, Helm charts) are verified while streaming: the response is sent chunked, and on a mismatch the connection is aborted before the response completes so the client never receives a tampered artifact as a good one. The same happens when the upstream connection fails mid-download. -`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve`, `mirror_api` or `mirror.schedules`, which all need stored artifacts, and the `mirror` command refuses to run with it. +`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve`, `mirror_api`, `mirror.schedules` or `mirror.refresh`, which all need stored artifacts, and the `mirror` command refuses to run with it. ### Retention @@ -1737,6 +1737,37 @@ Each run checks the tag with a HEAD request. If the tag still points to the cach - Mirrored layers are scanned when `scanning.enabled` is set. A blocked layer fails the run and keeps the previous image. - Schedules are read at startup; changes need a restart. They do not need `mirror_api`. +### Refreshing the most used packages + +For packages, the cache learns about a new release only when a client asks for it, and that first install waits on the upstream or fails while it is down. `mirror.refresh` mirrors new releases of the packages clients download most, so they are cached before anyone asks: + +```yaml +mirror: + refresh: + interval: 24h + top: 50 + since: 14d + max_versions: 200 + max_bytes: 2GB +``` + +| Setting | Env | Description | +|---------|-----|-------------| +| `mirror.refresh.interval` | - | How often the refresh runs, at least `1m` | +| `mirror.refresh.top` | - | Packages per ecosystem to refresh, the ones with the most downloads among those downloaded within `since`. Default: `50` | +| `mirror.refresh.since` | - | How recently a package must have been downloaded, with a `d` suffix for days. Default: `14d` | +| `mirror.refresh.max_versions` | - | Stop a run after mirroring this many new versions. Default: no cap | +| `mirror.refresh.max_bytes` | - | Stop a run once it has downloaded this much, such as `2GB`. It is checked before each version, so the last one can go over it. Default: no cap | + +For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. + +- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python, ABI and platform tags, and the sdist, or conda packages of the same subdir, build variant (such as `py312`) and file type. +- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newer file of each package and architecture. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. +- Container images are not refreshed; list the tags to follow in `mirror.schedules`. +- Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. +- Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. +- What a run mirrors is evicted by `storage.max_size` and retention like any other file. With `max_versions` and `max_bytes` a run stays bounded when many packages release at once. + ## Docker ### SQLite with Local Storage From e50179d0ccb3d80dc2fc9c1faf32775268b32934 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 14:09:59 +0100 Subject: [PATCH 05/15] Refresh Swift and Helm packages, leave Julia out --- docs/configuration.md | 2 +- internal/mirror/refresh.go | 12 +++++++----- internal/mirror/refresh_test.go | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d2cad093..2323ec3e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1763,7 +1763,7 @@ For each of those packages a run lists the releases on the configured upstream a - PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python, ABI and platform tags, and the sdist, or conda packages of the same subdir, build variant (such as `py312`) and file type. - Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newer file of each package and architecture. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. -- Container images are not refreshed; list the tags to follow in `mirror.schedules`. +- Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. - Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. - What a run mirrors is evicted by `storage.max_size` and retention like any other file. With `max_versions` and `max_bytes` a run stays bounded when many packages release at once. diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index a20ba8be..fc9317a2 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -3,6 +3,7 @@ package mirror import ( "context" "fmt" + "slices" "time" "github.com/git-pkgs/proxy/internal/database" @@ -162,12 +163,13 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] return items, nil } -// refreshable reports whether a refresh can list the releases of an -// ecosystem's packages. +// refreshable reports whether a refresh lists the releases of an +// ecosystem's packages. Images follow mirror.schedules, OS packages their +// repository indexes, and Julia files are cached by tree hash rather than by +// version. func refreshable(ecosystem string) bool { - if isImageEcosystem(ecosystem) || osIndexes[ecosystem].ecosystem != "" { + if isImageEcosystem(ecosystem) || osIndexes[ecosystem].ecosystem != "" || ecosystem == "julia" { return false } - _, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", nil) - return err == nil + return slices.Contains(registries.SupportedEcosystems(), purl.EcosystemToPURLType(ecosystem)) } diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index 684b44c8..4a9f0835 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -304,3 +304,20 @@ func TestFileVariants(t *testing.T) { } } } + +func TestRefreshable(t *testing.T) { + for ecosystem, want := range map[string]bool{ + "npm": true, "pypi": true, "conda": true, "composer": true, + // Their registries need the configured upstream URL. + "swift": true, "helm": true, + // Julia files are cached by tree hash, not by version. + "julia": false, + // Images follow mirror.schedules; OS packages their indexes. + "docker": false, "oci": false, "deb": false, "alpine": false, "rpm": false, + "generic": false, + } { + if got := refreshable(ecosystem); got != want { + t.Errorf("refreshable(%s) = %v, want %v", ecosystem, got, want) + } + } +} From b73980e8edaadcd5d0725d80e8b8bff8eaa05098 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 17:58:00 +0100 Subject: [PATCH 06/15] Refresh each suite of a shared Debian pool on its own An index file was mirrored only when it was newer than every cached version of the package and architecture. Debian suites share one pool, so once curl from trixie was cached, bookworm's security updates to curl were older than it and never mirrored. A run now mirrors the newest file an index lists when it isn't cached and an older version is, which still never downgrades and leaves a suite alone that only has a newer version cached. Tests also cover repositories listing several versions, an unreadable index next to working ones, Architecture: all packages, scoped npm packages, denied releases, a listing failure next to other packages, a cancelled run and Go version lines. --- docs/configuration.md | 2 +- internal/mirror/refresh_harden_test.go | 231 +++++++++++++++++++++++++ internal/mirror/refresh_index.go | 59 +++++-- 3 files changed, 272 insertions(+), 20 deletions(-) create mode 100644 internal/mirror/refresh_harden_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 2323ec3e..7aea7fc4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1762,7 +1762,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. - PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python, ABI and platform tags, and the sdist, or conda packages of the same subdir, build variant (such as `py312`) and file type. -- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newer file of each package and architecture. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. +- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. - Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go new file mode 100644 index 00000000..458f1e12 --- /dev/null +++ b/internal/mirror/refresh_harden_test.go @@ -0,0 +1,231 @@ +package mirror + +import ( + "context" + "reflect" + "slices" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/registries" +) + +// debCurlPackages returns a Packages index listing curl at version. +func debCurlPackages(t *testing.T, version string) []byte { + t.Helper() + return gzipped(t, "Package: curl\nVersion: "+version+"\nArchitecture: amd64\nFilename: pool/main/c/curl/curl_"+version+"_amd64.deb\n\n") +} + +// TestRefreshDebianIndexPerSuite checks that suites sharing a pool are +// refreshed each on its own: a newer curl cached from trixie must not keep +// bookworm's security update from being mirrored. +func TestRefreshDebianIndexPerSuite(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n") + pool := idx.URL + "/debian/pool/main/c/curl/" + for suite, version := range map[string]string{"bookworm": "7.88.1-10", "trixie": "8.14.1-2"} { + idx.set("/debian/dists/"+suite+"/InRelease", release) + idx.set("/debian/dists/"+suite+"/main/binary-amd64/Packages.gz", debCurlPackages(t, version)) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/"+suite+"/InRelease", + "deb", version, "curl_"+version+"_amd64.deb", pool+"curl_"+version+"_amd64.deb") + } + + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10+deb12u8")) + idx.set("/debian/dists/trixie/main/binary-amd64/Packages.gz", debCurlPackages(t, "8.14.1-2+deb13u1")) + fetcher.reset() + progress := refreshOnce(t, m) + + want := []string{pool + "curl_7.88.1-10+deb12u8_amd64.deb", pool + "curl_8.14.1-2+deb13u1_amd64.deb"} + got := fetcher.fetched() + slices.Sort(got) + if !slices.Equal(got, want) || progress.Completed != 2 { + t.Errorf("progress = %+v, fetched %v; want %v", progress, got, want) + } +} + +// TestRefreshDebianIndexSkipsOtherSuite checks that a suite's version of a +// package is not mirrored when only a newer one is cached: clients of that +// suite never fetched the package. +func TestRefreshDebianIndexSkipsOtherSuite(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n") + pool := idx.URL + "/debian/pool/main/c/curl/" + idx.set("/debian/dists/trixie/InRelease", release) + idx.set("/debian/dists/trixie/main/binary-amd64/Packages.gz", debCurlPackages(t, "8.14.1-2")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/trixie/InRelease", + "deb", "8.14.1-2", "curl_8.14.1-2_amd64.deb", pool+"curl_8.14.1-2_amd64.deb") + // A bookworm client fetched its index but never curl. + idx.set("/debian/dists/bookworm/InRelease", release) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10+deb12u8")) + if _, _, err := m.proxy.FetchOrCacheMetadata(t.Context(), "debian", "key:bookworm", idx.URL+"/debian/dists/bookworm/InRelease", "*/*"); err != nil { + t.Fatal(err) + } + fetcher.reset() + + if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } +} + +// TestRefreshRPMIndexListingSeveralVersions checks that a repository +// listing old versions next to the newest mirrors only the newest, and +// nothing once it is cached. +func TestRefreshRPMIndexListingSeveralVersions(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + base := "/fedora/updates/40/Everything/x86_64" + primary := func(rels ...string) []byte { + var pkgs string + for _, rel := range rels { + pkgs += `curlx86_64` + } + return gzipped(t, ``+pkgs+``) + } + idx.set(base+"/repodata/repomd.xml", []byte(``)) + idx.set(base+"/repodata/primary.xml.gz", primary("10")) + clientFetchesCurl(t, m, "rpm", idx.URL+base+"/repodata/repomd.xml", + "rpm", "8.6.0-10.fc40", "curl-8.6.0-10.fc40.x86_64.rpm", idx.URL+base+"/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm") + + idx.set(base+"/repodata/primary.xml.gz", primary("10", "12", "11")) + fetcher.reset() + progress := refreshOnce(t, m) + want := []string{idx.URL + base + "/Packages/c/curl-8.6.0-12.fc40.x86_64.rpm"} + if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 { + t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) + } + + fetcher.reset() + if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("second run: progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } +} + +// TestRefreshIndexFailureDoesNotStopRun checks that an index that can't be +// read is counted as failed and the other indexes are still refreshed. +func TestRefreshIndexFailureDoesNotStopRun(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n") + pool := idx.URL + "/debian/pool/main/c/curl/" + for suite, version := range map[string]string{"bookworm": "7.88.1-10", "trixie": "8.14.1-2"} { + idx.set("/debian/dists/"+suite+"/InRelease", release) + idx.set("/debian/dists/"+suite+"/main/binary-amd64/Packages.gz", debCurlPackages(t, version)) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/"+suite+"/InRelease", + "deb", version, "curl_"+version+"_amd64.deb", pool+"curl_"+version+"_amd64.deb") + } + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", []byte("not gzip")) + idx.set("/debian/dists/trixie/main/binary-amd64/Packages.gz", debCurlPackages(t, "8.14.1-2+deb13u1")) + fetcher.reset() + + progress := refreshOnce(t, m) + want := []string{pool + "curl_8.14.1-2+deb13u1_amd64.deb"} + if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 || progress.Failed != 1 { + t.Errorf("progress = %+v, fetched %v; want %v and one failure", progress, fetcher.fetched(), want) + } +} + +func TestNewestPerCachedLineGo(t *testing.T) { + allowAll := func(registries.Version) bool { return true } + for _, tt := range []struct { + name string + cached []string + releases []registries.Version + want []string + }{ + {"v prefix", []string{"v1.2.3"}, releases("v1.2.3", "v1.3.0"), []string{"v1.3.0"}}, + {"incompatible is its own line", []string{"v1.2.3"}, releases("v1.2.3", "v2.0.0+incompatible"), nil}, + {"pseudo-versions are prereleases", []string{"v1.2.3"}, releases("v1.2.3", "v1.2.4-0.20260101000000-abcdefabcdef"), nil}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := newestPerCachedLine("golang", tt.cached, tt.releases, allowAll); !reflect.DeepEqual(got, tt.want) { + t.Errorf("newestPerCachedLine = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRefreshScopedNPMPackage(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "@babel/core", "7.24.0") + m.Versions = stubReleases{"npm/@babel/core": oldReleases("7.24.0", "7.25.2")} + fetcher.reset() + + progress := refreshOnce(t, m) + if urls := fetcher.fetched(); progress.Completed != 1 || len(fetchedMatching(urls, "core-7.25.2.tgz")) != 1 { + t.Errorf("progress = %+v, fetched %v; want @babel/core 7.25.2", progress, urls) + } +} + +func TestRefreshSkipsDeniedRelease(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21", "4.17.22")} + policy, err := denylist.New([]string{"pkg:npm/lodash@4.17.22"}) + if err != nil { + t.Fatal(err) + } + m.proxy.Denylist = policy + fetcher.reset() + + progress := refreshOnce(t, m) + if urls := fetcher.fetched(); progress.Completed != 1 || len(fetchedMatching(urls, "lodash-4.17.21.tgz")) != 1 || len(fetchedMatching(urls, "4.17.22")) != 0 { + t.Errorf("progress = %+v, fetched %v; want 4.17.21, not the denied 4.17.22", progress, urls) + } +} + +func TestRefreshListingFailureDoesNotStopOthers(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + cacheNPM(t, m, "react", "18.3.0") + m.Versions = stubReleases{"npm/react": oldReleases("18.3.0", "18.3.1")} + fetcher.reset() + + progress := refreshOnce(t, m) + if urls := fetcher.fetched(); progress.Completed != 1 || progress.Failed != 1 || len(fetchedMatching(urls, "react-18.3.1.tgz")) != 1 { + t.Errorf("progress = %+v, fetched %v; want react refreshed and lodash failed", progress, urls) + } +} + +func TestRefreshStopsWhenCancelled(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21")} + fetcher.reset() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want nothing after cancelling", progress, fetcher.fetched()) + } +} + +// TestRefreshDebianArchitectureAll checks that a package for every +// architecture, listed in each binary-* index rather than one of its own, +// is refreshed. +func TestRefreshDebianArchitectureAll(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + packages := func(version string) []byte { + return gzipped(t, "Package: ca-certificates\nVersion: "+version+"\nArchitecture: all\nFilename: pool/main/c/ca-certificates/ca-certificates_"+version+"_all.deb\n\n") + } + idx.set("/debian/dists/bookworm/InRelease", []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n")) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("20230311")) + pool := idx.URL + "/debian/pool/main/c/ca-certificates/" + if _, _, err := m.proxy.FetchOrCacheMetadata(t.Context(), "debian", "key:bookworm", idx.URL+"/debian/dists/bookworm/InRelease", "*/*"); err != nil { + t.Fatal(err) + } + result, err := m.proxy.GetOrFetchArtifactFromURL(t.Context(), "deb", "ca-certificates", "20230311", "ca-certificates_20230311_all.deb", pool+"ca-certificates_20230311_all.deb") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("20230311+deb12u1")) + fetcher.reset() + progress := refreshOnce(t, m) + if want := []string{pool + "ca-certificates_20230311+deb12u1_all.deb"}; !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 { + t.Errorf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) + } +} diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go index dc7818a0..01168d2a 100644 --- a/internal/mirror/refresh_index.go +++ b/internal/mirror/refresh_index.go @@ -137,43 +137,64 @@ func cachedUnder(ecosystem, base string, packages []database.UsedPackage) []cach } // newerIndexFiles returns, per package and architecture cached below base, -// the newest file the index lists when it is newer than every cached one. +// the newest file the index lists when it isn't cached and an older version +// is. Suites sharing a pool are told apart that way: a newer version cached +// from another suite doesn't hide an update to this one, and a package only +// cached in a newer version than this suite has isn't mirrored for it. func (m *Mirror) newerIndexFiles(ix osIndex, base string, cached []cachedOSFile, files []indexFile) []PackageVersion { - newestCached := map[string]cachedOSFile{} + cachedByKey := map[string][]cachedOSFile{} for _, c := range cached { - if current, ok := newestCached[c.key()]; !ok || vers.CompareWithScheme(c.version, current.version, ix.scheme) > 0 { - newestCached[c.key()] = c - } + cachedByKey[c.key()] = append(cachedByKey[c.key()], c) + } + type candidate struct { + file indexFile + name, version string } - newest := map[string]PackageVersion{} + newest := map[string]candidate{} var order []string for _, f := range files { name, version, arch := handler.OSPackageIdentity(ix.ecosystem, f.Path) key := name + "\x00" + arch - c, ok := newestCached[key] - if name == "" || !ok || vers.CompareWithScheme(version, c.version, ix.scheme) <= 0 { + if name == "" || len(cachedByKey[key]) == 0 { continue } if m.proxy != nil && !m.proxy.ReleaseAllowed(ix.ecosystem, name, version, time.Time{}) { continue } - if current, seen := newest[key]; seen && vers.CompareWithScheme(version, current.Version, ix.scheme) <= 0 { - continue - } else if !seen { + current, seen := newest[key] + if !seen { order = append(order, key) } - cacheFilename := path.Base(f.Path) - if dir := path.Dir(c.file.Filename); dir != "." { - cacheFilename = dir + "/" + cacheFilename - } - newest[key] = PackageVersion{ - Ecosystem: ix.ecosystem, Name: name, Version: version, - fileURL: base + "/" + f.Path, cacheFilename: cacheFilename, digest: f.Digest, + if !seen || vers.CompareWithScheme(version, current.version, ix.scheme) > 0 { + newest[key] = candidate{file: f, name: name, version: version} } } + items := make([]PackageVersion, 0, len(order)) for _, key := range order { - items = append(items, newest[key]) + n := newest[key] + var older *cachedOSFile + for i, c := range cachedByKey[key] { + cmp := vers.CompareWithScheme(n.version, c.version, ix.scheme) + if cmp == 0 { + older = nil + break + } + if cmp > 0 && older == nil { + older = &cachedByKey[key][i] + } + } + if older == nil { + continue + } + cacheFilename := path.Base(n.file.Path) + if dir := path.Dir(older.file.Filename); dir != "." { + cacheFilename = dir + "/" + cacheFilename + } + items = append(items, PackageVersion{ + Ecosystem: ix.ecosystem, Name: n.name, Version: n.version, + fileURL: base + "/" + n.file.Path, cacheFilename: cacheFilename, digest: n.file.Digest, + }) } return items } From 17c2d525009fa71d00d9ea29bb7dc24217951890 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 12:34:36 +0100 Subject: [PATCH 07/15] Note that refreshing Debian, Alpine and RPM needs cache_metadata --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index bb18db05..e78fbc05 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1842,7 +1842,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. - PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python, ABI and platform tags, and the sdist, or conda packages of the same subdir, build variant (such as `py312`) and file type. -- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. +- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. - Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. From 010ed4761aa17c73f07addb6c1fbc02ade5108cb Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 13:51:07 +0100 Subject: [PATCH 08/15] Skip index entries whose path leaves the repository --- internal/mirror/refresh_harden_test.go | 28 ++++++++++++++++++++++++++ internal/mirror/refresh_index.go | 9 +++++++++ 2 files changed, 37 insertions(+) diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index 458f1e12..599a9932 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -229,3 +229,31 @@ func TestRefreshDebianArchitectureAll(t *testing.T) { t.Errorf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) } } + +// TestRefreshSkipsIndexPathsOutsideTheRepository checks that an index entry +// whose path leaves the repository base is not fetched. +func TestRefreshSkipsIndexPathsOutsideTheRepository(t *testing.T) { + for _, filename := range []string{ + "../other/pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb", + "pool/main/../../../other/curl_7.88.1-10+deb12u8_amd64.deb", + "/other/curl_7.88.1-10+deb12u8_amd64.deb", + } { + t.Run(filename, func(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n") + pool := idx.URL + "/debian/pool/main/c/curl/" + idx.set("/debian/dists/bookworm/InRelease", release) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease", + "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", pool+"curl_7.88.1-10_amd64.deb") + + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", gzipped(t, + "Package: curl\nVersion: 7.88.1-10+deb12u8\nArchitecture: amd64\nFilename: "+filename+"\n\n")) + fetcher.reset() + + if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } + }) + } +} diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go index 01168d2a..93857369 100644 --- a/internal/mirror/refresh_index.go +++ b/internal/mirror/refresh_index.go @@ -153,6 +153,9 @@ func (m *Mirror) newerIndexFiles(ix osIndex, base string, cached []cachedOSFile, newest := map[string]candidate{} var order []string for _, f := range files { + if !confinedIndexPath(f.Path) { + continue + } name, version, arch := handler.OSPackageIdentity(ix.ecosystem, f.Path) key := name + "\x00" + arch if name == "" || len(cachedByKey[key]) == 0 { @@ -199,6 +202,12 @@ func (m *Mirror) newerIndexFiles(ix osIndex, base string, cached []cachedOSFile, return items } +// confinedIndexPath reports whether an index entry's path is relative and +// stays below the repository base. +func confinedIndexPath(p string) bool { + return p != "" && !path.IsAbs(p) && path.Clean(p) == p && p != ".." && !strings.HasPrefix(p, "../") +} + // debIndexFiles reads the Packages indexes an InRelease or Release file // lists for the components and architectures of the cached files. func debIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, cached []cachedOSFile) ([]indexFile, error) { From 55f1b1e2da7b894720ba02c1285f4c4c9491177c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 14:53:36 +0100 Subject: [PATCH 09/15] Reject a negative mirror.refresh.max_bytes --- internal/config/mirror.go | 3 +++ internal/config/mirror_test.go | 1 + 2 files changed, 4 insertions(+) diff --git a/internal/config/mirror.go b/internal/config/mirror.go index 473b7440..625f58d6 100644 --- a/internal/config/mirror.go +++ b/internal/config/mirror.go @@ -108,6 +108,9 @@ func (r *MirrorRefresh) validate() error { if err != nil { return fmt.Errorf("invalid mirror.refresh.max_bytes %q: %w", r.MaxBytes, err) } + if n < 0 { + return fmt.Errorf("invalid mirror.refresh.max_bytes %q: must not be negative", r.MaxBytes) + } r.maxBytes = n } return nil diff --git a/internal/config/mirror_test.go b/internal/config/mirror_test.go index 2a535851..153677be 100644 --- a/internal/config/mirror_test.go +++ b/internal/config/mirror_test.go @@ -103,6 +103,7 @@ func TestMirrorRefreshValidate(t *testing.T) { {name: "negative since", refresh: &MirrorRefresh{Interval: "1h", Since: "-1d"}, wantErr: true}, {name: "negative max_versions", refresh: &MirrorRefresh{Interval: "1h", MaxVersions: -1}, wantErr: true}, {name: "bad max_bytes", refresh: &MirrorRefresh{Interval: "1h", MaxBytes: "lots"}, wantErr: true}, + {name: "negative max_bytes", refresh: &MirrorRefresh{Interval: "1h", MaxBytes: "-1GB"}, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From f8c5fa9a1d15b90bbfde5d17b2b7dcf9710ae7de Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:18:56 +0100 Subject: [PATCH 10/15] Match wheels on OS and architecture, and conda builds without the hash anywhere --- internal/mirror/files.go | 42 +++++++++++---- internal/mirror/refresh_test.go | 93 ++++++++++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 12 deletions(-) diff --git a/internal/mirror/files.go b/internal/mirror/files.go index 641dfd94..0c0a2167 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -81,8 +81,8 @@ var fileEcosystems = map[string]fileEcosystem{ }, } -// pypiVariant names a wheel by its tags, python-abi-platform; every sdist -// is one variant. +// pypiVariant names a wheel by its python and ABI tags and the OS and +// architecture of its platform tags; every sdist is one variant. func pypiVariant(filename string) string { stem, wheel := strings.CutSuffix(filename, ".whl") if !wheel { @@ -92,22 +92,42 @@ func pypiVariant(filename string) string { if len(parts) < 5 { //nolint:mnd // name, version and three tags return stem } - return strings.Join(parts[len(parts)-3:], "-") + tags := parts[len(parts)-3:] + return tags[0] + "-" + tags[1] + "-" + wheelPlatform(tags[2]) } -// condaBuildHash matches the hash and build number a conda build string -// ends in, such as h8753938_0 in py312h8753938_0. -var condaBuildHash = regexp.MustCompile(`h[0-9a-f]{7}_\d+$`) +// wheelPlatformMinimum matches the glibc, musl or macOS minimum version in a +// platform tag, such as the 2_17 in manylinux_2_17_x86_64 or the 2014 in +// manylinux2014_x86_64. +var wheelPlatformMinimum = regexp.MustCompile(`^(manylinux|musllinux|macosx)_?(?:\d+_\d+|1|2010|2014)_`) -// condaVariant names a conda package file by its subdir, the part of its -// build string before the hash, such as py312, and its type. +// wheelPlatform reduces a platform tag, which may list several joined by +// dots, to OS and architecture, so manylinux_2_17_x86_64 and +// manylinux_2_28_x86_64 name the same platform. +func wheelPlatform(tag string) string { + var platforms []string + for _, t := range strings.Split(tag, ".") { + t = wheelPlatformMinimum.ReplaceAllString(t, "${1}_") + if !slices.Contains(platforms, t) { + platforms = append(platforms, t) + } + } + slices.Sort(platforms) + return strings.Join(platforms, ".") +} + +// condaBuildHash matches the hash and build number in a conda build string, +// such as h8753938_0 in py312h8753938_0 or h5f976f7_0 in +// h5f976f7_0_cpython. +var condaBuildHash = regexp.MustCompile(`h[0-9a-f]{7}_\d+`) + +// condaVariant names a conda package file by its subdir, its build string +// without the hash, such as py312 or _cpython, and its type. func condaVariant(subdir, filename string) string { fileType := condaFileType(filename) stem := strings.TrimSuffix(strings.TrimSuffix(filename, ".conda"), ".tar.bz2") build := stem[strings.LastIndex(stem, "-")+1:] - if loc := condaBuildHash.FindStringIndex(build); loc != nil { - build = build[:loc[0]] - } + build = condaBuildHash.ReplaceAllString(build, "") return subdir + "/" + build + "/" + fileType } diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index 4a9f0835..2c23e640 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -291,12 +291,18 @@ func TestFileVariants(t *testing.T) { ecosystem, cacheFilename, want string }{ {"pypi", "demo-1.0.0-py3-none-any.whl", "py3-none-any"}, - {"pypi", "demo-1.0.0-1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64"}, + {"pypi", "demo-1.0.0-1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "cp312-cp312-manylinux_x86_64"}, + {"pypi", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux_2_17_aarch64.whl", "cp312-cp312-manylinux_aarch64.manylinux_x86_64"}, + {"pypi", "demo-1.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", "cp312-cp312-musllinux_x86_64"}, + {"pypi", "demo-1.0.0-cp312-cp312-macosx_10_13_universal2.whl", "cp312-cp312-macosx_universal2"}, + {"pypi", "demo-1.0.0-cp312-cp312-win_amd64.whl", "cp312-cp312-win_amd64"}, {"pypi", "demo-1.0.0.tar.gz", "sdist"}, {"pypi", "demo-1.0.0.zip", "sdist"}, {"conda", "linux-64/numpy-1.26.4-py312h8753938_0.conda", "linux-64/py312/conda"}, {"conda", "noarch/requests-2.32.3-pyhd8ed1ab_0.conda", "noarch/py/conda"}, {"conda", "osx-arm64/zlib-1.3.1-h8359307_2.tar.bz2", "osx-arm64//tar.bz2"}, + {"conda", "linux-64/python-3.12.16-h5f976f7_0_cpython.conda", "linux-64/_cpython/conda"}, + {"conda", "linux-64/python-3.12.16-h5f976f7_0_debug_cpython.conda", "linux-64/_debug_cpython/conda"}, } for _, tt := range tests { if got := fileEcosystems[tt.ecosystem].cachedVariant(tt.cacheFilename); got != tt.want { @@ -321,3 +327,88 @@ func TestRefreshable(t *testing.T) { } } } + +// refreshFetches caches files of an old version of name the way a client +// would, lists newFiles for newVersion and returns what a refresh fetched. +func refreshFetches(t *testing.T, ecosystem, name, oldVersion string, cached []string, newVersion string, newFiles []registries.Artifact) []string { + t.Helper() + m, fetcher := setupRefreshMirror(t, "file") + ctx := context.Background() + old := registries.Version{Number: oldVersion, PublishedAt: time.Now().Add(-60 * 24 * time.Hour)} + for _, f := range cached { + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, ecosystem, name, oldVersion, f, "https://files.example/"+f, "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + } + for i := range newFiles { + newFiles[i].URL = "https://files.example/" + newFiles[i].Filename + } + m.Versions = stubReleases{ecosystem + "/" + name: { + old, + {Number: newVersion, PublishedAt: time.Now().Add(-30 * 24 * time.Hour), Artifacts: newFiles}, + }} + fetcher.reset() + + progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Failed != 0 { + t.Errorf("progress = %+v, want no failures", progress) + } + got := fetcher.fetched() + slices.Sort(got) + return got +} + +func wheels(names ...string) []registries.Artifact { + files := make([]registries.Artifact, 0, len(names)) + for _, n := range names { + files = append(files, registries.Artifact{Filename: n}) + } + return files +} + +// A new wheel built for a newer glibc or macOS still replaces the cached +// one: platform tags match on OS and architecture. +func TestRefreshMatchesWheelsAcrossPlatformMinimums(t *testing.T) { + got := refreshFetches(t, "pypi", "numpy", "2.2.6", + []string{"numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl"}, + "2.3.0", wheels( + "numpy-2.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", + "numpy-2.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", + "numpy-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", + "numpy-2.3.0-cp313-cp313-manylinux_2_28_x86_64.whl", + "numpy-2.3.0-cp312-cp312-macosx_11_0_x86_64.whl", + "numpy-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", + "numpy-2.3.0-cp312-cp312-win_amd64.whl", + )) + want := []string{ + "https://files.example/numpy-2.3.0-cp312-cp312-macosx_11_0_x86_64.whl", + "https://files.example/numpy-2.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", + } + if !slices.Equal(got, want) { + t.Errorf("fetched %v, want %v", got, want) + } +} + +// conda-forge's python builds carry the hash before a suffix; the suffix +// stays part of the kind. +func TestRefreshMatchesCondaBuildsWithHashInside(t *testing.T) { + conda := func(subdir, filename, build string) registries.Artifact { + return registries.Artifact{Filename: filename, Metadata: map[string]any{"subdir": subdir, "build": build}} + } + got := refreshFetches(t, "conda", "conda-forge/python", "3.12.15", + []string{"linux-64/python-3.12.15-h5f976f7_0_cpython.conda"}, + "3.12.16", []registries.Artifact{ + conda("linux-64", "python-3.12.16-hab00c5b_1_cpython.conda", "hab00c5b_1_cpython"), + conda("linux-64", "python-3.12.16-hd12c33a_1_debug_cpython.conda", "hd12c33a_1_debug_cpython"), + conda("osx-arm64", "python-3.12.16-h99e199e_1_cpython.conda", "h99e199e_1_cpython"), + }) + want := []string{"https://files.example/python-3.12.16-hab00c5b_1_cpython.conda"} + if !slices.Equal(got, want) { + t.Errorf("fetched %v, want %v", got, want) + } +} From 34524a40564e813cf273ed25d0457838561b900d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:20:06 +0100 Subject: [PATCH 11/15] Limit a refreshed index to 512 MiB once decompressed --- internal/mirror/refresh_harden_test.go | 26 +++++++++++++++++++ internal/mirror/refresh_index.go | 28 +++++++++++++++++++-- internal/mirror/refresh_index_limit_test.go | 20 +++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 internal/mirror/refresh_index_limit_test.go diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index 599a9932..574a8736 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -4,6 +4,7 @@ import ( "context" "reflect" "slices" + "strings" "testing" "time" @@ -257,3 +258,28 @@ func TestRefreshSkipsIndexPathsOutsideTheRepository(t *testing.T) { }) } } + +// TestRefreshRefusesIndexesOverTheSizeLimit checks that an index that +// decompresses to more than maxIndexBytes fails instead of being parsed in +// full. +func TestRefreshRefusesIndexesOverTheSizeLimit(t *testing.T) { + limit := maxIndexBytes + maxIndexBytes = 4 << 10 + t.Cleanup(func() { maxIndexBytes = limit }) + + m, fetcher, idx := setupIndexRefresh(t) + idx.set("/debian/dists/bookworm/InRelease", []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n")) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease", + "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", idx.URL+"/debian/pool/main/c/curl/curl_7.88.1-10_amd64.deb") + + padding := strings.Repeat("Package: filler\nVersion: 1\nArchitecture: amd64\nFilename: pool/main/f/filler/filler_1_amd64.deb\n\n", 100) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", gzipped(t, + "Package: curl\nVersion: 7.88.1-10+deb12u8\nArchitecture: amd64\nFilename: pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb\n\n"+padding)) + fetcher.reset() + + progress := refreshOnce(t, m) + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "larger than") || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want the index failing for its size", progress, fetcher.fetched()) + } +} diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go index 93857369..50bb20d3 100644 --- a/internal/mirror/refresh_index.go +++ b/internal/mirror/refresh_index.go @@ -254,7 +254,7 @@ func rpmIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, } // readIndex streams an index from upstream through parse, decompressed as -// its name says. It is read directly rather than through the metadata +// its name says and limited to maxIndexBytes. It is read directly rather than through the metadata // cache, since it can be far larger than a metadata document may be. func (m *Mirror) readIndex(ctx context.Context, indexURL string, parse func(io.Reader) ([]indexFile, error)) ([]indexFile, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil) @@ -274,7 +274,31 @@ func (m *Mirror) readIndex(ctx context.Context, indexURL string, parse func(io.R return nil, err } defer func() { _ = r.Close() }() - return parse(r) + return parse(&indexLimitReader{r: r, url: indexURL, remaining: maxIndexBytes}) +} + +// maxIndexBytes bounds an index once decompressed. Debian's largest +// Packages files are under 100 MB and Fedora's primary.xml under 200 MB. +var maxIndexBytes int64 = 512 << 20 + +// indexLimitReader fails once more than maxIndexBytes have been read, so an +// oversized index fails rather than looking complete. +type indexLimitReader struct { + r io.Reader + url string + remaining int64 +} + +func (l *indexLimitReader) Read(p []byte) (int, error) { + if int64(len(p)) > l.remaining+1 { + p = p[:l.remaining+1] + } + n, err := l.r.Read(p) + if int64(n) > l.remaining { + return int(l.remaining), fmt.Errorf("%s: index larger than %d bytes", l.url, maxIndexBytes) + } + l.remaining -= int64(n) + return n, err } // urlDir returns a URL without its last path segment. path.Dir would fold diff --git a/internal/mirror/refresh_index_limit_test.go b/internal/mirror/refresh_index_limit_test.go new file mode 100644 index 00000000..7512fc5d --- /dev/null +++ b/internal/mirror/refresh_index_limit_test.go @@ -0,0 +1,20 @@ +package mirror + +import ( + "io" + "strings" + "testing" +) + +func TestIndexLimitReaderAllowsExactlyTheLimit(t *testing.T) { + limit := maxIndexBytes + maxIndexBytes = 8 + t.Cleanup(func() { maxIndexBytes = limit }) + + if _, err := io.ReadAll(&indexLimitReader{r: strings.NewReader("12345678"), remaining: maxIndexBytes}); err != nil { + t.Errorf("index of exactly the limit: %v", err) + } + if _, err := io.ReadAll(&indexLimitReader{r: strings.NewReader("123456789"), remaining: maxIndexBytes}); err == nil { + t.Error("index one byte over the limit read without an error") + } +} From 6d971716d9fcf3afa297f98541fa5389bf060686 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:20:16 +0100 Subject: [PATCH 12/15] Describe the looser file kind matching --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index dacc93cc..b5131589 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1924,7 +1924,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. -- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python, ABI and platform tags, and the sdist, or conda packages of the same subdir, build variant (such as `py312`) and file type. +- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need, and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. - Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. From 14d04955b4c62c101a1b9a3d16ae5cf7dc09b8c3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:43:33 +0100 Subject: [PATCH 13/15] Match wheels on any of their alternative tags --- internal/mirror/files.go | 66 +++++++++++++++++---------------- internal/mirror/refresh.go | 4 +- internal/mirror/refresh_test.go | 24 +++++++++++- internal/mirror/source.go | 2 +- 4 files changed, 61 insertions(+), 35 deletions(-) diff --git a/internal/mirror/files.go b/internal/mirror/files.go index 0c0a2167..ff0d2981 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -24,12 +24,12 @@ type fileEcosystem struct { // check refuses a package the ecosystem can't mirror, before its files // are listed. check func(pv PackageVersion) error - // variant names the kind of a file, such as a wheel's tags or a conda - // package's platform, so a refresh mirrors the same kinds of files of a - // new version as were cached of an old one. cachedVariant names it for - // the filename cacheKey returns. - variant func(file registries.Artifact) string - cachedVariant func(cacheFilename string) string + // variants name the kinds of a file, such as each tag combination of a + // wheel or a conda package's platform, so a refresh mirrors the same + // kinds of files of a new version as were cached of an old one. + // cachedVariants names them for the filename cacheKey returns. + variants func(file registries.Artifact) []string + cachedVariants func(cacheFilename string) []string } var fileEcosystems = map[string]fileEcosystem{ @@ -45,8 +45,8 @@ var fileEcosystems = map[string]fileEcosystem{ } return name, version, file.Filename, true }, - variant: func(file registries.Artifact) string { return pypiVariant(file.Filename) }, - cachedVariant: pypiVariant, + variants: func(file registries.Artifact) []string { return pypiVariants(file.Filename) }, + cachedVariants: pypiVariants, }, "conda": { qualifiers: []string{"subdir", "build", "type"}, @@ -70,30 +70,43 @@ var fileEcosystems = map[string]fileEcosystem{ } return nil }, - variant: func(file registries.Artifact) string { + variants: func(file registries.Artifact) []string { subdir, _ := file.Metadata["subdir"].(string) - return condaVariant(subdir, file.Filename) + return []string{condaVariant(subdir, file.Filename)} }, - cachedVariant: func(cacheFilename string) string { + cachedVariants: func(cacheFilename string) []string { subdir, filename := path.Split(cacheFilename) - return condaVariant(strings.TrimSuffix(subdir, "/"), filename) + return []string{condaVariant(strings.TrimSuffix(subdir, "/"), filename)} }, }, } -// pypiVariant names a wheel by its python and ABI tags and the OS and -// architecture of its platform tags; every sdist is one variant. -func pypiVariant(filename string) string { +// pypiVariants names a wheel by each combination of its python, ABI and +// platform tags, which are alternatives where a tag lists several joined by +// dots, with the platform reduced to OS and architecture. Every sdist is +// one variant. +func pypiVariants(filename string) []string { stem, wheel := strings.CutSuffix(filename, ".whl") if !wheel { - return "sdist" + return []string{"sdist"} } parts := strings.Split(stem, "-") if len(parts) < 5 { //nolint:mnd // name, version and three tags - return stem + return []string{stem} } tags := parts[len(parts)-3:] - return tags[0] + "-" + tags[1] + "-" + wheelPlatform(tags[2]) + var variants []string + for _, python := range strings.Split(tags[0], ".") { + for _, abi := range strings.Split(tags[1], ".") { + for _, platform := range strings.Split(tags[2], ".") { + v := python + "-" + abi + "-" + wheelPlatform(platform) + if !slices.Contains(variants, v) { + variants = append(variants, v) + } + } + } + } + return variants } // wheelPlatformMinimum matches the glibc, musl or macOS minimum version in a @@ -101,19 +114,10 @@ func pypiVariant(filename string) string { // manylinux2014_x86_64. var wheelPlatformMinimum = regexp.MustCompile(`^(manylinux|musllinux|macosx)_?(?:\d+_\d+|1|2010|2014)_`) -// wheelPlatform reduces a platform tag, which may list several joined by -// dots, to OS and architecture, so manylinux_2_17_x86_64 and -// manylinux_2_28_x86_64 name the same platform. +// wheelPlatform reduces a platform tag to OS and architecture, so +// manylinux_2_17_x86_64 and manylinux_2_28_x86_64 name the same platform. func wheelPlatform(tag string) string { - var platforms []string - for _, t := range strings.Split(tag, ".") { - t = wheelPlatformMinimum.ReplaceAllString(t, "${1}_") - if !slices.Contains(platforms, t) { - platforms = append(platforms, t) - } - } - slices.Sort(platforms) - return strings.Join(platforms, ".") + return wheelPlatformMinimum.ReplaceAllString(tag, "${1}_") } // condaBuildHash matches the hash and build number in a conda build string, @@ -160,7 +164,7 @@ func (e fileEcosystem) selection(pv PackageVersion) string { // selects reports whether file matches every selecting qualifier of pv. func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool { - if pv.variants != nil && !pv.variants[e.variant(file)] { + if pv.variants != nil && !slices.ContainsFunc(e.variants(file), func(v string) bool { return pv.variants[v] }) { return false } attrs := e.attrs(file) diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index fc9317a2..6fc036b6 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -155,7 +155,9 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] if eco, ok := fileEcosystems[ecosystem]; ok { pv.variants = map[string]bool{} for _, f := range pkg.Files { - pv.variants[eco.cachedVariant(f.Filename)] = true + for _, v := range eco.cachedVariants(f.Filename) { + pv.variants[v] = true + } } } items = append(items, pv) diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index 2c23e640..af0d1be3 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -292,7 +292,8 @@ func TestFileVariants(t *testing.T) { }{ {"pypi", "demo-1.0.0-py3-none-any.whl", "py3-none-any"}, {"pypi", "demo-1.0.0-1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "cp312-cp312-manylinux_x86_64"}, - {"pypi", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux_2_17_aarch64.whl", "cp312-cp312-manylinux_aarch64.manylinux_x86_64"}, + {"pypi", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux_2_17_aarch64.whl", "cp312-cp312-manylinux_x86_64 cp312-cp312-manylinux_aarch64"}, + {"pypi", "demo-1.0.0-py2.py3-none-any.whl", "py2-none-any py3-none-any"}, {"pypi", "demo-1.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", "cp312-cp312-musllinux_x86_64"}, {"pypi", "demo-1.0.0-cp312-cp312-macosx_10_13_universal2.whl", "cp312-cp312-macosx_universal2"}, {"pypi", "demo-1.0.0-cp312-cp312-win_amd64.whl", "cp312-cp312-win_amd64"}, @@ -305,7 +306,7 @@ func TestFileVariants(t *testing.T) { {"conda", "linux-64/python-3.12.16-h5f976f7_0_debug_cpython.conda", "linux-64/_debug_cpython/conda"}, } for _, tt := range tests { - if got := fileEcosystems[tt.ecosystem].cachedVariant(tt.cacheFilename); got != tt.want { + if got := strings.Join(fileEcosystems[tt.ecosystem].cachedVariants(tt.cacheFilename), " "); got != tt.want { t.Errorf("%s variant of %q = %q, want %q", tt.ecosystem, tt.cacheFilename, got, tt.want) } } @@ -412,3 +413,22 @@ func TestRefreshMatchesCondaBuildsWithHashInside(t *testing.T) { t.Errorf("fetched %v, want %v", got, want) } } + +// The dot-separated tags of a wheel are alternatives: a wheel matches a +// cached one when any of its tags does. +func TestRefreshMatchesWheelsOnAnyAlternativeTag(t *testing.T) { + got := refreshFetches(t, "pypi", "demo", "1.0.0", + []string{"demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.whl", "demo-1.0.0-py2.py3-none-macosx_11_0_arm64.macosx_11_0_x86_64.whl"}, + "1.1.0", wheels( + "demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.linux_x86_64.whl", + "demo-1.1.0-py3-none-macosx_11_0_x86_64.whl", + "demo-1.1.0-cp312-cp312-win_amd64.whl", + )) + want := []string{ + "https://files.example/demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.linux_x86_64.whl", + "https://files.example/demo-1.1.0-py3-none-macosx_11_0_x86_64.whl", + } + if !slices.Equal(got, want) { + t.Errorf("fetched %v, want %v", got, want) + } +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 5e32ae27..6e01c6df 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -27,7 +27,7 @@ type PackageVersion struct { // Platforms limit which platforms of an image index are mirrored. Platforms []string // variants, when set, limit the files of a version that are mirrored to - // those of these variants (see fileEcosystem.variant). + // those of these variants (see fileEcosystem.variants). variants map[string]bool // fileURL, when set, is the one file to mirror, cached as cacheFilename // and checked against digest when there is one. A refresh finds it in From 0300ad252278b1b22d0d9be9620a47c87033add3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:44:12 +0100 Subject: [PATCH 14/15] Limit APKINDEX to maxIndexBytes once decompressed --- internal/mirror/refresh_harden_test.go | 24 +++++++++++++++++++++ internal/mirror/refresh_index.go | 4 ++-- internal/mirror/refresh_index_parse.go | 9 ++++---- internal/mirror/refresh_index_parse_test.go | 2 +- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index 574a8736..9c311dd7 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -283,3 +283,27 @@ func TestRefreshRefusesIndexesOverTheSizeLimit(t *testing.T) { t.Errorf("progress = %+v, fetched %v; want the index failing for its size", progress, fetcher.fetched()) } } + +// TestRefreshRefusesAPKIndexesOverTheSizeLimit checks that the limit also +// covers APKINDEX.tar.gz, which comes from the metadata cache rather than +// readIndex. +func TestRefreshRefusesAPKIndexesOverTheSizeLimit(t *testing.T) { + limit := maxIndexBytes + maxIndexBytes = 4 << 10 + t.Cleanup(func() { maxIndexBytes = limit }) + + m, fetcher, idx := setupIndexRefresh(t) + dir := "/alpine/v3.20/main/aarch64" + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.0-r0\nA:aarch64\n\n")) + clientFetchesCurl(t, m, "alpine", idx.URL+dir+"/APKINDEX.tar.gz", + "alpine", "8.9.0-r0", "edge/v3.20/main/aarch64/curl-8.9.0-r0.apk", idx.URL+dir+"/curl-8.9.0-r0.apk") + + padding := strings.Repeat("P:filler\nV:1-r0\nA:aarch64\n\n", 400) + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.1-r0\nA:aarch64\n\n"+padding)) + fetcher.reset() + + progress := refreshOnce(t, m) + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "larger than") || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want the index failing for its size", progress, fetcher.fetched()) + } +} diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go index 50bb20d3..3a7ad7c4 100644 --- a/internal/mirror/refresh_index.go +++ b/internal/mirror/refresh_index.go @@ -43,8 +43,8 @@ var osIndexes = map[string]osIndex{ base: func(u string) (string, bool) { return strings.CutSuffix(u, "/APKINDEX.tar.gz") }, - files: func(_ context.Context, _ *Mirror, _ string, body []byte, _ []cachedOSFile) ([]indexFile, error) { - return parseAPKIndex(bytes.NewReader(body)) + files: func(_ context.Context, _ *Mirror, indexURL string, body []byte, _ []cachedOSFile) ([]indexFile, error) { + return parseAPKIndex(bytes.NewReader(body), indexURL) }, }, "deb": { diff --git a/internal/mirror/refresh_index_parse.go b/internal/mirror/refresh_index_parse.go index 4ae98576..9b0cd272 100644 --- a/internal/mirror/refresh_index_parse.go +++ b/internal/mirror/refresh_index_parse.go @@ -25,9 +25,10 @@ type indexFile struct { // maxIndexLine bounds one line of a Debian or Alpine index. const maxIndexLine = 1 << 20 -// parseAPKIndex reads the files an APKINDEX.tar.gz lists. Its checksums -// cover only the control section of a package, so no digest is returned. -func parseAPKIndex(r io.Reader) ([]indexFile, error) { +// parseAPKIndex reads the files an APKINDEX.tar.gz lists, limited to +// maxIndexBytes once decompressed. Its checksums cover only the control +// section of a package, so no digest is returned. +func parseAPKIndex(r io.Reader, indexURL string) ([]indexFile, error) { gz, err := gzip.NewReader(r) if err != nil { return nil, err @@ -35,7 +36,7 @@ func parseAPKIndex(r io.Reader) ([]indexFile, error) { defer func() { _ = gz.Close() }() // An APKINDEX.tar.gz may be several gzip streams: the signature, then the // index. Reading on across them finds the index whichever comes first. - tr := tar.NewReader(gz) + tr := tar.NewReader(&indexLimitReader{r: gz, url: indexURL, remaining: maxIndexBytes}) for { hdr, err := tr.Next() if errors.Is(err, io.EOF) { diff --git a/internal/mirror/refresh_index_parse_test.go b/internal/mirror/refresh_index_parse_test.go index 3b68805f..934256e2 100644 --- a/internal/mirror/refresh_index_parse_test.go +++ b/internal/mirror/refresh_index_parse_test.go @@ -36,7 +36,7 @@ func apkIndexArchive(t *testing.T, index string) []byte { func TestParseAPKIndex(t *testing.T) { index := "C:Q1abc=\nP:curl\nV:8.9.1-r0\nA:aarch64\nS:100\n\nC:Q1def=\nP:busybox\nV:1.36.1-r29\nA:aarch64\n\n" - got, err := parseAPKIndex(bytes.NewReader(apkIndexArchive(t, index))) + got, err := parseAPKIndex(bytes.NewReader(apkIndexArchive(t, index)), "APKINDEX.tar.gz") if err != nil { t.Fatal(err) } From 794ea98a2810cd26eac95e21033c52f878c407aa Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 7 Nov 2026 17:04:29 +0100 Subject: [PATCH 15/15] Match abi3 wheels whatever minimum Python they name The python tag of an abi3 wheel is a minimum, so cp38-abi3 and cp311-abi3 wheels are now the same kind as a cached cp39-abi3 one. --- docs/configuration.md | 2 +- internal/mirror/files.go | 12 +++++++++++- internal/mirror/refresh_test.go | 25 +++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index e9d91952..5459cb69 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1973,7 +1973,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. -- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need, and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. +- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. - Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. diff --git a/internal/mirror/files.go b/internal/mirror/files.go index ff0d2981..ab6bafdb 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -99,7 +99,7 @@ func pypiVariants(filename string) []string { for _, python := range strings.Split(tags[0], ".") { for _, abi := range strings.Split(tags[1], ".") { for _, platform := range strings.Split(tags[2], ".") { - v := python + "-" + abi + "-" + wheelPlatform(platform) + v := wheelPython(python, abi) + "-" + abi + "-" + wheelPlatform(platform) if !slices.Contains(variants, v) { variants = append(variants, v) } @@ -109,6 +109,16 @@ func pypiVariants(filename string) []string { return variants } +// wheelPython reduces the python tag of an abi3 wheel, a minimum such as +// cp39, to its interpreter, so cp38-abi3 and cp311-abi3 wheels are the same +// kind. +func wheelPython(python, abi string) string { + if abi != "abi3" { + return python + } + return strings.TrimRight(python, "0123456789") +} + // wheelPlatformMinimum matches the glibc, musl or macOS minimum version in a // platform tag, such as the 2_17 in manylinux_2_17_x86_64 or the 2014 in // manylinux2014_x86_64. diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index af0d1be3..97624d6d 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -297,6 +297,7 @@ func TestFileVariants(t *testing.T) { {"pypi", "demo-1.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", "cp312-cp312-musllinux_x86_64"}, {"pypi", "demo-1.0.0-cp312-cp312-macosx_10_13_universal2.whl", "cp312-cp312-macosx_universal2"}, {"pypi", "demo-1.0.0-cp312-cp312-win_amd64.whl", "cp312-cp312-win_amd64"}, + {"pypi", "demo-1.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", "cp-abi3-manylinux_x86_64"}, {"pypi", "demo-1.0.0.tar.gz", "sdist"}, {"pypi", "demo-1.0.0.zip", "sdist"}, {"conda", "linux-64/numpy-1.26.4-py312h8753938_0.conda", "linux-64/py312/conda"}, @@ -432,3 +433,27 @@ func TestRefreshMatchesWheelsOnAnyAlternativeTag(t *testing.T) { t.Errorf("fetched %v, want %v", got, want) } } + +// In an abi3 wheel the python tag is a minimum: cryptography 44.0.0 ships +// cp37-abi3 and cp39-abi3 wheels, 46.0.0 cp38-abi3 and cp311-abi3 ones. A +// refresh stays on the cached major line, so the newer tags are on 44.1.0 +// here. +func TestRefreshMatchesAbi3WheelsAcrossPythonMinimums(t *testing.T) { + got := refreshFetches(t, "pypi", "cryptography", "44.0.0", + []string{"cryptography-44.0.0-cp39-abi3-manylinux_2_28_x86_64.whl"}, + "44.1.0", wheels( + "cryptography-44.1.0-cp38-abi3-manylinux_2_28_x86_64.whl", + "cryptography-44.1.0-cp311-abi3-manylinux_2_28_x86_64.whl", + "cryptography-44.1.0-cp311-abi3-manylinux_2_28_aarch64.whl", + "cryptography-44.1.0-cp311-abi3-macosx_10_9_universal2.whl", + "cryptography-44.1.0-cp311-abi3-win_amd64.whl", + "cryptography-44.1.0-cp314-cp314t-manylinux_2_28_x86_64.whl", + )) + want := []string{ + "https://files.example/cryptography-44.1.0-cp311-abi3-manylinux_2_28_x86_64.whl", + "https://files.example/cryptography-44.1.0-cp38-abi3-manylinux_2_28_x86_64.whl", + } + if !slices.Equal(got, want) { + t.Errorf("fetched %v, want %v", got, want) + } +}