diff --git a/config.example.yaml b/config.example.yaml
index c8ab11b1..002d5069 100644
--- a/config.example.yaml
+++ b/config.example.yaml
@@ -120,8 +120,8 @@ storage:
# Store fetched artifacts. Set to false to stream every download from
# upstream without storing it; metadata filtering, cooldown and the
# denylist still apply. Useful when another cache sits in front of the
- # proxy. false is incompatible with direct_serve, scanning, mirror_api and
- # mirror.schedules.
+ # proxy. false is incompatible with direct_serve, scanning, mirror_api,
+ # mirror.schedules and mirror.refresh.
cache_artifacts: true
# Redirect cached artifact downloads to presigned storage URLs (HTTP 302)
@@ -467,6 +467,16 @@ gradle:
# - images: ["pkg:docker/library/alpine@3.20", "pkg:docker/library/nginx@stable"]
# interval: "6h"
# platforms: ["linux/amd64"]
+#
+# Mirror new releases of the most used packages in the background, so the
+# first install after a release doesn't wait on the upstream. Cooldown and
+# the denylist apply.
+# refresh:
+# interval: "24h"
+# top: 50 # packages per ecosystem, by recent downloads
+# since: "14d"
+# max_versions: 200 # per run; default no cap
+# max_bytes: "2GB" # per run; default no cap
# Health endpoint configuration.
health:
diff --git a/docs/configuration.md b/docs/configuration.md
index 584896bc..5459cb69 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -91,7 +91,7 @@ storage:
Every download is a fresh upstream fetch, and concurrent requests for the same artifact are not combined. Artifacts with a digest known up front (OCI blobs, Swift archives, Helm charts) are verified while streaming: the response is sent chunked, and on a mismatch the connection is aborted before the response completes so the client never receives a tampered artifact as a good one. The same happens when the upstream connection fails mid-download.
-`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve`, `mirror_api` or `mirror.schedules`, which all need stored artifacts, and the `mirror` command refuses to run with it.
+`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve`, `mirror_api`, `mirror.schedules` or `mirror.refresh`, which all need stored artifacts, and the `mirror` command refuses to run with it.
### Retention
@@ -1949,6 +1949,37 @@ Each run checks the tag with a HEAD request. If the tag still points to the cach
- Mirrored layers are scanned when `scanning.enabled` is set. A blocked layer fails the run and keeps the previous image.
- Schedules are read at startup; changes need a restart. They do not need `mirror_api`.
+### Refreshing the most used packages
+
+For packages, the cache learns about a new release only when a client asks for it, and that first install waits on the upstream or fails while it is down. `mirror.refresh` mirrors new releases of the packages clients download most, so they are cached before anyone asks:
+
+```yaml
+mirror:
+ refresh:
+ interval: 24h
+ top: 50
+ since: 14d
+ max_versions: 200
+ max_bytes: 2GB
+```
+
+| Setting | Env | Description |
+|---------|-----|-------------|
+| `mirror.refresh.interval` | - | How often the refresh runs, at least `1m` |
+| `mirror.refresh.top` | - | Packages per ecosystem to refresh, the ones with the most downloads among those downloaded within `since`. Default: `50` |
+| `mirror.refresh.since` | - | How recently a package must have been downloaded, with a `d` suffix for days. Default: `14d` |
+| `mirror.refresh.max_versions` | - | Stop a run after mirroring this many new versions. Default: no cap |
+| `mirror.refresh.max_bytes` | - | Stop a run once it has downloaded this much, such as `2GB`. It is checked before each version, so the last one can go over it. Default: no cap |
+
+For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run.
+
+- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type.
+- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`.
+- Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too.
+- Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`.
+- Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time.
+- What a run mirrors is evicted by `storage.max_size` and retention like any other file. With `max_versions` and `max_bytes` a run stays bounded when many packages release at once.
+
## Docker
### SQLite with Local Storage
diff --git a/go.mod b/go.mod
index 2f4a0ce3..0c56d13b 100644
--- a/go.mod
+++ b/go.mod
@@ -29,6 +29,7 @@ require (
github.com/prometheus/client_golang v1.25.0
github.com/prometheus/client_model v0.6.3
github.com/swaggo/swag v1.16.6
+ github.com/ulikunitz/xz v0.5.17
github.com/vmihailenco/msgpack/v5 v5.4.1
gocloud.dev v0.47.0
golang.org/x/mod v0.42.0
@@ -108,7 +109,6 @@ require (
github.com/rogpeppe/go-internal v1.16.0 // indirect
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect
github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect
- github.com/ulikunitz/xz v0.5.17 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel v1.46.0 // indirect
diff --git a/internal/config/config.go b/internal/config/config.go
index 3b5c6cb0..b2fea064 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -1661,6 +1661,8 @@ func (c *Config) validateCacheArtifacts() error {
return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served")
case len(c.Mirror.Schedules) > 0:
return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror.schedules: mirrored artifacts would never be served")
+ case c.Mirror.Refresh != nil:
+ return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror.refresh: mirrored artifacts would never be served")
case c.Hosted.Enabled:
return fmt.Errorf("storage.cache_artifacts: false cannot be combined with hosted.enabled: pushed packages would never be served")
}
diff --git a/internal/config/mirror.go b/internal/config/mirror.go
index 7f6c668a..625f58d6 100644
--- a/internal/config/mirror.go
+++ b/internal/config/mirror.go
@@ -15,6 +15,105 @@ type MirrorConfig struct {
// Schedules re-mirror container image tags at a fixed interval, so a
// moved tag is picked up even when nobody pulls it.
Schedules []MirrorSchedule `json:"schedules" yaml:"schedules"`
+
+ // Refresh, when set, keeps the most used packages up to date in the
+ // background.
+ Refresh *MirrorRefresh `json:"refresh" yaml:"refresh"`
+}
+
+const (
+ defaultMirrorRefreshTop = 50
+ defaultMirrorRefreshSince = 14 * 24 * time.Hour
+)
+
+// MirrorRefresh mirrors new releases of the packages clients use most, so
+// they are cached before the first client asks for them.
+type MirrorRefresh struct {
+ // Interval is how often the refresh runs, at least one minute (e.g.
+ // "24h").
+ Interval string `json:"interval" yaml:"interval"`
+
+ // Top is how many packages per ecosystem are refreshed, the ones with
+ // the most downloads among those used within Since. Default: 50.
+ Top int `json:"top" yaml:"top"`
+
+ // Since is how recently a package must have been downloaded to be
+ // refreshed (e.g. "14d"). Default: 14d.
+ Since string `json:"since" yaml:"since"`
+
+ // MaxVersions caps how many new versions one run mirrors. 0 means no
+ // cap.
+ MaxVersions int `json:"max_versions" yaml:"max_versions"`
+
+ // MaxBytes caps how much one run downloads (e.g. "2GB"). Empty means no
+ // cap.
+ MaxBytes string `json:"max_bytes" yaml:"max_bytes"`
+
+ interval time.Duration
+ since time.Duration
+ maxBytes int64
+}
+
+// IntervalDuration returns the parsed Interval. Validate must run first.
+func (r *MirrorRefresh) IntervalDuration() time.Duration { return r.interval }
+
+// TopPackages returns Top, or its default.
+func (r *MirrorRefresh) TopPackages() int {
+ if r.Top == 0 {
+ return defaultMirrorRefreshTop
+ }
+ return r.Top
+}
+
+// SinceDuration returns the parsed Since, or its default. Validate must run
+// first.
+func (r *MirrorRefresh) SinceDuration() time.Duration { return r.since }
+
+// MaxVersionsPerRun returns MaxVersions; 0 means no cap.
+func (r *MirrorRefresh) MaxVersionsPerRun() int { return r.MaxVersions }
+
+// MaxBytesPerRun returns the parsed MaxBytes; 0 means no cap. Validate must
+// run first.
+func (r *MirrorRefresh) MaxBytesPerRun() int64 { return r.maxBytes }
+
+func (r *MirrorRefresh) validate() error {
+ d, err := time.ParseDuration(r.Interval)
+ if err != nil {
+ return fmt.Errorf("invalid mirror.refresh.interval %q: %w", r.Interval, err)
+ }
+ if d < minMirrorScheduleInterval {
+ return fmt.Errorf("invalid mirror.refresh.interval %q: must be at least %s", r.Interval, minMirrorScheduleInterval)
+ }
+ r.interval = d
+ if r.Top < 0 {
+ return fmt.Errorf("invalid mirror.refresh.top %d: must not be negative", r.Top)
+ }
+ r.since = defaultMirrorRefreshSince
+ if r.Since != "" {
+ since, err := parseDayDuration(r.Since)
+ if err != nil {
+ return fmt.Errorf("invalid mirror.refresh.since %q: %w", r.Since, err)
+ }
+ if since <= 0 {
+ return fmt.Errorf("invalid mirror.refresh.since %q: must be positive", r.Since)
+ }
+ r.since = since
+ }
+ if r.MaxVersions < 0 {
+ return fmt.Errorf("invalid mirror.refresh.max_versions %d: must not be negative", r.MaxVersions)
+ }
+ r.maxBytes = 0
+ if r.MaxBytes != "" {
+ n, err := ParseSize(r.MaxBytes)
+ if err != nil {
+ return fmt.Errorf("invalid mirror.refresh.max_bytes %q: %w", r.MaxBytes, err)
+ }
+ if n < 0 {
+ return fmt.Errorf("invalid mirror.refresh.max_bytes %q: must not be negative", r.MaxBytes)
+ }
+ r.maxBytes = n
+ }
+ return nil
}
// MirrorSchedule is a set of image PURLs mirrored every Interval.
@@ -45,6 +144,9 @@ func (m *MirrorConfig) Validate() error {
return err
}
}
+ if m.Refresh != nil {
+ return m.Refresh.validate()
+ }
return nil
}
diff --git a/internal/config/mirror_test.go b/internal/config/mirror_test.go
index ae1220dc..153677be 100644
--- a/internal/config/mirror_test.go
+++ b/internal/config/mirror_test.go
@@ -87,3 +87,85 @@ mirror:
t.Error("Validate() accepted a 30s interval in the second schedule")
}
}
+
+func TestMirrorRefreshValidate(t *testing.T) {
+ tests := []struct {
+ name string
+ refresh *MirrorRefresh
+ wantErr bool
+ }{
+ {name: "defaults", refresh: &MirrorRefresh{Interval: "24h"}},
+ {name: "all fields", refresh: &MirrorRefresh{Interval: "12h", Top: 20, Since: "14d", MaxVersions: 100, MaxBytes: "2GB"}},
+ {name: "missing interval", refresh: &MirrorRefresh{}, wantErr: true},
+ {name: "interval under a minute", refresh: &MirrorRefresh{Interval: "10s"}, wantErr: true},
+ {name: "negative top", refresh: &MirrorRefresh{Interval: "1h", Top: -1}, wantErr: true},
+ {name: "bad since", refresh: &MirrorRefresh{Interval: "1h", Since: "recently"}, wantErr: true},
+ {name: "negative since", refresh: &MirrorRefresh{Interval: "1h", Since: "-1d"}, wantErr: true},
+ {name: "negative max_versions", refresh: &MirrorRefresh{Interval: "1h", MaxVersions: -1}, wantErr: true},
+ {name: "bad max_bytes", refresh: &MirrorRefresh{Interval: "1h", MaxBytes: "lots"}, wantErr: true},
+ {name: "negative max_bytes", refresh: &MirrorRefresh{Interval: "1h", MaxBytes: "-1GB"}, wantErr: true},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ m := MirrorConfig{Refresh: tt.refresh}
+ err := m.Validate()
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ })
+ }
+}
+
+func TestMirrorRefreshDefaults(t *testing.T) {
+ m := MirrorConfig{Refresh: &MirrorRefresh{Interval: "24h"}}
+ if err := m.Validate(); err != nil {
+ t.Fatalf("Validate: %v", err)
+ }
+ r := m.Refresh
+ if r.IntervalDuration() != 24*time.Hour || r.TopPackages() != 50 || r.SinceDuration() != 14*24*time.Hour ||
+ r.MaxVersionsPerRun() != 0 || r.MaxBytesPerRun() != 0 {
+ t.Errorf("got interval %v top %d since %v max_versions %d max_bytes %d, want 24h 50 336h 0 0",
+ r.IntervalDuration(), r.TopPackages(), r.SinceDuration(), r.MaxVersionsPerRun(), r.MaxBytesPerRun())
+ }
+}
+
+func TestValidateRejectsMirrorRefreshWithoutCachedArtifacts(t *testing.T) {
+ cfg := Default()
+ cfg.Mirror.Refresh = &MirrorRefresh{Interval: "24h"}
+ cfg.Storage.CacheArtifacts = false
+ if err := cfg.Validate(); err == nil {
+ t.Error("Validate() accepted mirror.refresh with storage.cache_artifacts: false")
+ }
+}
+
+func TestLoadYAMLMirrorRefresh(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "config.yaml")
+ data := `
+mirror:
+ refresh:
+ interval: 12h
+ top: 10
+ since: 7d
+ max_versions: 200
+ max_bytes: 1GB
+`
+ if err := os.WriteFile(path, []byte(data), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ cfg, err := Load(path)
+ if err != nil {
+ t.Fatalf("Load: %v", err)
+ }
+ r := cfg.Mirror.Refresh
+ if r == nil {
+ t.Fatal("mirror.refresh not loaded")
+ }
+ if err := cfg.Validate(); err != nil {
+ t.Fatalf("Validate: %v", err)
+ }
+ if r.IntervalDuration() != 12*time.Hour || r.TopPackages() != 10 || r.SinceDuration() != 7*24*time.Hour ||
+ r.MaxVersionsPerRun() != 200 || r.MaxBytesPerRun() != 1<<30 {
+ t.Errorf("got interval %v top %d since %v max_versions %d max_bytes %d",
+ r.IntervalDuration(), r.TopPackages(), r.SinceDuration(), r.MaxVersionsPerRun(), r.MaxBytesPerRun())
+ }
+}
diff --git a/internal/database/metadata_upstream_url_test.go b/internal/database/metadata_upstream_url_test.go
new file mode 100644
index 00000000..361a2cc0
--- /dev/null
+++ b/internal/database/metadata_upstream_url_test.go
@@ -0,0 +1,90 @@
+package database
+
+import (
+ "database/sql"
+ "path/filepath"
+ "reflect"
+ "testing"
+ "time"
+)
+
+func TestMigrationAddsMetadataUpstreamURL(t *testing.T) {
+ dbPath := filepath.Join(t.TempDir(), "existing.db")
+ sqlDB, err := sql.Open("sqlite", dbPath)
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ if _, err := sqlDB.Exec(schemaSQLite); err != nil {
+ t.Fatalf("create schema: %v", err)
+ }
+ if _, err := sqlDB.Exec(`ALTER TABLE metadata_cache DROP COLUMN upstream_url`); err != nil {
+ t.Fatalf("drop upstream_url column: %v", err)
+ }
+ if _, err := sqlDB.Exec(`DELETE FROM migrations WHERE name = '023_add_metadata_upstream_url'`); err != nil {
+ t.Fatalf("forget migration: %v", err)
+ }
+ if err := sqlDB.Close(); err != nil {
+ t.Fatalf("close: %v", err)
+ }
+
+ db, err := Open(dbPath)
+ if err != nil {
+ t.Fatalf("Open: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+ if err := db.MigrateSchema(); err != nil {
+ t.Fatalf("MigrateSchema: %v", err)
+ }
+ has, err := db.HasColumn("metadata_cache", "upstream_url")
+ if err != nil || !has {
+ t.Errorf("metadata_cache.upstream_url after migration = %v, %v", has, err)
+ }
+}
+
+func upsertMetadataURL(t *testing.T, db *DB, ecosystem, name, url string) {
+ t.Helper()
+ entry := &MetadataCacheEntry{Ecosystem: ecosystem, Name: name, StoragePath: "_metadata/" + name}
+ if url != "" {
+ entry.UpstreamURL = sql.NullString{String: url, Valid: true}
+ }
+ if err := db.UpsertMetadataCache(entry); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestUpsertMetadataCacheKeepsUpstreamURL(t *testing.T) {
+ runWithBothDatabases(t, func(t *testing.T, db *DB) {
+ upsertMetadataURL(t, db, "apk", "k1", "https://dl-cdn.alpinelinux.org/alpine/v3.20/main/aarch64/APKINDEX.tar.gz")
+ // A writer that doesn't know the URL leaves it as it was.
+ upsertMetadataURL(t, db, "apk", "k1", "")
+ got, err := db.GetMetadataCache("apk", "k1")
+ if err != nil || got == nil {
+ t.Fatalf("GetMetadataCache: %+v, %v", got, err)
+ }
+ if got.UpstreamURL.String != "https://dl-cdn.alpinelinux.org/alpine/v3.20/main/aarch64/APKINDEX.tar.gz" {
+ t.Errorf("upstream_url = %q", got.UpstreamURL.String)
+ }
+ })
+}
+
+func TestRecentMetadataDocuments(t *testing.T) {
+ runWithBothDatabases(t, func(t *testing.T, db *DB) {
+ upsertMetadataURL(t, db, "apk", "k1", "https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz")
+ upsertMetadataURL(t, db, "apk", "k2", "https://a.test/v3.19/main/aarch64/APKINDEX.tar.gz")
+ upsertMetadataURL(t, db, "apk", "k3", "")
+ upsertMetadataURL(t, db, "debian", "k4", "https://d.test/dists/bookworm/InRelease")
+ old := time.Now().Add(-30 * 24 * time.Hour).UTC()
+ if _, err := db.Exec(db.Rebind("UPDATE metadata_cache SET last_accessed_at = ?, updated_at = ? WHERE name = ?"), old, old, "k2"); err != nil {
+ t.Fatal(err)
+ }
+
+ got, err := db.RecentMetadataDocuments("apk", time.Now().Add(-14*24*time.Hour))
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []MetadataDocument{{Name: "k1", UpstreamURL: "https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz"}}
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("RecentMetadataDocuments = %+v, want %+v", got, want)
+ }
+ })
+}
diff --git a/internal/database/most_used.go b/internal/database/most_used.go
new file mode 100644
index 00000000..c68e8335
--- /dev/null
+++ b/internal/database/most_used.go
@@ -0,0 +1,78 @@
+package database
+
+import (
+ "time"
+)
+
+// UsedPackage is a package clients download from the cache, with the files
+// cached for it.
+type UsedPackage struct {
+ Ecosystem string
+ Name string
+ PURL string
+ // Hits counts the downloads of the files downloaded since the cutoff.
+ Hits int64
+ Files []UsedFile
+}
+
+// UsedFile is a cached file of a UsedPackage.
+type UsedFile struct {
+ Version string
+ Filename string
+ UpstreamURL string
+}
+
+// MostUsedPackages returns, for each ecosystem, the top packages with the most
+// downloads among the cached files downloaded since cutoff, ordered by
+// ecosystem and then by hits. Each lists every file cached for it. Files
+// from local sources, which have no upstream, are left out.
+func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) {
+ var rows []struct {
+ Ecosystem string `db:"ecosystem"`
+ Name string `db:"name"`
+ PURL string `db:"purl"`
+ Hits int64 `db:"hits"`
+ }
+ err := db.Select(&rows, db.Rebind(`
+ SELECT ecosystem, name, purl, hits FROM (
+ SELECT p.ecosystem, p.name, p.purl, SUM(a.hit_count) AS hits,
+ ROW_NUMBER() OVER (PARTITION BY p.ecosystem ORDER BY SUM(a.hit_count) DESC, p.name) AS rank
+ FROM artifacts a
+ JOIN versions v ON v.purl = a.version_purl
+ JOIN packages p ON p.purl = v.package_purl
+ WHERE a.storage_path IS NOT NULL AND a.source = ?
+ AND COALESCE(a.last_accessed_at, a.fetched_at) >= ?
+ GROUP BY p.ecosystem, p.name, p.purl
+ ) ranked
+ WHERE rank <= ?
+ ORDER BY ecosystem, hits DESC, name
+ `), SourceUpstream, cutoff.UTC(), top)
+ if err != nil {
+ return nil, err
+ }
+ packages := make([]UsedPackage, len(rows))
+ for i, row := range rows {
+ packages[i] = UsedPackage{Ecosystem: row.Ecosystem, Name: row.Name, PURL: row.PURL, Hits: row.Hits}
+ var files []struct {
+ VersionPURL string `db:"version_purl"`
+ Filename string `db:"filename"`
+ UpstreamURL string `db:"upstream_url"`
+ }
+ err := db.Select(&files, db.Rebind(`
+ SELECT a.version_purl, a.filename, a.upstream_url
+ FROM artifacts a
+ JOIN versions v ON v.purl = a.version_purl
+ WHERE v.package_purl = ? AND a.storage_path IS NOT NULL AND a.source = ?
+ ORDER BY a.version_purl, a.filename
+ `), row.PURL, SourceUpstream)
+ if err != nil {
+ return nil, err
+ }
+ for _, f := range files {
+ packages[i].Files = append(packages[i].Files, UsedFile{
+ Version: VersionFromPURL(f.VersionPURL), Filename: f.Filename, UpstreamURL: f.UpstreamURL,
+ })
+ }
+ }
+ return packages, nil
+}
diff --git a/internal/database/most_used_test.go b/internal/database/most_used_test.go
new file mode 100644
index 00000000..8704f904
--- /dev/null
+++ b/internal/database/most_used_test.go
@@ -0,0 +1,94 @@
+package database
+
+import (
+ "reflect"
+ "testing"
+ "time"
+)
+
+// seedUsedArtifact stores a cached artifact with hits, last downloaded at
+// accessed.
+func seedUsedArtifact(t *testing.T, db *DB, ecosystem, name, version, filename string, hits int64, accessed time.Time) {
+ t.Helper()
+ packagePURL := "pkg:" + ecosystem + "/" + name
+ versionPURL := packagePURL + "@" + version
+ if err := db.UpsertPackage(&Package{PURL: packagePURL, Ecosystem: ecosystem, Name: name}); err != nil {
+ t.Fatal(err)
+ }
+ if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: packagePURL}); err != nil {
+ t.Fatal(err)
+ }
+ if err := db.UpsertArtifact(&Artifact{VersionPURL: versionPURL, Filename: filename, UpstreamURL: "https://example.test/" + filename}); err != nil {
+ t.Fatal(err)
+ }
+ if err := db.MarkArtifactCached(versionPURL, filename, ecosystem+"/"+filename, "sha256:00", 1, "application/octet-stream"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec(db.Rebind("UPDATE artifacts SET hit_count = ?, last_accessed_at = ? WHERE version_purl = ? AND filename = ?"),
+ hits, accessed.UTC(), versionPURL, filename); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestMostUsedPackages(t *testing.T) {
+ runWithBothDatabases(t, func(t *testing.T, db *DB) {
+ now := time.Now()
+ seedUsedArtifact(t, db, "npm", "lodash", "4.17.20", "lodash-4.17.20.tgz", 5, now.Add(-time.Hour))
+ seedUsedArtifact(t, db, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", 4, now.Add(-20*24*time.Hour))
+ seedUsedArtifact(t, db, "npm", "react", "18.3.1", "react-18.3.1.tgz", 10, now.Add(-time.Hour))
+ seedUsedArtifact(t, db, "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", 100, now.Add(-30*24*time.Hour))
+ seedUsedArtifact(t, db, "cargo", "serde", "1.0.200", "serde-1.0.200.crate", 1, now.Add(-time.Hour))
+
+ // Not cached: never counts.
+ if err := db.UpsertArtifact(&Artifact{VersionPURL: "pkg:npm/react@19.0.0", Filename: "react-19.0.0.tgz", UpstreamURL: "https://example.test/x"}); err != nil {
+ t.Fatal(err)
+ }
+
+ got, err := db.MostUsedPackages(now.Add(-14*24*time.Hour), 2)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []UsedPackage{
+ {Ecosystem: "cargo", Name: "serde", PURL: "pkg:cargo/serde", Hits: 1, Files: []UsedFile{
+ {Version: "1.0.200", Filename: "serde-1.0.200.crate", UpstreamURL: "https://example.test/serde-1.0.200.crate"},
+ }},
+ // react has more hits than lodash. Every cached file of lodash is
+ // listed, but only recent downloads count.
+ {Ecosystem: "npm", Name: "react", PURL: "pkg:npm/react", Hits: 10, Files: []UsedFile{
+ {Version: "18.3.1", Filename: "react-18.3.1.tgz", UpstreamURL: "https://example.test/react-18.3.1.tgz"},
+ }},
+ {Ecosystem: "npm", Name: "lodash", PURL: "pkg:npm/lodash", Hits: 5, Files: []UsedFile{
+ {Version: "4.17.20", Filename: "lodash-4.17.20.tgz", UpstreamURL: "https://example.test/lodash-4.17.20.tgz"},
+ {Version: "4.17.21", Filename: "lodash-4.17.21.tgz", UpstreamURL: "https://example.test/lodash-4.17.21.tgz"},
+ }},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("MostUsedPackages =\n%+v\nwant\n%+v", got, want)
+ }
+
+ top1, err := db.MostUsedPackages(now.Add(-14*24*time.Hour), 1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(top1) != 2 || top1[0].Name != "serde" || top1[1].Name != "react" {
+ t.Errorf("top 1 per ecosystem = %+v, want serde and react", top1)
+ }
+ })
+}
+
+func TestMostUsedPackagesSkipsLocalArtifacts(t *testing.T) {
+ runWithBothDatabases(t, func(t *testing.T, db *DB) {
+ now := time.Now()
+ seedUsedArtifact(t, db, "npm", "internal-lib", "1.0.0", "internal-lib-1.0.0.tgz", 50, now)
+ if _, err := db.Exec(db.Rebind("UPDATE artifacts SET source = ? WHERE filename = ?"), SourceLocal, "internal-lib-1.0.0.tgz"); err != nil {
+ t.Fatal(err)
+ }
+ got, err := db.MostUsedPackages(now.Add(-time.Hour), 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(got) != 0 {
+ t.Errorf("MostUsedPackages = %+v, want none: local packages have no upstream to refresh from", got)
+ }
+ })
+}
diff --git a/internal/database/queries.go b/internal/database/queries.go
index 86f31ef9..1b58d721 100644
--- a/internal/database/queries.go
+++ b/internal/database/queries.go
@@ -1321,7 +1321,7 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err
var entry MetadataCacheEntry
query := db.Rebind(`
SELECT id, ecosystem, name, storage_path, etag, link, content_type, content_encoding,
- content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at
+ content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at
FROM metadata_cache WHERE ecosystem = ? AND name = ?
`)
err := db.Get(&entry, query, ecosystem, name)
@@ -1334,6 +1334,26 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err
return &entry, nil
}
+// MetadataDocument names a cached metadata document and where it was
+// fetched from.
+type MetadataDocument struct {
+ Name string `db:"name"`
+ UpstreamURL string `db:"upstream_url"`
+}
+
+// RecentMetadataDocuments returns the documents of an ecosystem cached or
+// served since cutoff whose upstream URL was recorded.
+func (db *DB) RecentMetadataDocuments(ecosystem string, cutoff time.Time) ([]MetadataDocument, error) {
+ var docs []MetadataDocument
+ err := db.Select(&docs, db.Rebind(`
+ SELECT name, upstream_url FROM metadata_cache
+ WHERE ecosystem = ? AND upstream_url IS NOT NULL
+ AND COALESCE(last_accessed_at, updated_at) >= ?
+ ORDER BY upstream_url, name
+ `), ecosystem, cutoff.UTC())
+ return docs, err
+}
+
// RefreshMetadataCache sets fetched_at on the metadata cache row for ecosystem
// and name, provided the row still records digest, and reports whether it did.
// It is for an upstream 304, which leaves the stored document as it is.
@@ -1350,6 +1370,14 @@ func (db *DB) RefreshMetadataCache(ecosystem, name, digest string, fetchedAt tim
return n > 0, err
}
+// SetMetadataUpstreamURL records the upstream URL of the metadata cache row
+// for ecosystem and name, for rows cached before URLs were recorded.
+func (db *DB) SetMetadataUpstreamURL(ecosystem, name, upstreamURL string) error {
+ _, err := db.Exec(db.Rebind(`UPDATE metadata_cache SET upstream_url = ? WHERE ecosystem = ? AND name = ?`),
+ upstreamURL, ecosystem, name)
+ return err
+}
+
// MoveMetadataCache points the metadata cache row for ecosystem and name at
// to, provided it is still at from and records digest, and reports whether
// it did. The caller has copied the document from from to to.
@@ -1439,8 +1467,8 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error {
if db.dialect == DialectPostgres {
query = `
INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding,
- content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at)
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
+ content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
ON CONFLICT(ecosystem, name) DO UPDATE SET
storage_path = EXCLUDED.storage_path,
etag = EXCLUDED.etag,
@@ -1448,6 +1476,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error {
content_type = EXCLUDED.content_type,
content_encoding = EXCLUDED.content_encoding,
content_digest = EXCLUDED.content_digest,
+ upstream_url = COALESCE(EXCLUDED.upstream_url, metadata_cache.upstream_url),
size = EXCLUDED.size,
last_modified = EXCLUDED.last_modified,
fetched_at = EXCLUDED.fetched_at,
@@ -1457,8 +1486,8 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error {
} else {
query = `
INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding,
- content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(ecosystem, name) DO UPDATE SET
storage_path = excluded.storage_path,
etag = excluded.etag,
@@ -1466,6 +1495,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error {
content_type = excluded.content_type,
content_encoding = excluded.content_encoding,
content_digest = excluded.content_digest,
+ upstream_url = COALESCE(excluded.upstream_url, metadata_cache.upstream_url),
size = excluded.size,
last_modified = excluded.last_modified,
fetched_at = excluded.fetched_at,
@@ -1476,7 +1506,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error {
_, err := db.Exec(query,
entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag, entry.Link,
- entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.Size, entry.LastModified, fetchedAt, now.UTC(), now, now,
+ entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.UpstreamURL, entry.Size, entry.LastModified, fetchedAt, now.UTC(), now, now,
)
if err != nil {
return fmt.Errorf("upserting metadata cache: %w", err)
diff --git a/internal/database/schema.go b/internal/database/schema.go
index 6a9adaaa..bca37bd4 100644
--- a/internal/database/schema.go
+++ b/internal/database/schema.go
@@ -129,6 +129,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache (
content_type TEXT,
content_encoding TEXT,
content_digest TEXT,
+ upstream_url TEXT,
size INTEGER,
last_modified DATETIME,
fetched_at DATETIME,
@@ -307,6 +308,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache (
content_type TEXT,
content_encoding TEXT,
content_digest TEXT,
+ upstream_url TEXT,
size BIGINT,
last_modified TIMESTAMP,
fetched_at TIMESTAMP,
@@ -543,6 +545,7 @@ var migrations = []migration{
{"020_add_metadata_snapshots", migrateAddMetadataSnapshots},
{"021_add_projects", migrateAddProjects},
{"022_add_artifact_mirrored_at", migrateAddArtifactMirroredAt},
+ {"023_add_metadata_upstream_url", migrateAddMetadataUpstreamURL},
}
// isTableNotFound returns true if the error indicates a missing table.
@@ -803,6 +806,22 @@ func migrateAddMetadataLink(db *DB) error {
return nil
}
+// migrateAddMetadataUpstreamURL adds the URL a metadata document was
+// fetched from, which the mirror refresh fetches it from again.
+func migrateAddMetadataUpstreamURL(db *DB) error {
+ hasColumn, err := db.HasColumn("metadata_cache", "upstream_url")
+ if err != nil {
+ return fmt.Errorf("checking metadata_cache upstream_url column: %w", err)
+ }
+ if hasColumn {
+ return nil
+ }
+ if _, err := db.Exec("ALTER TABLE metadata_cache ADD COLUMN upstream_url TEXT"); err != nil {
+ return fmt.Errorf("adding metadata_cache upstream_url column: %w", err)
+ }
+ return nil
+}
+
func migrateAddMetadataContentEncoding(db *DB) error {
hasColumn, err := db.HasColumn("metadata_cache", "content_encoding")
if err != nil {
diff --git a/internal/database/types.go b/internal/database/types.go
index c2675d1c..4e31a683 100644
--- a/internal/database/types.go
+++ b/internal/database/types.go
@@ -187,12 +187,15 @@ type MetadataCacheEntry struct {
ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"`
ContentEncoding sql.NullString `db:"content_encoding" json:"content_encoding,omitempty"`
ContentDigest sql.NullString `db:"content_digest" json:"content_digest,omitempty"`
- Size sql.NullInt64 `db:"size" json:"size,omitempty"`
- LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"`
- FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"`
- LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"`
- CreatedAt time.Time `db:"created_at" json:"created_at"`
- UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
+ // UpstreamURL is where the document was fetched from. Rows written
+ // before it was recorded, and by writers that don't know it, have none.
+ UpstreamURL sql.NullString `db:"upstream_url" json:"upstream_url,omitempty"`
+ Size sql.NullInt64 `db:"size" json:"size,omitempty"`
+ LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"`
+ FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"`
+ LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"`
+ CreatedAt time.Time `db:"created_at" json:"created_at"`
+ UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
}
// Vulnerability represents a cached vulnerability record.
diff --git a/internal/handler/handler.go b/internal/handler/handler.go
index d316aabc..495a92bf 100644
--- a/internal/handler/handler.go
+++ b/internal/handler/handler.go
@@ -1980,6 +1980,8 @@ type upstreamMetadata struct {
contentEncoding string
etag string
lastModified time.Time
+ // url is the upstream URL the document was fetched from.
+ url string
// notModified is the cache row upstream answered 304 for, whose stored
// bytes body holds. It is nil for a document upstream sent.
notModified *database.MetadataCacheEntry
@@ -2009,7 +2011,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
if readErr != nil {
return nil, errStale304
}
- meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, notModified: entry}
+ meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, url: upstreamURL, notModified: entry}
if entry.ContentType.Valid {
meta.contentType = entry.ContentType.String
}
@@ -2038,6 +2040,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
contentType: resp.Header.Get(headerContentType),
contentEncoding: resp.Header.Get(headerContentEncoding),
etag: resp.Header.Get(headerETag),
+ url: upstreamURL,
}
if meta.contentType == "" {
meta.contentType = contentTypeJSON
@@ -2105,6 +2108,11 @@ func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMe
if !refreshed {
return ""
}
+ if !entry.UpstreamURL.Valid && meta.url != "" {
+ if err := p.DB.SetMetadataUpstreamURL(ecosystem, cacheKey, meta.url); err != nil {
+ p.Logger.Warn("failed to record metadata upstream URL", "ecosystem", ecosystem, "key", cacheKey, "error", err)
+ }
+ }
p.recordMetadataRead(entry)
return entry.ContentDigest.String
}
@@ -2145,6 +2153,7 @@ func (p *Proxy) storeMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor
// The digest identifies the stored bytes, so a rewrite cached for them
// can be found without reading them back (see storedRewrite).
ContentDigest: sql.NullString{String: digest, Valid: digest != ""},
+ UpstreamURL: sql.NullString{String: meta.url, Valid: meta.url != ""},
Size: sql.NullInt64{Int64: size, Valid: true},
LastModified: sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()},
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go
index 8ce636c9..ed56b631 100644
--- a/internal/handler/handler_test.go
+++ b/internal/handler/handler_test.go
@@ -13,6 +13,7 @@ import (
"slices"
"strings"
"sync"
+ "sync/atomic"
"testing"
"time"
@@ -1584,3 +1585,54 @@ func TestCanonicalPackagePURLMatchesConfig(t *testing.T) {
})
}
}
+
+func TestProxyCachedRecordsUpstreamURL(t *testing.T) {
+ proxy, upstream := setupCachedProxy(t, `"abc123"`, "")
+ req := httptest.NewRequest(http.MethodGet, "/test", nil)
+ proxy.ProxyCached(httptest.NewRecorder(), req, upstream.URL+"/test", "test-eco", "url-key")
+
+ entry, err := proxy.DB.GetMetadataCache("test-eco", "url-key")
+ if err != nil || entry == nil {
+ t.Fatalf("GetMetadataCache: %+v, %v", entry, err)
+ }
+ if entry.UpstreamURL.String != upstream.URL+"/test" {
+ t.Errorf("upstream_url = %q, want %q", entry.UpstreamURL.String, upstream.URL+"/test")
+ }
+}
+
+// A row cached before upstream URLs were recorded gets its URL on the next
+// revalidation, even when upstream answers 304.
+func TestProxyCachedRecordsUpstreamURLOnNotModified(t *testing.T) {
+ var notModified atomic.Int32
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("ETag", `"v1"`)
+ if r.Header.Get("If-None-Match") == `"v1"` {
+ notModified.Add(1)
+ w.WriteHeader(http.StatusNotModified)
+ return
+ }
+ _, _ = w.Write([]byte(`{"ok":true}`))
+ }))
+ t.Cleanup(upstream.Close)
+ proxy, _, _, _ := setupTestProxy(t)
+ proxy.CacheMetadata = true
+ proxy.HTTPClient = upstream.Client()
+
+ proxy.ProxyCached(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/test", nil), upstream.URL+"/test", "test-eco", "url-304-key")
+ if _, err := proxy.DB.Exec("UPDATE metadata_cache SET upstream_url = NULL"); err != nil {
+ t.Fatal(err)
+ }
+
+ w := httptest.NewRecorder()
+ proxy.ProxyCached(w, httptest.NewRequest(http.MethodGet, "/test", nil), upstream.URL+"/test", "test-eco", "url-304-key")
+ if w.Code != http.StatusOK || notModified.Load() != 1 {
+ t.Fatalf("status = %d, upstream 304s = %d; want 200 after one 304", w.Code, notModified.Load())
+ }
+ entry, err := proxy.DB.GetMetadataCache("test-eco", "url-304-key")
+ if err != nil || entry == nil {
+ t.Fatalf("GetMetadataCache: %+v, %v", entry, err)
+ }
+ if entry.UpstreamURL.String != upstream.URL+"/test" {
+ t.Errorf("upstream_url after a 304 = %q, want %q", entry.UpstreamURL.String, upstream.URL+"/test")
+ }
+}
diff --git a/internal/handler/os_package_identity.go b/internal/handler/os_package_identity.go
new file mode 100644
index 00000000..0e0400be
--- /dev/null
+++ b/internal/handler/os_package_identity.go
@@ -0,0 +1,17 @@
+package handler
+
+// OSPackageIdentity returns the name, version and architecture the deb,
+// alpine and rpm download handlers cache a package file under, read from its
+// path below the repository the way they read it. It returns empty strings
+// for other ecosystems and for paths they would not cache.
+func OSPackageIdentity(ecosystem, path string) (name, version, arch string) {
+ switch ecosystem {
+ case "deb":
+ return (*DebianHandler)(nil).parsePoolPath("/" + path)
+ case apkEcosystem:
+ return (*APKHandler)(nil).parseAPKPath(path)
+ case "rpm":
+ return (*RPMHandler)(nil).parseRPMPath(path)
+ }
+ return "", "", ""
+}
diff --git a/internal/handler/os_package_identity_test.go b/internal/handler/os_package_identity_test.go
new file mode 100644
index 00000000..3d487bec
--- /dev/null
+++ b/internal/handler/os_package_identity_test.go
@@ -0,0 +1,23 @@
+package handler
+
+import "testing"
+
+func TestOSPackageIdentity(t *testing.T) {
+ tests := []struct {
+ ecosystem, path string
+ name, version, arch string
+ }{
+ {"deb", "pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb", "curl", "7.88.1-10+deb12u8", "amd64"},
+ {"alpine", "v3.20/main/aarch64/curl-8.9.1-r0.apk", "curl", "8.9.1-r0", "aarch64"},
+ {"rpm", "Packages/c/curl-8.6.0-10.fc40.x86_64.rpm", "curl", "8.6.0-10.fc40", "x86_64"},
+ {"deb", "pool/main/c/curl/README", "", "", ""},
+ {"npm", "lodash-4.17.21.tgz", "", "", ""},
+ }
+ for _, tt := range tests {
+ name, version, arch := OSPackageIdentity(tt.ecosystem, tt.path)
+ if name != tt.name || version != tt.version || arch != tt.arch {
+ t.Errorf("OSPackageIdentity(%s, %s) = %q %q %q, want %q %q %q",
+ tt.ecosystem, tt.path, name, version, arch, tt.name, tt.version, tt.arch)
+ }
+ }
+}
diff --git a/internal/handler/release_allowed.go b/internal/handler/release_allowed.go
new file mode 100644
index 00000000..4c70214c
--- /dev/null
+++ b/internal/handler/release_allowed.go
@@ -0,0 +1,19 @@
+package handler
+
+import (
+ "time"
+)
+
+// ReleaseAllowed reports whether clients would be served a version published
+// at publishedAt: the denylist doesn't name it, and cooldown, where the
+// ecosystem applies it, doesn't withhold it. Without a publish time a version
+// counts as withheld wherever cooldown applies.
+func (p *Proxy) ReleaseAllowed(ecosystem, name, version string, publishedAt time.Time) bool {
+ if p.versionDenied(ecosystem, name, version) {
+ return false
+ }
+ if !CooldownSupported(ecosystem) || !p.cooldownApplies(ecosystem, name) {
+ return true
+ }
+ return !publishedAt.IsZero() && p.cooldown().IsAllowed(ecosystem, canonicalPackagePURL(ecosystem, name), publishedAt)
+}
diff --git a/internal/handler/release_allowed_test.go b/internal/handler/release_allowed_test.go
new file mode 100644
index 00000000..0d8dd88e
--- /dev/null
+++ b/internal/handler/release_allowed_test.go
@@ -0,0 +1,44 @@
+package handler
+
+import (
+ "testing"
+ "time"
+
+ "github.com/git-pkgs/cooldown"
+ "github.com/git-pkgs/proxy/internal/denylist"
+)
+
+func TestReleaseAllowed(t *testing.T) {
+ deny, err := denylist.New([]string{"pkg:npm/lodash@4.17.22"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ p := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}, Denylist: deny}
+ old := time.Now().Add(-10 * 24 * time.Hour)
+ fresh := time.Now().Add(-time.Hour)
+
+ tests := []struct {
+ name string
+ ecosystem, pkg, version string
+ published time.Time
+ want bool
+ }{
+ {"old enough", "npm", "lodash", "4.17.21", old, true},
+ {"within cooldown", "npm", "lodash", "4.17.23", fresh, false},
+ {"denied", "npm", "lodash", "4.17.22", old, false},
+ {"no publish time under cooldown", "npm", "lodash", "4.17.24", time.Time{}, false},
+ // deb has no publish times, so cooldown never applies to it.
+ {"ecosystem without cooldown", "deb", "curl", "8.5.0-2", time.Time{}, true},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := p.ReleaseAllowed(tt.ecosystem, tt.pkg, tt.version, tt.published); got != tt.want {
+ t.Errorf("ReleaseAllowed = %v, want %v", got, tt.want)
+ }
+ })
+ }
+
+ if !(&Proxy{}).ReleaseAllowed("npm", "lodash", "4.17.23", fresh) {
+ t.Error("a proxy without cooldown or denylist refused a release")
+ }
+}
diff --git a/internal/mirror/files.go b/internal/mirror/files.go
index 7486e8c8..ab6bafdb 100644
--- a/internal/mirror/files.go
+++ b/internal/mirror/files.go
@@ -2,6 +2,8 @@ package mirror
import (
"fmt"
+ "path"
+ "regexp"
"slices"
"strings"
@@ -22,6 +24,12 @@ type fileEcosystem struct {
// check refuses a package the ecosystem can't mirror, before its files
// are listed.
check func(pv PackageVersion) error
+ // variants name the kinds of a file, such as each tag combination of a
+ // wheel or a conda package's platform, so a refresh mirrors the same
+ // kinds of files of a new version as were cached of an old one.
+ // cachedVariants names them for the filename cacheKey returns.
+ variants func(file registries.Artifact) []string
+ cachedVariants func(cacheFilename string) []string
}
var fileEcosystems = map[string]fileEcosystem{
@@ -37,6 +45,8 @@ var fileEcosystems = map[string]fileEcosystem{
}
return name, version, file.Filename, true
},
+ variants: func(file registries.Artifact) []string { return pypiVariants(file.Filename) },
+ cachedVariants: pypiVariants,
},
"conda": {
qualifiers: []string{"subdir", "build", "type"},
@@ -60,9 +70,81 @@ var fileEcosystems = map[string]fileEcosystem{
}
return nil
},
+ variants: func(file registries.Artifact) []string {
+ subdir, _ := file.Metadata["subdir"].(string)
+ return []string{condaVariant(subdir, file.Filename)}
+ },
+ cachedVariants: func(cacheFilename string) []string {
+ subdir, filename := path.Split(cacheFilename)
+ return []string{condaVariant(strings.TrimSuffix(subdir, "/"), filename)}
+ },
},
}
+// pypiVariants names a wheel by each combination of its python, ABI and
+// platform tags, which are alternatives where a tag lists several joined by
+// dots, with the platform reduced to OS and architecture. Every sdist is
+// one variant.
+func pypiVariants(filename string) []string {
+ stem, wheel := strings.CutSuffix(filename, ".whl")
+ if !wheel {
+ return []string{"sdist"}
+ }
+ parts := strings.Split(stem, "-")
+ if len(parts) < 5 { //nolint:mnd // name, version and three tags
+ return []string{stem}
+ }
+ tags := parts[len(parts)-3:]
+ var variants []string
+ for _, python := range strings.Split(tags[0], ".") {
+ for _, abi := range strings.Split(tags[1], ".") {
+ for _, platform := range strings.Split(tags[2], ".") {
+ v := wheelPython(python, abi) + "-" + abi + "-" + wheelPlatform(platform)
+ if !slices.Contains(variants, v) {
+ variants = append(variants, v)
+ }
+ }
+ }
+ }
+ return variants
+}
+
+// wheelPython reduces the python tag of an abi3 wheel, a minimum such as
+// cp39, to its interpreter, so cp38-abi3 and cp311-abi3 wheels are the same
+// kind.
+func wheelPython(python, abi string) string {
+ if abi != "abi3" {
+ return python
+ }
+ return strings.TrimRight(python, "0123456789")
+}
+
+// wheelPlatformMinimum matches the glibc, musl or macOS minimum version in a
+// platform tag, such as the 2_17 in manylinux_2_17_x86_64 or the 2014 in
+// manylinux2014_x86_64.
+var wheelPlatformMinimum = regexp.MustCompile(`^(manylinux|musllinux|macosx)_?(?:\d+_\d+|1|2010|2014)_`)
+
+// wheelPlatform reduces a platform tag to OS and architecture, so
+// manylinux_2_17_x86_64 and manylinux_2_28_x86_64 name the same platform.
+func wheelPlatform(tag string) string {
+ return wheelPlatformMinimum.ReplaceAllString(tag, "${1}_")
+}
+
+// condaBuildHash matches the hash and build number in a conda build string,
+// such as h8753938_0 in py312h8753938_0 or h5f976f7_0 in
+// h5f976f7_0_cpython.
+var condaBuildHash = regexp.MustCompile(`h[0-9a-f]{7}_\d+`)
+
+// condaVariant names a conda package file by its subdir, its build string
+// without the hash, such as py312 or _cpython, and its type.
+func condaVariant(subdir, filename string) string {
+ fileType := condaFileType(filename)
+ stem := strings.TrimSuffix(strings.TrimSuffix(filename, ".conda"), ".tar.bz2")
+ build := stem[strings.LastIndex(stem, "-")+1:]
+ build = condaBuildHash.ReplaceAllString(build, "")
+ return subdir + "/" + build + "/" + fileType
+}
+
// condaFileType returns the PURL type qualifier of a conda package file.
func condaFileType(filename string) string {
switch {
@@ -78,6 +160,9 @@ func condaFileType(filename string) string {
// selection returns the qualifiers of pv that select files, as
// "key=value" pairs, for error messages.
func (e fileEcosystem) selection(pv PackageVersion) string {
+ if pv.variants != nil {
+ return "the kinds of files cached of earlier versions"
+ }
var pairs []string
for _, key := range e.qualifiers {
if value := pv.Qualifiers[key]; value != "" {
@@ -89,6 +174,9 @@ func (e fileEcosystem) selection(pv PackageVersion) string {
// selects reports whether file matches every selecting qualifier of pv.
func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool {
+ if pv.variants != nil && !slices.ContainsFunc(e.variants(file), func(v string) bool { return pv.variants[v] }) {
+ return false
+ }
attrs := e.attrs(file)
for _, key := range e.qualifiers {
if want := pv.Qualifiers[key]; want != "" && attrs[key] != want {
diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go
index c08b1fbd..b32c7cab 100644
--- a/internal/mirror/mirror.go
+++ b/internal/mirror/mirror.go
@@ -255,6 +255,15 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog
"ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages)
return
}
+ if pv.fileURL != "" {
+ result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, pv.Name, pv.Version, pv.cacheFilename, pv.fileURL, pv.digest)
+ if err != nil {
+ m.recordFailure(tracker, pv, err)
+ return
+ }
+ m.recordResult(tracker, pv, result)
+ return
+ }
if eco, ok := fileEcosystems[pv.Ecosystem]; ok {
m.mirrorFiles(ctx, pv, eco, tracker)
return
diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go
new file mode 100644
index 00000000..6fc036b6
--- /dev/null
+++ b/internal/mirror/refresh.go
@@ -0,0 +1,177 @@
+package mirror
+
+import (
+ "context"
+ "fmt"
+ "slices"
+ "time"
+
+ "github.com/git-pkgs/proxy/internal/database"
+ "github.com/git-pkgs/purl"
+ "github.com/git-pkgs/registries"
+)
+
+// Refresh mirrors new releases of the packages clients download most, so
+// they are cached before the first client asks for them.
+type Refresh struct {
+ // Top is how many packages per ecosystem are refreshed.
+ Top int
+ // Since is how recently a package must have been downloaded.
+ Since time.Duration
+ // MaxVersions stops a run after mirroring this many versions. 0 means
+ // no cap.
+ MaxVersions int
+ // MaxBytes stops a run once it has downloaded this much. It is checked
+ // before each version, so the last one may go over it. 0 means no cap.
+ MaxBytes int64
+}
+
+func (r Refresh) spent(tracker *progressTracker) bool {
+ return (r.MaxVersions > 0 && tracker.completed.Load() >= int64(r.MaxVersions)) ||
+ (r.MaxBytes > 0 && tracker.bytes.Load() >= r.MaxBytes)
+}
+
+// ReleaseLister lists every release of a package, with its publish time and
+// status, for refreshes.
+type ReleaseLister interface {
+ ListReleases(ctx context.Context, ecosystem, name string) ([]registries.Version, error)
+}
+
+// ListReleases implements ReleaseLister.
+func (r RegistryVersions) ListReleases(ctx context.Context, ecosystem, name string) ([]registries.Version, error) {
+ return r.fetchVersions(ctx, ecosystem, name)
+}
+
+// releaseLister lists releases where Versions lists versions, so both come
+// from the configured upstream.
+func (m *Mirror) releaseLister() ReleaseLister { //nolint:ireturn // any lister will do
+ if lister, ok := m.Versions.(ReleaseLister); ok {
+ return lister
+ }
+ return RegistryVersions{}
+}
+
+// Refresh mirrors, for each of the most used packages, the newest release
+// of each major line it has a version cached on, as clients would be served
+// it: releases within cooldown or on the denylist are left out. Packages it
+// can't list are counted as failed.
+func (m *Mirror) Refresh(ctx context.Context, r Refresh) (*Progress, error) {
+ cutoff := time.Now().Add(-r.Since)
+ packages, err := m.db.MostUsedPackages(cutoff, r.Top)
+ if err != nil {
+ return nil, fmt.Errorf("listing the most used packages: %w", err)
+ }
+ tracker := newProgressTracker()
+ tracker.phase.Store("downloading")
+ run := func(items []PackageVersion) bool {
+ for _, item := range items {
+ if ctx.Err() != nil {
+ return false
+ }
+ if r.spent(tracker) {
+ m.logger.Info("mirror refresh reached its cap for one run",
+ "mirrored", tracker.completed.Load(), "bytes", tracker.bytes.Load())
+ return false
+ }
+ tracker.total.Add(1)
+ m.mirrorOne(ctx, item, tracker)
+ }
+ return true
+ }
+ for _, group := range byEcosystem(packages) {
+ if ix, ok := osIndexes[group[0].Ecosystem]; ok {
+ items, errs := m.indexRefreshItems(ctx, ix, group, cutoff)
+ for _, err := range errs {
+ m.recordFailure(tracker, PackageVersion{Ecosystem: ix.ecosystem}, err)
+ }
+ if !run(items) {
+ break
+ }
+ continue
+ }
+ done := true
+ for _, pkg := range group {
+ items, err := m.refreshItems(ctx, pkg)
+ if err != nil {
+ m.recordFailure(tracker, PackageVersion{Ecosystem: pkg.Ecosystem, Name: pkg.Name}, err)
+ continue
+ }
+ if done = run(items); !done {
+ break
+ }
+ }
+ if !done {
+ break
+ }
+ }
+ tracker.phase.Store("complete")
+ p := tracker.snapshot()
+ return &p, nil
+}
+
+// byEcosystem splits packages, ordered by ecosystem, into one group per
+// ecosystem.
+func byEcosystem(packages []database.UsedPackage) [][]database.UsedPackage {
+ var groups [][]database.UsedPackage
+ for i, pkg := range packages {
+ if i == 0 || pkg.Ecosystem != packages[i-1].Ecosystem {
+ groups = append(groups, nil)
+ }
+ groups[len(groups)-1] = append(groups[len(groups)-1], pkg)
+ }
+ return groups
+}
+
+// refreshItems returns the versions to mirror for a used package.
+func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([]PackageVersion, error) {
+ p, err := purl.Parse(pkg.PURL)
+ if err != nil {
+ return nil, err
+ }
+ ecosystem, name, _ := mirrorTarget(p)
+ if !refreshable(ecosystem) {
+ return nil, nil
+ }
+ releases, err := m.releaseLister().ListReleases(ctx, ecosystem, name)
+ if err != nil {
+ return nil, fmt.Errorf("listing releases: %w", err)
+ }
+
+ var cached []string
+ seen := map[string]bool{}
+ for _, f := range pkg.Files {
+ if !seen[f.Version] {
+ seen[f.Version] = true
+ cached = append(cached, f.Version)
+ }
+ }
+ allowed := func(v registries.Version) bool {
+ return m.proxy == nil || m.proxy.ReleaseAllowed(pkg.Ecosystem, pkg.Name, v.Number, v.PublishedAt)
+ }
+
+ var items []PackageVersion
+ for _, version := range newestPerCachedLine(ecosystem, cached, releases, allowed) {
+ pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: version}
+ if eco, ok := fileEcosystems[ecosystem]; ok {
+ pv.variants = map[string]bool{}
+ for _, f := range pkg.Files {
+ for _, v := range eco.cachedVariants(f.Filename) {
+ pv.variants[v] = true
+ }
+ }
+ }
+ items = append(items, pv)
+ }
+ return items, nil
+}
+
+// refreshable reports whether a refresh lists the releases of an
+// ecosystem's packages. Images follow mirror.schedules, OS packages their
+// repository indexes, and Julia files are cached by tree hash rather than by
+// version.
+func refreshable(ecosystem string) bool {
+ if isImageEcosystem(ecosystem) || osIndexes[ecosystem].ecosystem != "" || ecosystem == "julia" {
+ return false
+ }
+ return slices.Contains(registries.SupportedEcosystems(), purl.EcosystemToPURLType(ecosystem))
+}
diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go
new file mode 100644
index 00000000..9c311dd7
--- /dev/null
+++ b/internal/mirror/refresh_harden_test.go
@@ -0,0 +1,309 @@
+package mirror
+
+import (
+ "context"
+ "reflect"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/git-pkgs/proxy/internal/denylist"
+ "github.com/git-pkgs/registries"
+)
+
+// debCurlPackages returns a Packages index listing curl at version.
+func debCurlPackages(t *testing.T, version string) []byte {
+ t.Helper()
+ return gzipped(t, "Package: curl\nVersion: "+version+"\nArchitecture: amd64\nFilename: pool/main/c/curl/curl_"+version+"_amd64.deb\n\n")
+}
+
+// TestRefreshDebianIndexPerSuite checks that suites sharing a pool are
+// refreshed each on its own: a newer curl cached from trixie must not keep
+// bookworm's security update from being mirrored.
+func TestRefreshDebianIndexPerSuite(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n")
+ pool := idx.URL + "/debian/pool/main/c/curl/"
+ for suite, version := range map[string]string{"bookworm": "7.88.1-10", "trixie": "8.14.1-2"} {
+ idx.set("/debian/dists/"+suite+"/InRelease", release)
+ idx.set("/debian/dists/"+suite+"/main/binary-amd64/Packages.gz", debCurlPackages(t, version))
+ clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/"+suite+"/InRelease",
+ "deb", version, "curl_"+version+"_amd64.deb", pool+"curl_"+version+"_amd64.deb")
+ }
+
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10+deb12u8"))
+ idx.set("/debian/dists/trixie/main/binary-amd64/Packages.gz", debCurlPackages(t, "8.14.1-2+deb13u1"))
+ fetcher.reset()
+ progress := refreshOnce(t, m)
+
+ want := []string{pool + "curl_7.88.1-10+deb12u8_amd64.deb", pool + "curl_8.14.1-2+deb13u1_amd64.deb"}
+ got := fetcher.fetched()
+ slices.Sort(got)
+ if !slices.Equal(got, want) || progress.Completed != 2 {
+ t.Errorf("progress = %+v, fetched %v; want %v", progress, got, want)
+ }
+}
+
+// TestRefreshDebianIndexSkipsOtherSuite checks that a suite's version of a
+// package is not mirrored when only a newer one is cached: clients of that
+// suite never fetched the package.
+func TestRefreshDebianIndexSkipsOtherSuite(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n")
+ pool := idx.URL + "/debian/pool/main/c/curl/"
+ idx.set("/debian/dists/trixie/InRelease", release)
+ idx.set("/debian/dists/trixie/main/binary-amd64/Packages.gz", debCurlPackages(t, "8.14.1-2"))
+ clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/trixie/InRelease",
+ "deb", "8.14.1-2", "curl_8.14.1-2_amd64.deb", pool+"curl_8.14.1-2_amd64.deb")
+ // A bookworm client fetched its index but never curl.
+ idx.set("/debian/dists/bookworm/InRelease", release)
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10+deb12u8"))
+ if _, _, err := m.proxy.FetchOrCacheMetadata(t.Context(), "debian", "key:bookworm", idx.URL+"/debian/dists/bookworm/InRelease", "*/*"); err != nil {
+ t.Fatal(err)
+ }
+ fetcher.reset()
+
+ if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 {
+ t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched())
+ }
+}
+
+// TestRefreshRPMIndexListingSeveralVersions checks that a repository
+// listing old versions next to the newest mirrors only the newest, and
+// nothing once it is cached.
+func TestRefreshRPMIndexListingSeveralVersions(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ base := "/fedora/updates/40/Everything/x86_64"
+ primary := func(rels ...string) []byte {
+ var pkgs string
+ for _, rel := range rels {
+ pkgs += `curlx86_64`
+ }
+ return gzipped(t, ``+pkgs+``)
+ }
+ idx.set(base+"/repodata/repomd.xml", []byte(``))
+ idx.set(base+"/repodata/primary.xml.gz", primary("10"))
+ clientFetchesCurl(t, m, "rpm", idx.URL+base+"/repodata/repomd.xml",
+ "rpm", "8.6.0-10.fc40", "curl-8.6.0-10.fc40.x86_64.rpm", idx.URL+base+"/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm")
+
+ idx.set(base+"/repodata/primary.xml.gz", primary("10", "12", "11"))
+ fetcher.reset()
+ progress := refreshOnce(t, m)
+ want := []string{idx.URL + base + "/Packages/c/curl-8.6.0-12.fc40.x86_64.rpm"}
+ if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 {
+ t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want)
+ }
+
+ fetcher.reset()
+ if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 {
+ t.Errorf("second run: progress = %+v, fetched %v; want nothing", progress, fetcher.fetched())
+ }
+}
+
+// TestRefreshIndexFailureDoesNotStopRun checks that an index that can't be
+// read is counted as failed and the other indexes are still refreshed.
+func TestRefreshIndexFailureDoesNotStopRun(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n")
+ pool := idx.URL + "/debian/pool/main/c/curl/"
+ for suite, version := range map[string]string{"bookworm": "7.88.1-10", "trixie": "8.14.1-2"} {
+ idx.set("/debian/dists/"+suite+"/InRelease", release)
+ idx.set("/debian/dists/"+suite+"/main/binary-amd64/Packages.gz", debCurlPackages(t, version))
+ clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/"+suite+"/InRelease",
+ "deb", version, "curl_"+version+"_amd64.deb", pool+"curl_"+version+"_amd64.deb")
+ }
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", []byte("not gzip"))
+ idx.set("/debian/dists/trixie/main/binary-amd64/Packages.gz", debCurlPackages(t, "8.14.1-2+deb13u1"))
+ fetcher.reset()
+
+ progress := refreshOnce(t, m)
+ want := []string{pool + "curl_8.14.1-2+deb13u1_amd64.deb"}
+ if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 || progress.Failed != 1 {
+ t.Errorf("progress = %+v, fetched %v; want %v and one failure", progress, fetcher.fetched(), want)
+ }
+}
+
+func TestNewestPerCachedLineGo(t *testing.T) {
+ allowAll := func(registries.Version) bool { return true }
+ for _, tt := range []struct {
+ name string
+ cached []string
+ releases []registries.Version
+ want []string
+ }{
+ {"v prefix", []string{"v1.2.3"}, releases("v1.2.3", "v1.3.0"), []string{"v1.3.0"}},
+ {"incompatible is its own line", []string{"v1.2.3"}, releases("v1.2.3", "v2.0.0+incompatible"), nil},
+ {"pseudo-versions are prereleases", []string{"v1.2.3"}, releases("v1.2.3", "v1.2.4-0.20260101000000-abcdefabcdef"), nil},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := newestPerCachedLine("golang", tt.cached, tt.releases, allowAll); !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("newestPerCachedLine = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestRefreshScopedNPMPackage(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "tarball")
+ cacheNPM(t, m, "@babel/core", "7.24.0")
+ m.Versions = stubReleases{"npm/@babel/core": oldReleases("7.24.0", "7.25.2")}
+ fetcher.reset()
+
+ progress := refreshOnce(t, m)
+ if urls := fetcher.fetched(); progress.Completed != 1 || len(fetchedMatching(urls, "core-7.25.2.tgz")) != 1 {
+ t.Errorf("progress = %+v, fetched %v; want @babel/core 7.25.2", progress, urls)
+ }
+}
+
+func TestRefreshSkipsDeniedRelease(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "tarball")
+ cacheNPM(t, m, "lodash", "4.17.20")
+ m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21", "4.17.22")}
+ policy, err := denylist.New([]string{"pkg:npm/lodash@4.17.22"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ m.proxy.Denylist = policy
+ fetcher.reset()
+
+ progress := refreshOnce(t, m)
+ if urls := fetcher.fetched(); progress.Completed != 1 || len(fetchedMatching(urls, "lodash-4.17.21.tgz")) != 1 || len(fetchedMatching(urls, "4.17.22")) != 0 {
+ t.Errorf("progress = %+v, fetched %v; want 4.17.21, not the denied 4.17.22", progress, urls)
+ }
+}
+
+func TestRefreshListingFailureDoesNotStopOthers(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "tarball")
+ cacheNPM(t, m, "lodash", "4.17.20")
+ cacheNPM(t, m, "react", "18.3.0")
+ m.Versions = stubReleases{"npm/react": oldReleases("18.3.0", "18.3.1")}
+ fetcher.reset()
+
+ progress := refreshOnce(t, m)
+ if urls := fetcher.fetched(); progress.Completed != 1 || progress.Failed != 1 || len(fetchedMatching(urls, "react-18.3.1.tgz")) != 1 {
+ t.Errorf("progress = %+v, fetched %v; want react refreshed and lodash failed", progress, urls)
+ }
+}
+
+func TestRefreshStopsWhenCancelled(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "tarball")
+ cacheNPM(t, m, "lodash", "4.17.20")
+ m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21")}
+ fetcher.reset()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: time.Hour})
+ if err != nil {
+ t.Fatalf("Refresh: %v", err)
+ }
+ if progress.Completed != 0 || len(fetcher.fetched()) != 0 {
+ t.Errorf("progress = %+v, fetched %v; want nothing after cancelling", progress, fetcher.fetched())
+ }
+}
+
+// TestRefreshDebianArchitectureAll checks that a package for every
+// architecture, listed in each binary-* index rather than one of its own,
+// is refreshed.
+func TestRefreshDebianArchitectureAll(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ packages := func(version string) []byte {
+ return gzipped(t, "Package: ca-certificates\nVersion: "+version+"\nArchitecture: all\nFilename: pool/main/c/ca-certificates/ca-certificates_"+version+"_all.deb\n\n")
+ }
+ idx.set("/debian/dists/bookworm/InRelease", []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n"))
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("20230311"))
+ pool := idx.URL + "/debian/pool/main/c/ca-certificates/"
+ if _, _, err := m.proxy.FetchOrCacheMetadata(t.Context(), "debian", "key:bookworm", idx.URL+"/debian/dists/bookworm/InRelease", "*/*"); err != nil {
+ t.Fatal(err)
+ }
+ result, err := m.proxy.GetOrFetchArtifactFromURL(t.Context(), "deb", "ca-certificates", "20230311", "ca-certificates_20230311_all.deb", pool+"ca-certificates_20230311_all.deb")
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = result.Reader.Close()
+
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("20230311+deb12u1"))
+ fetcher.reset()
+ progress := refreshOnce(t, m)
+ if want := []string{pool + "ca-certificates_20230311+deb12u1_all.deb"}; !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 {
+ t.Errorf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want)
+ }
+}
+
+// TestRefreshSkipsIndexPathsOutsideTheRepository checks that an index entry
+// whose path leaves the repository base is not fetched.
+func TestRefreshSkipsIndexPathsOutsideTheRepository(t *testing.T) {
+ for _, filename := range []string{
+ "../other/pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb",
+ "pool/main/../../../other/curl_7.88.1-10+deb12u8_amd64.deb",
+ "/other/curl_7.88.1-10+deb12u8_amd64.deb",
+ } {
+ t.Run(filename, func(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n")
+ pool := idx.URL + "/debian/pool/main/c/curl/"
+ idx.set("/debian/dists/bookworm/InRelease", release)
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10"))
+ clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease",
+ "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", pool+"curl_7.88.1-10_amd64.deb")
+
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", gzipped(t,
+ "Package: curl\nVersion: 7.88.1-10+deb12u8\nArchitecture: amd64\nFilename: "+filename+"\n\n"))
+ fetcher.reset()
+
+ if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 {
+ t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched())
+ }
+ })
+ }
+}
+
+// TestRefreshRefusesIndexesOverTheSizeLimit checks that an index that
+// decompresses to more than maxIndexBytes fails instead of being parsed in
+// full.
+func TestRefreshRefusesIndexesOverTheSizeLimit(t *testing.T) {
+ limit := maxIndexBytes
+ maxIndexBytes = 4 << 10
+ t.Cleanup(func() { maxIndexBytes = limit })
+
+ m, fetcher, idx := setupIndexRefresh(t)
+ idx.set("/debian/dists/bookworm/InRelease", []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n"))
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10"))
+ clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease",
+ "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", idx.URL+"/debian/pool/main/c/curl/curl_7.88.1-10_amd64.deb")
+
+ padding := strings.Repeat("Package: filler\nVersion: 1\nArchitecture: amd64\nFilename: pool/main/f/filler/filler_1_amd64.deb\n\n", 100)
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", gzipped(t,
+ "Package: curl\nVersion: 7.88.1-10+deb12u8\nArchitecture: amd64\nFilename: pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb\n\n"+padding))
+ fetcher.reset()
+
+ progress := refreshOnce(t, m)
+ if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "larger than") || len(fetcher.fetched()) != 0 {
+ t.Errorf("progress = %+v, fetched %v; want the index failing for its size", progress, fetcher.fetched())
+ }
+}
+
+// TestRefreshRefusesAPKIndexesOverTheSizeLimit checks that the limit also
+// covers APKINDEX.tar.gz, which comes from the metadata cache rather than
+// readIndex.
+func TestRefreshRefusesAPKIndexesOverTheSizeLimit(t *testing.T) {
+ limit := maxIndexBytes
+ maxIndexBytes = 4 << 10
+ t.Cleanup(func() { maxIndexBytes = limit })
+
+ m, fetcher, idx := setupIndexRefresh(t)
+ dir := "/alpine/v3.20/main/aarch64"
+ idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.0-r0\nA:aarch64\n\n"))
+ clientFetchesCurl(t, m, "alpine", idx.URL+dir+"/APKINDEX.tar.gz",
+ "alpine", "8.9.0-r0", "edge/v3.20/main/aarch64/curl-8.9.0-r0.apk", idx.URL+dir+"/curl-8.9.0-r0.apk")
+
+ padding := strings.Repeat("P:filler\nV:1-r0\nA:aarch64\n\n", 400)
+ idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.1-r0\nA:aarch64\n\n"+padding))
+ fetcher.reset()
+
+ progress := refreshOnce(t, m)
+ if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "larger than") || len(fetcher.fetched()) != 0 {
+ t.Errorf("progress = %+v, fetched %v; want the index failing for its size", progress, fetcher.fetched())
+ }
+}
diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go
new file mode 100644
index 00000000..3a7ad7c4
--- /dev/null
+++ b/internal/mirror/refresh_index.go
@@ -0,0 +1,308 @@
+package mirror
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/git-pkgs/proxy/internal/database"
+ "github.com/git-pkgs/proxy/internal/handler"
+ "github.com/git-pkgs/vers"
+)
+
+// A Debian, Alpine or RPM client finds new versions in a repository index
+// rather than in metadata per package. A refresh reads the indexes clients
+// fetched again, from the URLs the metadata cache recorded, and mirrors the
+// newer files they list of the packages downloaded most.
+
+// osIndex describes the repository index of one ecosystem.
+type osIndex struct {
+ // ecosystem is what package files are cached under, metaEcosystem what
+ // the index documents are cached under.
+ ecosystem, metaEcosystem string
+ // scheme orders versions.
+ scheme string
+ // base returns the repository base package paths are relative to, for
+ // the URL of an index document a refresh starts from, or false for
+ // other documents.
+ base func(indexURL string) (string, bool)
+ // files lists the package files of the index document body fetched
+ // from indexURL. cached are the files of the most used packages under
+ // the repository base, for indexes that list their files in parts.
+ files func(ctx context.Context, m *Mirror, indexURL string, body []byte, cached []cachedOSFile) ([]indexFile, error)
+}
+
+var osIndexes = map[string]osIndex{
+ "alpine": {
+ ecosystem: "alpine", metaEcosystem: "alpine", scheme: "alpine",
+ base: func(u string) (string, bool) {
+ return strings.CutSuffix(u, "/APKINDEX.tar.gz")
+ },
+ files: func(_ context.Context, _ *Mirror, indexURL string, body []byte, _ []cachedOSFile) ([]indexFile, error) {
+ return parseAPKIndex(bytes.NewReader(body), indexURL)
+ },
+ },
+ "deb": {
+ ecosystem: "deb", metaEcosystem: "debian", scheme: "deb",
+ base: func(u string) (string, bool) {
+ if !strings.HasSuffix(u, "/InRelease") && !strings.HasSuffix(u, "/Release") {
+ return "", false
+ }
+ base, _, ok := strings.Cut(u, "/dists/")
+ return base, ok
+ },
+ files: debIndexFiles,
+ },
+ "rpm": {
+ ecosystem: "rpm", metaEcosystem: "rpm", scheme: "rpm",
+ base: func(u string) (string, bool) {
+ return strings.CutSuffix(u, "/repodata/repomd.xml")
+ },
+ files: rpmIndexFiles,
+ },
+}
+
+// cachedOSFile is a cached file of a most used package, by its path below a
+// repository base.
+type cachedOSFile struct {
+ file database.UsedFile
+ rel string
+ name, version, arch string
+}
+
+func (f cachedOSFile) key() string { return f.name + "\x00" + f.arch }
+
+// indexRefreshItems returns the newer files the recently fetched indexes of
+// an ecosystem list for packages, and an error per index it couldn't read.
+func (m *Mirror) indexRefreshItems(ctx context.Context, ix osIndex, packages []database.UsedPackage, cutoff time.Time) ([]PackageVersion, []error) {
+ docs, err := m.db.RecentMetadataDocuments(ix.metaEcosystem, cutoff)
+ if err != nil {
+ return nil, []error{fmt.Errorf("listing %s indexes: %w", ix.metaEcosystem, err)}
+ }
+ var items []PackageVersion
+ var errs []error
+ seenIndex := map[string]bool{}
+ seenFile := map[string]bool{}
+ for _, doc := range docs {
+ base, ok := ix.base(doc.UpstreamURL)
+ if !ok || seenIndex[urlDir(doc.UpstreamURL)] {
+ continue
+ }
+ cached := cachedUnder(ix.ecosystem, base, packages)
+ if len(cached) == 0 {
+ continue
+ }
+ seenIndex[urlDir(doc.UpstreamURL)] = true
+ body, _, err := m.proxy.FetchOrCacheMetadata(ctx, ix.metaEcosystem, doc.Name, doc.UpstreamURL, "*/*")
+ if err != nil {
+ errs = append(errs, fmt.Errorf("%s: %w", doc.UpstreamURL, err))
+ continue
+ }
+ files, err := ix.files(ctx, m, doc.UpstreamURL, body, cached)
+ if err != nil {
+ errs = append(errs, fmt.Errorf("%s: %w", doc.UpstreamURL, err))
+ continue
+ }
+ for _, item := range m.newerIndexFiles(ix, base, cached, files) {
+ if !seenFile[item.fileURL] {
+ seenFile[item.fileURL] = true
+ items = append(items, item)
+ }
+ }
+ }
+ return items, errs
+}
+
+// cachedUnder returns the cached files of packages below base.
+func cachedUnder(ecosystem, base string, packages []database.UsedPackage) []cachedOSFile {
+ var cached []cachedOSFile
+ for _, pkg := range packages {
+ for _, f := range pkg.Files {
+ rel, ok := strings.CutPrefix(f.UpstreamURL, base+"/")
+ if !ok {
+ continue
+ }
+ name, version, arch := handler.OSPackageIdentity(ecosystem, rel)
+ if name != "" {
+ cached = append(cached, cachedOSFile{file: f, rel: rel, name: name, version: version, arch: arch})
+ }
+ }
+ }
+ return cached
+}
+
+// newerIndexFiles returns, per package and architecture cached below base,
+// the newest file the index lists when it isn't cached and an older version
+// is. Suites sharing a pool are told apart that way: a newer version cached
+// from another suite doesn't hide an update to this one, and a package only
+// cached in a newer version than this suite has isn't mirrored for it.
+func (m *Mirror) newerIndexFiles(ix osIndex, base string, cached []cachedOSFile, files []indexFile) []PackageVersion {
+ cachedByKey := map[string][]cachedOSFile{}
+ for _, c := range cached {
+ cachedByKey[c.key()] = append(cachedByKey[c.key()], c)
+ }
+ type candidate struct {
+ file indexFile
+ name, version string
+ }
+ newest := map[string]candidate{}
+ var order []string
+ for _, f := range files {
+ if !confinedIndexPath(f.Path) {
+ continue
+ }
+ name, version, arch := handler.OSPackageIdentity(ix.ecosystem, f.Path)
+ key := name + "\x00" + arch
+ if name == "" || len(cachedByKey[key]) == 0 {
+ continue
+ }
+ if m.proxy != nil && !m.proxy.ReleaseAllowed(ix.ecosystem, name, version, time.Time{}) {
+ continue
+ }
+ current, seen := newest[key]
+ if !seen {
+ order = append(order, key)
+ }
+ if !seen || vers.CompareWithScheme(version, current.version, ix.scheme) > 0 {
+ newest[key] = candidate{file: f, name: name, version: version}
+ }
+ }
+
+ items := make([]PackageVersion, 0, len(order))
+ for _, key := range order {
+ n := newest[key]
+ var older *cachedOSFile
+ for i, c := range cachedByKey[key] {
+ cmp := vers.CompareWithScheme(n.version, c.version, ix.scheme)
+ if cmp == 0 {
+ older = nil
+ break
+ }
+ if cmp > 0 && older == nil {
+ older = &cachedByKey[key][i]
+ }
+ }
+ if older == nil {
+ continue
+ }
+ cacheFilename := path.Base(n.file.Path)
+ if dir := path.Dir(older.file.Filename); dir != "." {
+ cacheFilename = dir + "/" + cacheFilename
+ }
+ items = append(items, PackageVersion{
+ Ecosystem: ix.ecosystem, Name: n.name, Version: n.version,
+ fileURL: base + "/" + n.file.Path, cacheFilename: cacheFilename, digest: n.file.Digest,
+ })
+ }
+ return items
+}
+
+// confinedIndexPath reports whether an index entry's path is relative and
+// stays below the repository base.
+func confinedIndexPath(p string) bool {
+ return p != "" && !path.IsAbs(p) && path.Clean(p) == p && p != ".." && !strings.HasPrefix(p, "../")
+}
+
+// debIndexFiles reads the Packages indexes an InRelease or Release file
+// lists for the components and architectures of the cached files.
+func debIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, cached []cachedOSFile) ([]indexFile, error) {
+ indexes := debPackagesIndexes(body)
+ wanted := map[string]bool{}
+ for _, c := range cached {
+ rest, ok := strings.CutPrefix(c.rel, "pool/")
+ if !ok {
+ continue
+ }
+ component, _, _ := strings.Cut(rest, "/")
+ if dir := component + "/binary-" + c.arch; indexes[dir] != "" {
+ wanted[dir] = true
+ continue
+ }
+ // Files for every architecture are listed in each binary-* index.
+ for dir := range indexes {
+ if strings.HasPrefix(dir, component+"/binary-") {
+ wanted[dir] = true
+ break
+ }
+ }
+ }
+ suite := urlDir(indexURL)
+ var files []indexFile
+ for dir := range wanted {
+ listed, err := m.readIndex(ctx, suite+"/"+indexes[dir], parseDebPackages)
+ if err != nil {
+ return nil, err
+ }
+ files = append(files, listed...)
+ }
+ return files, nil
+}
+
+// rpmIndexFiles reads the primary metadata a repomd.xml points at.
+func rpmIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, _ []cachedOSFile) ([]indexFile, error) {
+ primary, err := repomdPrimary(body)
+ if err != nil {
+ return nil, err
+ }
+ base := strings.TrimSuffix(indexURL, "/repodata/repomd.xml")
+ return m.readIndex(ctx, base+"/"+primary, parseRPMPrimary)
+}
+
+// readIndex streams an index from upstream through parse, decompressed as
+// its name says and limited to maxIndexBytes. It is read directly rather than through the metadata
+// cache, since it can be far larger than a metadata document may be.
+func (m *Mirror) readIndex(ctx context.Context, indexURL string, parse func(io.Reader) ([]indexFile, error)) ([]indexFile, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := m.proxy.HTTPClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer func() { _ = resp.Body.Close() }()
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("%s: upstream returned %d", indexURL, resp.StatusCode)
+ }
+ r, err := decompressByName(indexURL, resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ defer func() { _ = r.Close() }()
+ return parse(&indexLimitReader{r: r, url: indexURL, remaining: maxIndexBytes})
+}
+
+// maxIndexBytes bounds an index once decompressed. Debian's largest
+// Packages files are under 100 MB and Fedora's primary.xml under 200 MB.
+var maxIndexBytes int64 = 512 << 20
+
+// indexLimitReader fails once more than maxIndexBytes have been read, so an
+// oversized index fails rather than looking complete.
+type indexLimitReader struct {
+ r io.Reader
+ url string
+ remaining int64
+}
+
+func (l *indexLimitReader) Read(p []byte) (int, error) {
+ if int64(len(p)) > l.remaining+1 {
+ p = p[:l.remaining+1]
+ }
+ n, err := l.r.Read(p)
+ if int64(n) > l.remaining {
+ return int(l.remaining), fmt.Errorf("%s: index larger than %d bytes", l.url, maxIndexBytes)
+ }
+ l.remaining -= int64(n)
+ return n, err
+}
+
+// urlDir returns a URL without its last path segment. path.Dir would fold
+// the slashes after the scheme.
+func urlDir(u string) string {
+ return u[:strings.LastIndex(u, "/")]
+}
diff --git a/internal/mirror/refresh_index_limit_test.go b/internal/mirror/refresh_index_limit_test.go
new file mode 100644
index 00000000..7512fc5d
--- /dev/null
+++ b/internal/mirror/refresh_index_limit_test.go
@@ -0,0 +1,20 @@
+package mirror
+
+import (
+ "io"
+ "strings"
+ "testing"
+)
+
+func TestIndexLimitReaderAllowsExactlyTheLimit(t *testing.T) {
+ limit := maxIndexBytes
+ maxIndexBytes = 8
+ t.Cleanup(func() { maxIndexBytes = limit })
+
+ if _, err := io.ReadAll(&indexLimitReader{r: strings.NewReader("12345678"), remaining: maxIndexBytes}); err != nil {
+ t.Errorf("index of exactly the limit: %v", err)
+ }
+ if _, err := io.ReadAll(&indexLimitReader{r: strings.NewReader("123456789"), remaining: maxIndexBytes}); err == nil {
+ t.Error("index one byte over the limit read without an error")
+ }
+}
diff --git a/internal/mirror/refresh_index_parse.go b/internal/mirror/refresh_index_parse.go
new file mode 100644
index 00000000..9b0cd272
--- /dev/null
+++ b/internal/mirror/refresh_index_parse.go
@@ -0,0 +1,239 @@
+package mirror
+
+import (
+ "archive/tar"
+ "bufio"
+ "compress/bzip2"
+ "compress/gzip"
+ "encoding/xml"
+ "errors"
+ "fmt"
+ "io"
+ "strings"
+
+ "github.com/klauspost/compress/zstd"
+ "github.com/ulikunitz/xz"
+)
+
+// indexFile is a package file a repository index lists: its path below the
+// repository base and, where the index gives one, its SHA-256 digest.
+type indexFile struct {
+ Path string
+ Digest string
+}
+
+// maxIndexLine bounds one line of a Debian or Alpine index.
+const maxIndexLine = 1 << 20
+
+// parseAPKIndex reads the files an APKINDEX.tar.gz lists, limited to
+// maxIndexBytes once decompressed. Its checksums cover only the control
+// section of a package, so no digest is returned.
+func parseAPKIndex(r io.Reader, indexURL string) ([]indexFile, error) {
+ gz, err := gzip.NewReader(r)
+ if err != nil {
+ return nil, err
+ }
+ defer func() { _ = gz.Close() }()
+ // An APKINDEX.tar.gz may be several gzip streams: the signature, then the
+ // index. Reading on across them finds the index whichever comes first.
+ tr := tar.NewReader(&indexLimitReader{r: gz, url: indexURL, remaining: maxIndexBytes})
+ for {
+ hdr, err := tr.Next()
+ if errors.Is(err, io.EOF) {
+ return nil, errors.New("APKINDEX.tar.gz has no APKINDEX")
+ }
+ if err != nil {
+ return nil, err
+ }
+ if hdr.Name == "APKINDEX" {
+ return parseAPKIndexText(tr)
+ }
+ }
+}
+
+func parseAPKIndexText(r io.Reader) ([]indexFile, error) {
+ var files []indexFile
+ var name, version string
+ flush := func() {
+ if name != "" && version != "" {
+ files = append(files, indexFile{Path: name + "-" + version + ".apk"})
+ }
+ name, version = "", ""
+ }
+ scanner := bufio.NewScanner(r)
+ scanner.Buffer(nil, maxIndexLine)
+ for scanner.Scan() {
+ line := scanner.Text()
+ switch {
+ case line == "":
+ flush()
+ case strings.HasPrefix(line, "P:"):
+ name = line[2:]
+ case strings.HasPrefix(line, "V:"):
+ version = line[2:]
+ }
+ }
+ flush()
+ return files, scanner.Err()
+}
+
+// debIndexPreference ranks the compressions of a Packages index, best first.
+var debIndexPreference = []string{".xz", ".gz", ".bz2", ""}
+
+// debPackagesIndexes returns, per component/binary-arch directory, the path
+// of the Packages index a Release or InRelease file lists, in the best
+// compression it offers.
+func debPackagesIndexes(release []byte) map[string]string {
+ listed := map[string]bool{}
+ inSHA256 := false
+ for _, line := range strings.Split(string(release), "\n") {
+ line = strings.TrimRight(line, "\r")
+ if !strings.HasPrefix(line, " ") {
+ inSHA256 = line == "SHA256:"
+ continue
+ }
+ if !inSHA256 {
+ continue
+ }
+ fields := strings.Fields(line)
+ if len(fields) == 3 { //nolint:mnd // checksum, size, path
+ listed[fields[2]] = true
+ }
+ }
+ indexes := map[string]string{}
+ for path := range listed {
+ dir, file, ok := strings.Cut(path, "/Packages")
+ if !ok || !strings.Contains(dir, "/binary-") || strings.Contains(file, "/") {
+ continue
+ }
+ for _, ext := range debIndexPreference {
+ if listed[dir+"/Packages"+ext] {
+ indexes[dir] = dir + "/Packages" + ext
+ break
+ }
+ }
+ }
+ return indexes
+}
+
+// parseDebPackages reads the files a Packages index lists.
+func parseDebPackages(r io.Reader) ([]indexFile, error) {
+ var files []indexFile
+ var current indexFile
+ flush := func() {
+ if current.Path != "" {
+ files = append(files, current)
+ }
+ current = indexFile{}
+ }
+ scanner := bufio.NewScanner(r)
+ scanner.Buffer(nil, maxIndexLine)
+ for scanner.Scan() {
+ line := scanner.Text()
+ if line == "" {
+ flush()
+ continue
+ }
+ key, value, ok := strings.Cut(line, ": ")
+ if !ok || strings.HasPrefix(line, " ") {
+ continue
+ }
+ switch key {
+ case "Filename":
+ current.Path = value
+ case "SHA256":
+ current.Digest = "sha256:" + value
+ }
+ }
+ flush()
+ return files, scanner.Err()
+}
+
+// repomdPrimary returns the location of the primary metadata a repomd.xml
+// points at.
+func repomdPrimary(body []byte) (string, error) {
+ var doc struct {
+ Data []struct {
+ Type string `xml:"type,attr"`
+ Location struct {
+ Href string `xml:"href,attr"`
+ } `xml:"location"`
+ } `xml:"data"`
+ }
+ if err := xml.Unmarshal(body, &doc); err != nil {
+ return "", err
+ }
+ for _, d := range doc.Data {
+ if d.Type == "primary" && d.Location.Href != "" {
+ return d.Location.Href, nil
+ }
+ }
+ return "", errors.New("repomd.xml lists no primary metadata")
+}
+
+// parseRPMPrimary reads the binary packages a primary.xml lists, one at a
+// time. Source packages are left out.
+func parseRPMPrimary(r io.Reader) ([]indexFile, error) {
+ dec := xml.NewDecoder(r)
+ var files []indexFile
+ for {
+ tok, err := dec.Token()
+ if errors.Is(err, io.EOF) {
+ return files, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ start, ok := tok.(xml.StartElement)
+ if !ok || start.Name.Local != "package" {
+ continue
+ }
+ var pkg struct {
+ Arch string `xml:"arch"`
+ Checksum struct {
+ Type string `xml:"type,attr"`
+ Value string `xml:",chardata"`
+ } `xml:"checksum"`
+ Location struct {
+ Href string `xml:"href,attr"`
+ } `xml:"location"`
+ }
+ if err := dec.DecodeElement(&pkg, &start); err != nil {
+ return nil, err
+ }
+ if pkg.Arch == "src" || pkg.Location.Href == "" {
+ continue
+ }
+ file := indexFile{Path: pkg.Location.Href}
+ if pkg.Checksum.Type == "sha256" {
+ file.Digest = "sha256:" + strings.TrimSpace(pkg.Checksum.Value)
+ }
+ files = append(files, file)
+ }
+}
+
+// decompressByName returns r decompressed as the extension of name says.
+func decompressByName(name string, r io.Reader) (io.ReadCloser, error) {
+ switch {
+ case strings.HasSuffix(name, ".gz"):
+ return gzip.NewReader(r)
+ case strings.HasSuffix(name, ".xz"):
+ x, err := xz.NewReader(r)
+ if err != nil {
+ return nil, err
+ }
+ return io.NopCloser(x), nil
+ case strings.HasSuffix(name, ".zst"):
+ z, err := zstd.NewReader(r)
+ if err != nil {
+ return nil, err
+ }
+ return z.IOReadCloser(), nil
+ case strings.HasSuffix(name, ".bz2"):
+ return io.NopCloser(bzip2.NewReader(r)), nil
+ case strings.HasSuffix(name, ".xml"), !strings.Contains(name[strings.LastIndex(name, "/")+1:], "."):
+ return io.NopCloser(r), nil
+ default:
+ return nil, fmt.Errorf("unsupported compression: %s", name)
+ }
+}
diff --git a/internal/mirror/refresh_index_parse_test.go b/internal/mirror/refresh_index_parse_test.go
new file mode 100644
index 00000000..934256e2
--- /dev/null
+++ b/internal/mirror/refresh_index_parse_test.go
@@ -0,0 +1,188 @@
+package mirror
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/klauspost/compress/zstd"
+)
+
+// apkIndexArchive packs an APKINDEX into the tar.gz Alpine serves.
+func apkIndexArchive(t *testing.T, index string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ tw := tar.NewWriter(gz)
+ for name, body := range map[string]string{"DESCRIPTION": "v3.20.3-1-g0000000", "APKINDEX": index} {
+ if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := tw.Write([]byte(body)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := tw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if err := gz.Close(); err != nil {
+ t.Fatal(err)
+ }
+ return buf.Bytes()
+}
+
+func TestParseAPKIndex(t *testing.T) {
+ index := "C:Q1abc=\nP:curl\nV:8.9.1-r0\nA:aarch64\nS:100\n\nC:Q1def=\nP:busybox\nV:1.36.1-r29\nA:aarch64\n\n"
+ got, err := parseAPKIndex(bytes.NewReader(apkIndexArchive(t, index)), "APKINDEX.tar.gz")
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []indexFile{{Path: "curl-8.9.1-r0.apk"}, {Path: "busybox-1.36.1-r29.apk"}}
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("parseAPKIndex = %+v, want %+v", got, want)
+ }
+}
+
+const debRelease = `-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA256
+
+Origin: Debian
+Suite: stable
+Codename: bookworm
+Architectures: all amd64 arm64
+Components: main contrib
+MD5Sum:
+ 0123 100 main/binary-amd64/Packages.xz
+SHA256:
+ aaaa 1000 main/binary-amd64/Packages
+ bbbb 200 main/binary-amd64/Packages.xz
+ cccc 300 main/binary-amd64/Packages.gz
+ dddd 400 main/binary-arm64/Packages.gz
+ eeee 500 contrib/binary-amd64/Packages.xz
+ ffff 600 main/i18n/Translation-en.bz2
+-----BEGIN PGP SIGNATURE-----
+xyz
+-----END PGP SIGNATURE-----
+`
+
+func TestDebPackagesIndexes(t *testing.T) {
+ got := debPackagesIndexes([]byte(debRelease))
+ want := map[string]string{
+ "main/binary-amd64": "main/binary-amd64/Packages.xz",
+ "main/binary-arm64": "main/binary-arm64/Packages.gz",
+ "contrib/binary-amd64": "contrib/binary-amd64/Packages.xz",
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("debPackagesIndexes = %v, want %v", got, want)
+ }
+}
+
+func TestParseDebPackages(t *testing.T) {
+ packages := `Package: curl
+Version: 7.88.1-10+deb12u8
+Architecture: amd64
+Description: a tool
+ with a long description
+Filename: pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb
+SHA256: 1111
+
+Package: zlib1g
+Version: 1:1.2.13.dfsg-1
+Architecture: amd64
+Filename: pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb
+SHA256: 2222
+`
+ got, err := parseDebPackages(strings.NewReader(packages))
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []indexFile{
+ {Path: "pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb", Digest: "sha256:1111"},
+ {Path: "pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb", Digest: "sha256:2222"},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("parseDebPackages = %+v, want %+v", got, want)
+ }
+}
+
+const repomd = `
+
+ 1700000000
+
+
+
+`
+
+func TestRepomdPrimary(t *testing.T) {
+ got, err := repomdPrimary([]byte(repomd))
+ if err != nil || got != "repodata/bbbb-primary.xml.zst" {
+ t.Errorf("repomdPrimary = %q, %v", got, err)
+ }
+ if _, err := repomdPrimary([]byte(``)); err == nil {
+ t.Error("repomdPrimary accepted a repomd without primary")
+ }
+}
+
+const rpmPrimary = `
+
+
+ curl
+ x86_64
+
+ 3333
+
+
+
+ curl
+ src
+
+ 4444
+
+
+
+ tzdata
+ noarch
+
+ 5555
+
+
+
+`
+
+func TestParseRPMPrimary(t *testing.T) {
+ got, err := parseRPMPrimary(strings.NewReader(rpmPrimary))
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []indexFile{
+ {Path: "Packages/c/curl-8.6.0-10.fc40.x86_64.rpm", Digest: "sha256:3333"},
+ {Path: "Packages/t/tzdata-2024a-5.fc40.noarch.rpm"},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("parseRPMPrimary = %+v, want %+v", got, want)
+ }
+}
+
+func TestDecompressByName(t *testing.T) {
+ var gz bytes.Buffer
+ w := gzip.NewWriter(&gz)
+ _, _ = w.Write([]byte("hello"))
+ _ = w.Close()
+ enc, _ := zstd.NewWriter(nil)
+ zst := enc.EncodeAll([]byte("hello"), nil)
+
+ for name, data := range map[string][]byte{"Packages": []byte("hello"), "Packages.gz": gz.Bytes(), "primary.xml.zst": zst} {
+ r, err := decompressByName(name, bytes.NewReader(data))
+ if err != nil {
+ t.Fatalf("%s: %v", name, err)
+ }
+ var out bytes.Buffer
+ if _, err := out.ReadFrom(r); err != nil || out.String() != "hello" {
+ t.Errorf("%s: got %q, %v", name, out.String(), err)
+ }
+ _ = r.Close()
+ }
+}
diff --git a/internal/mirror/refresh_index_test.go b/internal/mirror/refresh_index_test.go
new file mode 100644
index 00000000..1f66c388
--- /dev/null
+++ b/internal/mirror/refresh_index_test.go
@@ -0,0 +1,231 @@
+package mirror
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "slices"
+ "sync"
+ "testing"
+ "time"
+)
+
+// indexServer serves repository index documents that a test can change.
+type indexServer struct {
+ *httptest.Server
+ mu sync.Mutex
+ docs map[string][]byte
+}
+
+func newIndexServer(t *testing.T) *indexServer {
+ t.Helper()
+ s := &indexServer{docs: map[string][]byte{}}
+ s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ s.mu.Lock()
+ body, ok := s.docs[r.URL.Path]
+ s.mu.Unlock()
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+ _, _ = w.Write(body)
+ }))
+ t.Cleanup(s.Close)
+ return s
+}
+
+func (s *indexServer) set(path string, body []byte) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.docs[path] = body
+}
+
+func gzipped(t *testing.T, s string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ w := gzip.NewWriter(&buf)
+ if _, err := w.Write([]byte(s)); err != nil {
+ t.Fatal(err)
+ }
+ if err := w.Close(); err != nil {
+ t.Fatal(err)
+ }
+ return buf.Bytes()
+}
+
+// setupIndexRefresh returns a mirror whose package files come from a
+// servingFetcher and whose indexes come from an indexServer.
+func setupIndexRefresh(t *testing.T) (*Mirror, *servingFetcher, *indexServer) {
+ t.Helper()
+ m, fetcher := setupRefreshMirror(t, "package")
+ idx := newIndexServer(t)
+ m.proxy.HTTPClient = idx.Client()
+ m.proxy.CacheMetadata = true
+ return m, fetcher, idx
+}
+
+// clientFetchesCurl caches an index document and a curl package file the
+// way the handlers do for a client.
+func clientFetchesCurl(t *testing.T, m *Mirror, metaEcosystem, indexURL, ecosystem, version, cacheFilename, fileURL string) {
+ t.Helper()
+ ctx := context.Background()
+ if _, _, err := m.proxy.FetchOrCacheMetadata(ctx, metaEcosystem, "key:"+indexURL, indexURL, "*/*"); err != nil {
+ t.Fatalf("fetching %s: %v", indexURL, err)
+ }
+ result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, ecosystem, "curl", version, cacheFilename, fileURL)
+ if err != nil {
+ t.Fatalf("fetching %s: %v", fileURL, err)
+ }
+ _ = result.Reader.Close()
+}
+
+func refreshOnce(t *testing.T, m *Mirror) *Progress {
+ t.Helper()
+ progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: time.Hour})
+ if err != nil {
+ t.Fatalf("Refresh: %v", err)
+ }
+ return progress
+}
+
+func cachedNames(t *testing.T, m *Mirror, ecosystem, filename string) []string {
+ t.Helper()
+ names, err := m.db.CachedArtifactNames(ecosystem, filename)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return names
+}
+
+func TestRefreshAlpineIndex(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ dir := "/alpine/v3.20/main/aarch64"
+ idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.0-r0\nA:aarch64\n\nP:busybox\nV:1.36.1-r29\n\n"))
+ clientFetchesCurl(t, m, "alpine", idx.URL+dir+"/APKINDEX.tar.gz",
+ "alpine", "8.9.0-r0", "edge/v3.20/main/aarch64/curl-8.9.0-r0.apk", idx.URL+dir+"/curl-8.9.0-r0.apk")
+
+ // A new release of curl; busybox was never downloaded and stays out.
+ idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.1-r0\nA:aarch64\n\nP:busybox\nV:1.36.1-r30\n\n"))
+ fetcher.reset()
+ progress := refreshOnce(t, m)
+
+ if want := []string{idx.URL + dir + "/curl-8.9.1-r0.apk"}; !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 {
+ t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want)
+ }
+ // Cached where the APK handler looks for it.
+ if names := cachedNames(t, m, "alpine", "edge/v3.20/main/aarch64/curl-8.9.1-r0.apk"); !slices.Equal(names, []string{"curl"}) {
+ t.Errorf("cached names = %v, want [curl]", names)
+ }
+
+ fetcher.reset()
+ if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 {
+ t.Errorf("second run: progress = %+v, fetched %v; want nothing", progress, fetcher.fetched())
+ }
+}
+
+func TestRefreshIndexNeverDowngrades(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ dir := "/alpine/v3.20/main/aarch64"
+ idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.1-r0\n\n"))
+ clientFetchesCurl(t, m, "alpine", idx.URL+dir+"/APKINDEX.tar.gz",
+ "alpine", "8.9.1-r0", "edge/v3.20/main/aarch64/curl-8.9.1-r0.apk", idx.URL+dir+"/curl-8.9.1-r0.apk")
+ idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.0-r0\n\n"))
+ fetcher.reset()
+
+ if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 {
+ t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched())
+ }
+}
+
+func TestRefreshDebianIndex(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ release := []byte("Suite: stable\nSHA256:\n 0000 1 main/binary-amd64/Packages.gz\n 0000 1 main/binary-arm64/Packages.gz\n")
+ packages := func(curl string) []byte {
+ return gzipped(t, "Package: curl\nVersion: "+curl+"\nArchitecture: amd64\nFilename: pool/main/c/curl/curl_"+curl+"_amd64.deb\nSHA256: "+
+ "bc4a71180870f7945155fbb02f4b0a2e3faa2a62d6d31b7039013055ed19869a\n\n"+
+ "Package: zlib1g\nVersion: 1:1.2.13.dfsg-1\nArchitecture: amd64\nFilename: pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb\n\n")
+ }
+ idx.set("/debian/dists/bookworm/InRelease", release)
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("7.88.1-10"))
+ clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease",
+ "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", idx.URL+"/debian/pool/main/c/curl/curl_7.88.1-10_amd64.deb")
+
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("7.88.1-10+deb12u8"))
+ fetcher.reset()
+ progress := refreshOnce(t, m)
+
+ // Only the amd64 index is read: nothing was downloaded for arm64. The
+ // file is checked against the index's SHA-256 of "package".
+ want := []string{idx.URL + "/debian/pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb"}
+ if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 || progress.Failed != 0 {
+ t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want)
+ }
+ if names := cachedNames(t, m, "deb", "curl_7.88.1-10+deb12u8_amd64.deb"); !slices.Equal(names, []string{"curl"}) {
+ t.Errorf("cached names = %v, want [curl]", names)
+ }
+}
+
+func TestRefreshDebianIndexChecksDigest(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ idx.set("/debian/dists/bookworm/InRelease", []byte("SHA256:\n 0000 1 main/binary-amd64/Packages\n"))
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages", []byte("Package: curl\nFilename: pool/main/c/curl/curl_7.88.1-10_amd64.deb\n\n"))
+ clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease",
+ "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", idx.URL+"/debian/pool/main/c/curl/curl_7.88.1-10_amd64.deb")
+ idx.set("/debian/dists/bookworm/main/binary-amd64/Packages", []byte(
+ "Package: curl\nFilename: pool/main/c/curl/curl_7.88.1-11_amd64.deb\nSHA256: 0000000000000000000000000000000000000000000000000000000000000000\n\n"))
+ fetcher.reset()
+
+ progress := refreshOnce(t, m)
+ if progress.Failed != 1 || progress.Completed != 0 {
+ t.Errorf("progress = %+v, want the file failing its digest", progress)
+ }
+ if names := cachedNames(t, m, "deb", "curl_7.88.1-11_amd64.deb"); len(names) != 0 {
+ t.Errorf("a file failing its digest was cached as %v", names)
+ }
+}
+
+func TestRefreshRPMIndex(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ base := "/fedora/updates/40/Everything/x86_64"
+ primary := func(rel string) []byte {
+ return gzipped(t, `curlx86_64`+
+ ``)
+ }
+ repomdFor := func(href string) []byte {
+ return []byte(``)
+ }
+ idx.set(base+"/repodata/repomd.xml", repomdFor("repodata/aaaa-primary.xml.gz"))
+ idx.set(base+"/repodata/aaaa-primary.xml.gz", primary("10"))
+ clientFetchesCurl(t, m, "rpm", idx.URL+base+"/repodata/repomd.xml",
+ "rpm", "8.6.0-10.fc40", "curl-8.6.0-10.fc40.x86_64.rpm", idx.URL+base+"/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm")
+
+ idx.set(base+"/repodata/repomd.xml", repomdFor("repodata/bbbb-primary.xml.gz"))
+ idx.set(base+"/repodata/bbbb-primary.xml.gz", primary("11"))
+ fetcher.reset()
+ progress := refreshOnce(t, m)
+
+ want := []string{idx.URL + base + "/Packages/c/curl-8.6.0-11.fc40.x86_64.rpm"}
+ if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 {
+ t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want)
+ }
+ if names := cachedNames(t, m, "rpm", "curl-8.6.0-11.fc40.x86_64.rpm"); !slices.Equal(names, []string{"curl"}) {
+ t.Errorf("cached names = %v, want [curl]", names)
+ }
+}
+
+func TestRefreshIndexWithoutRecordedURLDoesNothing(t *testing.T) {
+ m, fetcher, idx := setupIndexRefresh(t)
+ result, err := m.proxy.GetOrFetchArtifactFromURL(context.Background(), "alpine", "curl", "8.9.0-r0",
+ "edge/v3.20/main/aarch64/curl-8.9.0-r0.apk", idx.URL+"/alpine/v3.20/main/aarch64/curl-8.9.0-r0.apk")
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = result.Reader.Close()
+ fetcher.reset()
+
+ if progress := refreshOnce(t, m); progress.Completed != 0 || progress.Failed != 0 || len(fetcher.fetched()) != 0 {
+ t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched())
+ }
+}
diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go
new file mode 100644
index 00000000..97624d6d
--- /dev/null
+++ b/internal/mirror/refresh_test.go
@@ -0,0 +1,459 @@
+package mirror
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "slices"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/git-pkgs/cooldown"
+ "github.com/git-pkgs/registries"
+ "github.com/git-pkgs/registries/fetch"
+)
+
+// servingFetcher answers every fetch with body and records the URLs.
+type servingFetcher struct {
+ body string
+ mu sync.Mutex
+ urls []string
+}
+
+func (f *servingFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) {
+ return f.FetchWithHeaders(ctx, url, nil)
+}
+
+func (f *servingFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) {
+ f.mu.Lock()
+ f.urls = append(f.urls, url)
+ f.mu.Unlock()
+ return &fetch.Artifact{Body: io.NopCloser(strings.NewReader(f.body)), Size: int64(len(f.body)), ContentType: "application/octet-stream"}, nil
+}
+
+func (f *servingFetcher) Head(context.Context, string) (int64, string, error) {
+ return int64(len(f.body)), "application/octet-stream", nil
+}
+
+func (f *servingFetcher) fetched() []string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return append([]string(nil), f.urls...)
+}
+
+func (f *servingFetcher) reset() {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.urls = nil
+}
+
+// stubReleases lists releases by "ecosystem/name".
+type stubReleases map[string][]registries.Version
+
+func (s stubReleases) ListReleases(_ context.Context, ecosystem, name string) ([]registries.Version, error) {
+ r, ok := s[ecosystem+"/"+name]
+ if !ok {
+ return nil, fmt.Errorf("%w: %s/%s", registries.ErrNotFound, ecosystem, name)
+ }
+ return r, nil
+}
+
+func (s stubReleases) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) {
+ r, err := s.ListReleases(ctx, ecosystem, name)
+ numbers := make([]string, len(r))
+ for i, v := range r {
+ numbers[i] = v.Number
+ }
+ return numbers, err
+}
+
+func (s stubReleases) FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) {
+ r, err := s.ListReleases(ctx, ecosystem, name)
+ if err != nil {
+ return registries.Version{}, err
+ }
+ for _, v := range r {
+ if v.Number == version {
+ return v, nil
+ }
+ }
+ return registries.Version{}, registries.ErrNotFound
+}
+
+func oldReleases(numbers ...string) []registries.Version {
+ r := releases(numbers...)
+ for i := range r {
+ r[i].PublishedAt = time.Now().Add(-time.Duration(len(r)-i) * 30 * 24 * time.Hour)
+ }
+ return r
+}
+
+func setupRefreshMirror(t *testing.T, body string) (*Mirror, *servingFetcher) {
+ t.Helper()
+ m := setupTestMirror(t, 1)
+ fetcher := &servingFetcher{body: body}
+ m.proxy.Fetcher = fetcher
+ return m, fetcher
+}
+
+// cacheNPM downloads a version the way a client would, so it counts as used.
+func cacheNPM(t *testing.T, m *Mirror, name, version string) {
+ t.Helper()
+ result, err := m.proxy.GetOrFetchArtifact(context.Background(), "npm", name, version, "")
+ if err != nil {
+ t.Fatalf("caching %s@%s: %v", name, version, err)
+ }
+ _ = result.Reader.Close()
+}
+
+func fetchedMatching(urls []string, part string) []string {
+ var out []string
+ for _, u := range urls {
+ if strings.Contains(u, part) {
+ out = append(out, u)
+ }
+ }
+ return out
+}
+
+func TestRefreshMirrorsNewerVersionOnCachedLine(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "tarball")
+ cacheNPM(t, m, "lodash", "4.17.20")
+ m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21", "5.0.0")}
+ fetcher.reset()
+
+ progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour})
+ if err != nil {
+ t.Fatalf("Refresh: %v", err)
+ }
+ if progress.Completed != 1 || progress.Failed != 0 {
+ t.Errorf("progress = %+v, want one mirrored", progress)
+ }
+ urls := fetcher.fetched()
+ if len(fetchedMatching(urls, "lodash-4.17.21.tgz")) != 1 || len(fetchedMatching(urls, "5.0.0")) != 0 {
+ t.Errorf("fetched %v, want lodash 4.17.21 only", urls)
+ }
+
+ // The new version is cached now, so the next run has nothing to do.
+ progress, err = m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour})
+ if err != nil {
+ t.Fatalf("second Refresh: %v", err)
+ }
+ if progress.Completed != 0 || progress.Failed != 0 {
+ t.Errorf("second run progress = %+v, want nothing to do", progress)
+ }
+}
+
+func TestRefreshKeepsCooldown(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "tarball")
+ cacheNPM(t, m, "lodash", "4.17.20")
+ r := oldReleases("4.17.20", "4.17.21")
+ r[1].PublishedAt = time.Now().Add(-time.Hour)
+ m.Versions = stubReleases{"npm/lodash": r}
+ m.proxy.Cooldown = &cooldown.Config{Default: "3d"}
+ fetcher.reset()
+
+ progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour})
+ if err != nil {
+ t.Fatalf("Refresh: %v", err)
+ }
+ if progress.Completed != 0 || len(fetcher.fetched()) != 0 {
+ t.Errorf("progress = %+v, fetched %v; want a version within cooldown left alone", progress, fetcher.fetched())
+ }
+}
+
+func TestRefreshOnlyTopPackagesRecentlyUsed(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "tarball")
+ cacheNPM(t, m, "lodash", "4.17.20")
+ cacheNPM(t, m, "react", "18.3.0")
+ cacheNPM(t, m, "react", "18.3.0") // a second download: react is used most
+ m.Versions = stubReleases{
+ "npm/lodash": oldReleases("4.17.20", "4.17.21"),
+ "npm/react": oldReleases("18.3.0", "18.3.1"),
+ }
+ if err := m.db.RecordArtifactHit("pkg:npm/react@18.3.0", "react-18.3.0.tgz"); err != nil {
+ t.Fatal(err)
+ }
+ fetcher.reset()
+
+ progress, err := m.Refresh(context.Background(), Refresh{Top: 1, Since: 24 * time.Hour})
+ if err != nil {
+ t.Fatalf("Refresh: %v", err)
+ }
+ urls := fetcher.fetched()
+ if progress.Completed != 1 || len(fetchedMatching(urls, "react-18.3.1.tgz")) != 1 || len(fetchedMatching(urls, "lodash")) != 0 {
+ t.Errorf("progress = %+v, fetched %v; want only react refreshed", progress, urls)
+ }
+}
+
+func TestRefreshStopsAtMaxVersions(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "tarball")
+ cacheNPM(t, m, "lodash", "4.17.20")
+ cacheNPM(t, m, "react", "18.3.0")
+ m.Versions = stubReleases{
+ "npm/lodash": oldReleases("4.17.20", "4.17.21"),
+ "npm/react": oldReleases("18.3.0", "18.3.1"),
+ }
+ fetcher.reset()
+
+ progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour, MaxVersions: 1})
+ if err != nil {
+ t.Fatalf("Refresh: %v", err)
+ }
+ if progress.Completed != 1 || len(fetcher.fetched()) != 1 {
+ t.Errorf("progress = %+v, fetched %v; want one version", progress, fetcher.fetched())
+ }
+}
+
+func TestRefreshStopsAtMaxBytes(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "0123456789")
+ cacheNPM(t, m, "lodash", "4.17.20")
+ cacheNPM(t, m, "react", "18.3.0")
+ m.Versions = stubReleases{
+ "npm/lodash": oldReleases("4.17.20", "4.17.21"),
+ "npm/react": oldReleases("18.3.0", "18.3.1"),
+ }
+ fetcher.reset()
+
+ // The cap is checked before each download, so the first one goes over it.
+ progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour, MaxBytes: 5})
+ if err != nil {
+ t.Fatalf("Refresh: %v", err)
+ }
+ if progress.Completed != 1 || progress.Bytes != 10 {
+ t.Errorf("progress = %+v, want one version of 10 bytes", progress)
+ }
+}
+
+func TestRefreshCountsListingFailure(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "tarball")
+ cacheNPM(t, m, "lodash", "4.17.20")
+ m.Versions = stubReleases{}
+ fetcher.reset()
+
+ progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour})
+ if err != nil {
+ t.Fatalf("Refresh: %v", err)
+ }
+ if progress.Failed != 1 || len(progress.Errors) != 1 || progress.Errors[0].Name != "lodash" {
+ t.Errorf("progress = %+v, want lodash failed", progress)
+ }
+}
+
+func TestRefreshMirrorsTheFileVariantsClientsFetched(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "wheel")
+ ctx := context.Background()
+ for _, f := range []string{"demo-1.0.0-py3-none-any.whl", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.whl"} {
+ result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "pypi", "demo", "1.0.0", f, "https://files.example/"+f, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = result.Reader.Close()
+ }
+ files := func(version string, names ...string) registries.Version {
+ v := registries.Version{Number: version, PublishedAt: time.Now().Add(-30 * 24 * time.Hour)}
+ for _, n := range names {
+ v.Artifacts = append(v.Artifacts, registries.Artifact{Filename: n, URL: "https://files.example/" + n})
+ }
+ return v
+ }
+ m.Versions = stubReleases{"pypi/demo": {
+ files("1.0.0", "demo-1.0.0-py3-none-any.whl", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.whl"),
+ files("1.1.0",
+ "demo-1.1.0.tar.gz",
+ "demo-1.1.0-py3-none-any.whl",
+ "demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.whl",
+ "demo-1.1.0-cp313-cp313-manylinux_2_17_x86_64.whl",
+ "demo-1.1.0-cp312-cp312-win_amd64.whl"),
+ }}
+ fetcher.reset()
+
+ progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour})
+ if err != nil {
+ t.Fatalf("Refresh: %v", err)
+ }
+ got := fetcher.fetched()
+ slices.Sort(got)
+ want := []string{
+ "https://files.example/demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.whl",
+ "https://files.example/demo-1.1.0-py3-none-any.whl",
+ }
+ if progress.Completed != 1 || !slices.Equal(got, want) {
+ t.Errorf("progress = %+v, fetched %v; want %v", progress, got, want)
+ }
+}
+
+func TestFileVariants(t *testing.T) {
+ tests := []struct {
+ ecosystem, cacheFilename, want string
+ }{
+ {"pypi", "demo-1.0.0-py3-none-any.whl", "py3-none-any"},
+ {"pypi", "demo-1.0.0-1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "cp312-cp312-manylinux_x86_64"},
+ {"pypi", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux_2_17_aarch64.whl", "cp312-cp312-manylinux_x86_64 cp312-cp312-manylinux_aarch64"},
+ {"pypi", "demo-1.0.0-py2.py3-none-any.whl", "py2-none-any py3-none-any"},
+ {"pypi", "demo-1.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", "cp312-cp312-musllinux_x86_64"},
+ {"pypi", "demo-1.0.0-cp312-cp312-macosx_10_13_universal2.whl", "cp312-cp312-macosx_universal2"},
+ {"pypi", "demo-1.0.0-cp312-cp312-win_amd64.whl", "cp312-cp312-win_amd64"},
+ {"pypi", "demo-1.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", "cp-abi3-manylinux_x86_64"},
+ {"pypi", "demo-1.0.0.tar.gz", "sdist"},
+ {"pypi", "demo-1.0.0.zip", "sdist"},
+ {"conda", "linux-64/numpy-1.26.4-py312h8753938_0.conda", "linux-64/py312/conda"},
+ {"conda", "noarch/requests-2.32.3-pyhd8ed1ab_0.conda", "noarch/py/conda"},
+ {"conda", "osx-arm64/zlib-1.3.1-h8359307_2.tar.bz2", "osx-arm64//tar.bz2"},
+ {"conda", "linux-64/python-3.12.16-h5f976f7_0_cpython.conda", "linux-64/_cpython/conda"},
+ {"conda", "linux-64/python-3.12.16-h5f976f7_0_debug_cpython.conda", "linux-64/_debug_cpython/conda"},
+ }
+ for _, tt := range tests {
+ if got := strings.Join(fileEcosystems[tt.ecosystem].cachedVariants(tt.cacheFilename), " "); got != tt.want {
+ t.Errorf("%s variant of %q = %q, want %q", tt.ecosystem, tt.cacheFilename, got, tt.want)
+ }
+ }
+}
+
+func TestRefreshable(t *testing.T) {
+ for ecosystem, want := range map[string]bool{
+ "npm": true, "pypi": true, "conda": true, "composer": true,
+ // Their registries need the configured upstream URL.
+ "swift": true, "helm": true,
+ // Julia files are cached by tree hash, not by version.
+ "julia": false,
+ // Images follow mirror.schedules; OS packages their indexes.
+ "docker": false, "oci": false, "deb": false, "alpine": false, "rpm": false,
+ "generic": false,
+ } {
+ if got := refreshable(ecosystem); got != want {
+ t.Errorf("refreshable(%s) = %v, want %v", ecosystem, got, want)
+ }
+ }
+}
+
+// refreshFetches caches files of an old version of name the way a client
+// would, lists newFiles for newVersion and returns what a refresh fetched.
+func refreshFetches(t *testing.T, ecosystem, name, oldVersion string, cached []string, newVersion string, newFiles []registries.Artifact) []string {
+ t.Helper()
+ m, fetcher := setupRefreshMirror(t, "file")
+ ctx := context.Background()
+ old := registries.Version{Number: oldVersion, PublishedAt: time.Now().Add(-60 * 24 * time.Hour)}
+ for _, f := range cached {
+ result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, ecosystem, name, oldVersion, f, "https://files.example/"+f, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = result.Reader.Close()
+ }
+ for i := range newFiles {
+ newFiles[i].URL = "https://files.example/" + newFiles[i].Filename
+ }
+ m.Versions = stubReleases{ecosystem + "/" + name: {
+ old,
+ {Number: newVersion, PublishedAt: time.Now().Add(-30 * 24 * time.Hour), Artifacts: newFiles},
+ }}
+ fetcher.reset()
+
+ progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour})
+ if err != nil {
+ t.Fatalf("Refresh: %v", err)
+ }
+ if progress.Failed != 0 {
+ t.Errorf("progress = %+v, want no failures", progress)
+ }
+ got := fetcher.fetched()
+ slices.Sort(got)
+ return got
+}
+
+func wheels(names ...string) []registries.Artifact {
+ files := make([]registries.Artifact, 0, len(names))
+ for _, n := range names {
+ files = append(files, registries.Artifact{Filename: n})
+ }
+ return files
+}
+
+// A new wheel built for a newer glibc or macOS still replaces the cached
+// one: platform tags match on OS and architecture.
+func TestRefreshMatchesWheelsAcrossPlatformMinimums(t *testing.T) {
+ got := refreshFetches(t, "pypi", "numpy", "2.2.6",
+ []string{"numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl"},
+ "2.3.0", wheels(
+ "numpy-2.3.0-cp312-cp312-manylinux_2_28_x86_64.whl",
+ "numpy-2.3.0-cp312-cp312-manylinux_2_28_aarch64.whl",
+ "numpy-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl",
+ "numpy-2.3.0-cp313-cp313-manylinux_2_28_x86_64.whl",
+ "numpy-2.3.0-cp312-cp312-macosx_11_0_x86_64.whl",
+ "numpy-2.3.0-cp312-cp312-macosx_11_0_arm64.whl",
+ "numpy-2.3.0-cp312-cp312-win_amd64.whl",
+ ))
+ want := []string{
+ "https://files.example/numpy-2.3.0-cp312-cp312-macosx_11_0_x86_64.whl",
+ "https://files.example/numpy-2.3.0-cp312-cp312-manylinux_2_28_x86_64.whl",
+ }
+ if !slices.Equal(got, want) {
+ t.Errorf("fetched %v, want %v", got, want)
+ }
+}
+
+// conda-forge's python builds carry the hash before a suffix; the suffix
+// stays part of the kind.
+func TestRefreshMatchesCondaBuildsWithHashInside(t *testing.T) {
+ conda := func(subdir, filename, build string) registries.Artifact {
+ return registries.Artifact{Filename: filename, Metadata: map[string]any{"subdir": subdir, "build": build}}
+ }
+ got := refreshFetches(t, "conda", "conda-forge/python", "3.12.15",
+ []string{"linux-64/python-3.12.15-h5f976f7_0_cpython.conda"},
+ "3.12.16", []registries.Artifact{
+ conda("linux-64", "python-3.12.16-hab00c5b_1_cpython.conda", "hab00c5b_1_cpython"),
+ conda("linux-64", "python-3.12.16-hd12c33a_1_debug_cpython.conda", "hd12c33a_1_debug_cpython"),
+ conda("osx-arm64", "python-3.12.16-h99e199e_1_cpython.conda", "h99e199e_1_cpython"),
+ })
+ want := []string{"https://files.example/python-3.12.16-hab00c5b_1_cpython.conda"}
+ if !slices.Equal(got, want) {
+ t.Errorf("fetched %v, want %v", got, want)
+ }
+}
+
+// The dot-separated tags of a wheel are alternatives: a wheel matches a
+// cached one when any of its tags does.
+func TestRefreshMatchesWheelsOnAnyAlternativeTag(t *testing.T) {
+ got := refreshFetches(t, "pypi", "demo", "1.0.0",
+ []string{"demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.whl", "demo-1.0.0-py2.py3-none-macosx_11_0_arm64.macosx_11_0_x86_64.whl"},
+ "1.1.0", wheels(
+ "demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.linux_x86_64.whl",
+ "demo-1.1.0-py3-none-macosx_11_0_x86_64.whl",
+ "demo-1.1.0-cp312-cp312-win_amd64.whl",
+ ))
+ want := []string{
+ "https://files.example/demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.linux_x86_64.whl",
+ "https://files.example/demo-1.1.0-py3-none-macosx_11_0_x86_64.whl",
+ }
+ if !slices.Equal(got, want) {
+ t.Errorf("fetched %v, want %v", got, want)
+ }
+}
+
+// In an abi3 wheel the python tag is a minimum: cryptography 44.0.0 ships
+// cp37-abi3 and cp39-abi3 wheels, 46.0.0 cp38-abi3 and cp311-abi3 ones. A
+// refresh stays on the cached major line, so the newer tags are on 44.1.0
+// here.
+func TestRefreshMatchesAbi3WheelsAcrossPythonMinimums(t *testing.T) {
+ got := refreshFetches(t, "pypi", "cryptography", "44.0.0",
+ []string{"cryptography-44.0.0-cp39-abi3-manylinux_2_28_x86_64.whl"},
+ "44.1.0", wheels(
+ "cryptography-44.1.0-cp38-abi3-manylinux_2_28_x86_64.whl",
+ "cryptography-44.1.0-cp311-abi3-manylinux_2_28_x86_64.whl",
+ "cryptography-44.1.0-cp311-abi3-manylinux_2_28_aarch64.whl",
+ "cryptography-44.1.0-cp311-abi3-macosx_10_9_universal2.whl",
+ "cryptography-44.1.0-cp311-abi3-win_amd64.whl",
+ "cryptography-44.1.0-cp314-cp314t-manylinux_2_28_x86_64.whl",
+ ))
+ want := []string{
+ "https://files.example/cryptography-44.1.0-cp311-abi3-manylinux_2_28_x86_64.whl",
+ "https://files.example/cryptography-44.1.0-cp38-abi3-manylinux_2_28_x86_64.whl",
+ }
+ if !slices.Equal(got, want) {
+ t.Errorf("fetched %v, want %v", got, want)
+ }
+}
diff --git a/internal/mirror/refresh_versions.go b/internal/mirror/refresh_versions.go
new file mode 100644
index 00000000..b489170d
--- /dev/null
+++ b/internal/mirror/refresh_versions.go
@@ -0,0 +1,67 @@
+package mirror
+
+import (
+ "strconv"
+
+ "github.com/git-pkgs/purl"
+ "github.com/git-pkgs/registries"
+ "github.com/git-pkgs/vers"
+)
+
+// newestPerCachedLine returns, for each major line a cached version is on,
+// the newest release of that line that allowed accepts, when it is newer
+// than every cached version of the line. A line is the major version, or
+// 0.minor below 1.0, as caret ranges read them. Prereleases and yanked,
+// retracted or deprecated releases are skipped.
+func newestPerCachedLine(ecosystem string, cached []string, releases []registries.Version, allowed func(registries.Version) bool) []string {
+ scheme := purl.EcosystemToPURLType(ecosystem)
+ newestCached := map[string]string{}
+ var lines []string
+ for _, v := range cached {
+ line, ok := versionLine(v)
+ if !ok {
+ continue
+ }
+ current, seen := newestCached[line]
+ if !seen {
+ lines = append(lines, line)
+ }
+ if !seen || vers.CompareWithScheme(v, current, scheme) > 0 {
+ newestCached[line] = v
+ }
+ }
+
+ newest := map[string]string{}
+ for _, r := range releases {
+ line, ok := versionLine(r.Number)
+ if !ok || r.Status != registries.StatusNone || vers.IsPrereleaseWithScheme(r.Number, scheme) {
+ continue
+ }
+ if _, cachedLine := newestCached[line]; !cachedLine || !allowed(r) {
+ continue
+ }
+ if current, seen := newest[line]; !seen || vers.CompareWithScheme(r.Number, current, scheme) > 0 {
+ newest[line] = r.Number
+ }
+ }
+
+ var out []string
+ for _, line := range lines {
+ if v, ok := newest[line]; ok && vers.CompareWithScheme(v, newestCached[line], scheme) > 0 {
+ out = append(out, v)
+ }
+ }
+ return out
+}
+
+// versionLine returns the major line of a version.
+func versionLine(v string) (string, bool) {
+ info, err := vers.ParseVersion(v)
+ if err != nil {
+ return "", false
+ }
+ if info.Major == 0 {
+ return "0." + strconv.Itoa(info.Minor), true
+ }
+ return strconv.Itoa(info.Major), true
+}
diff --git a/internal/mirror/refresh_versions_test.go b/internal/mirror/refresh_versions_test.go
new file mode 100644
index 00000000..6e1e2628
--- /dev/null
+++ b/internal/mirror/refresh_versions_test.go
@@ -0,0 +1,79 @@
+package mirror
+
+import (
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/git-pkgs/registries"
+)
+
+func releases(numbers ...string) []registries.Version {
+ out := make([]registries.Version, len(numbers))
+ for i, n := range numbers {
+ out[i] = registries.Version{Number: n, PublishedAt: time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC)}
+ }
+ return out
+}
+
+func TestNewestPerCachedLine(t *testing.T) {
+ allowAll := func(registries.Version) bool { return true }
+ yanked := releases("4.17.20", "4.17.21", "4.17.22")
+ yanked[2].Status = registries.StatusYanked
+
+ tests := []struct {
+ name string
+ cached []string
+ releases []registries.Version
+ allowed func(registries.Version) bool
+ want []string
+ }{
+ {name: "newer patch on the cached line", cached: []string{"4.17.20"},
+ releases: releases("4.17.20", "4.17.21", "5.0.0"), want: []string{"4.17.21"}},
+ {name: "newest already cached", cached: []string{"4.17.21"},
+ releases: releases("4.17.20", "4.17.21", "5.0.0")},
+ {name: "one per cached major line", cached: []string{"1.2.0", "2.0.0"},
+ releases: releases("1.2.0", "1.3.0", "2.0.0", "2.1.0", "3.0.0"), want: []string{"1.3.0", "2.1.0"}},
+ {name: "zero major lines are minor lines", cached: []string{"0.3.1"},
+ releases: releases("0.3.1", "0.3.2", "0.4.0"), want: []string{"0.3.2"}},
+ {name: "prereleases skipped", cached: []string{"2.0.0"},
+ releases: releases("2.0.0", "2.1.0-beta.1")},
+ {name: "yanked skipped", cached: []string{"4.17.20"},
+ releases: yanked, want: []string{"4.17.21"}},
+ {name: "newest withheld, older newer one taken", cached: []string{"4.17.20"},
+ releases: releases("4.17.20", "4.17.21", "4.17.22"),
+ allowed: func(v registries.Version) bool { return v.Number != "4.17.22" },
+ want: []string{"4.17.21"}},
+ {name: "nothing allowed newer than cached", cached: []string{"4.17.20"},
+ releases: releases("4.17.20", "4.17.21"),
+ allowed: func(v registries.Version) bool { return v.Number == "4.17.20" }},
+ {name: "cached newer than every listed one", cached: []string{"4.17.25"},
+ releases: releases("4.17.20", "4.17.21")},
+ {name: "only the newest cached of a line counts", cached: []string{"4.17.19", "4.17.21"},
+ releases: releases("4.17.19", "4.17.20", "4.17.21")},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ allowed := tt.allowed
+ if allowed == nil {
+ allowed = allowAll
+ }
+ got := newestPerCachedLine("npm", tt.cached, tt.releases, allowed)
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("newestPerCachedLine = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestNewestPerCachedLineUsesEcosystemOrdering(t *testing.T) {
+ // Maven orders 1.0.10 after 1.0.9, and a gem's 1.0.0.rc1 is a prerelease.
+ got := newestPerCachedLine("maven", []string{"1.0.9"}, releases("1.0.9", "1.0.10"), func(registries.Version) bool { return true })
+ if !reflect.DeepEqual(got, []string{"1.0.10"}) {
+ t.Errorf("maven: got %v, want [1.0.10]", got)
+ }
+ got = newestPerCachedLine("gem", []string{"1.0.0"}, releases("1.0.0", "1.0.1.rc1"), func(registries.Version) bool { return true })
+ if got != nil {
+ t.Errorf("gem: got %v, want nothing", got)
+ }
+}
diff --git a/internal/mirror/schedule.go b/internal/mirror/schedule.go
index 1df9a750..244fd89b 100644
--- a/internal/mirror/schedule.go
+++ b/internal/mirror/schedule.go
@@ -17,10 +17,12 @@ const (
scheduleJitter = 10
)
-// Schedule mirrors a set of image PURLs every Interval.
+// Schedule mirrors a set of image PURLs every Interval, or refreshes the
+// most used packages when Refresh is set.
type Schedule struct {
PURLs []string
Platforms []string
+ Refresh *Refresh
Interval time.Duration
}
@@ -102,7 +104,13 @@ func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) {
}
}
started := time.Now()
- progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms})
+ var progress *Progress
+ var err error
+ if schedule.Refresh != nil {
+ progress, err = s.mirror.Refresh(ctx, *schedule.Refresh)
+ } else {
+ progress, err = s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms})
+ }
if ctx.Err() != nil {
if s.Claim != nil && s.Release != nil {
if err := s.Release(key); err != nil {
@@ -140,6 +148,9 @@ func scheduleDelay(interval time.Duration, first bool, randN func(int64) int64)
// does not inherit the old one's last run.
func scheduleKey(schedule Schedule) string {
h := sha256.New()
+ if r := schedule.Refresh; r != nil {
+ _, _ = fmt.Fprintf(h, "refresh top %d since %s max_versions %d max_bytes %d\n", r.Top, r.Since, r.MaxVersions, r.MaxBytes)
+ }
for _, purl := range schedule.PURLs {
_, _ = fmt.Fprintf(h, "purl %s\n", purl)
}
diff --git a/internal/mirror/schedule_test.go b/internal/mirror/schedule_test.go
index b5b0143a..0a6edc31 100644
--- a/internal/mirror/schedule_test.go
+++ b/internal/mirror/schedule_test.go
@@ -279,3 +279,36 @@ func TestScheduleKey(t *testing.T) {
}
}
}
+
+func TestSchedulerRunsRefresh(t *testing.T) {
+ m, fetcher := setupRefreshMirror(t, "tarball")
+ cacheNPM(t, m, "lodash", "4.17.20")
+ m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21")}
+ fetcher.reset()
+
+ schedule := Schedule{Refresh: &Refresh{Top: 10, Since: time.Hour}, Interval: time.Hour}
+ s := NewScheduler(m, []Schedule{schedule}, m.logger)
+ s.run(context.Background(), 0, schedule)
+
+ if urls := fetcher.fetched(); len(urls) != 1 || len(fetchedMatching(urls, "lodash-4.17.21.tgz")) != 1 {
+ t.Errorf("fetched %v, want lodash 4.17.21", urls)
+ }
+}
+
+func TestScheduleKeyRefresh(t *testing.T) {
+ base := Schedule{Refresh: &Refresh{Top: 50, Since: time.Hour}, Interval: time.Hour}
+ same := Schedule{Refresh: &Refresh{Top: 50, Since: time.Hour}, Interval: time.Hour}
+ if scheduleKey(base) != scheduleKey(same) {
+ t.Error("same refresh gave different keys")
+ }
+ for name, other := range map[string]Schedule{
+ "no refresh": {Interval: time.Hour},
+ "top": {Refresh: &Refresh{Top: 10, Since: time.Hour}, Interval: time.Hour},
+ "since": {Refresh: &Refresh{Top: 50, Since: 2 * time.Hour}, Interval: time.Hour},
+ "interval": {Refresh: &Refresh{Top: 50, Since: time.Hour}, Interval: 2 * time.Hour},
+ } {
+ if scheduleKey(other) == scheduleKey(base) {
+ t.Errorf("changing %s kept the key", name)
+ }
+ }
+}
diff --git a/internal/mirror/source.go b/internal/mirror/source.go
index c0441c4e..bbfddf0b 100644
--- a/internal/mirror/source.go
+++ b/internal/mirror/source.go
@@ -26,6 +26,13 @@ type PackageVersion struct {
Qualifiers map[string]string
// Platforms limit which platforms of an image index are mirrored.
Platforms []string
+ // variants, when set, limit the files of a version that are mirrored to
+ // those of these variants (see fileEcosystem.variants).
+ variants map[string]bool
+ // fileURL, when set, is the one file to mirror, cached as cacheFilename
+ // and checked against digest when there is one. A refresh finds it in
+ // a repository index.
+ fileURL, cacheFilename, digest string
}
func (pv PackageVersion) String() string {
diff --git a/internal/server/mirror_schedule.go b/internal/server/mirror_schedule.go
index 172c2de2..78315d44 100644
--- a/internal/server/mirror_schedule.go
+++ b/internal/server/mirror_schedule.go
@@ -4,6 +4,7 @@ import (
"context"
"time"
+ "github.com/git-pkgs/proxy/internal/config"
"github.com/git-pkgs/proxy/internal/handler"
"github.com/git-pkgs/proxy/internal/mirror"
)
@@ -13,20 +14,38 @@ import (
const mirrorScheduleShutdownWait = 30 * time.Second
func (s *Server) startMirrorSchedules(ctx context.Context, proxy *handler.Proxy) {
- if len(s.cfg.Mirror.Schedules) == 0 {
+ schedules := mirrorSchedules(s.cfg.Mirror)
+ if len(schedules) == 0 {
return
}
// One worker, so a schedule never fans out against the registry.
m := s.newMirror(proxy, 1)
- schedules := make([]mirror.Schedule, len(s.cfg.Mirror.Schedules))
- for i, schedule := range s.cfg.Mirror.Schedules {
- schedules[i] = mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()}
- }
s.scheduler = mirror.NewScheduler(m, schedules, s.logger)
s.scheduler.Claim = s.db.ClaimJobRun
s.scheduler.Release = s.db.ReleaseJobRun
s.scheduler.Start(ctx)
- s.logger.Info("mirror schedules enabled", "schedules", len(schedules))
+ s.logger.Info("mirror schedules enabled", "schedules", len(schedules), "refresh", s.cfg.Mirror.Refresh != nil)
+}
+
+// mirrorSchedules returns the schedules of a validated mirror configuration,
+// the refresh last.
+func mirrorSchedules(cfg config.MirrorConfig) []mirror.Schedule {
+ var schedules []mirror.Schedule
+ for _, schedule := range cfg.Schedules {
+ schedules = append(schedules, mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()})
+ }
+ if r := cfg.Refresh; r != nil {
+ schedules = append(schedules, mirror.Schedule{
+ Refresh: &mirror.Refresh{
+ Top: r.TopPackages(),
+ Since: r.SinceDuration(),
+ MaxVersions: r.MaxVersionsPerRun(),
+ MaxBytes: r.MaxBytesPerRun(),
+ },
+ Interval: r.IntervalDuration(),
+ })
+ }
+ return schedules
}
func (s *Server) waitForMirrorSchedules(ctx context.Context) {
diff --git a/internal/server/mirror_schedule_test.go b/internal/server/mirror_schedule_test.go
index d2440e65..1c19559f 100644
--- a/internal/server/mirror_schedule_test.go
+++ b/internal/server/mirror_schedule_test.go
@@ -4,10 +4,12 @@ import (
"context"
"io"
"log/slog"
+ "reflect"
"sync/atomic"
"testing"
"time"
+ "github.com/git-pkgs/proxy/internal/config"
"github.com/git-pkgs/proxy/internal/handler"
"github.com/git-pkgs/proxy/internal/mirror"
)
@@ -96,3 +98,23 @@ func TestWaitForMirrorSchedulesWithoutSchedules(t *testing.T) {
s := &Server{}
s.waitForMirrorSchedules(context.Background())
}
+
+func TestMirrorSchedulesFromConfig(t *testing.T) {
+ cfg := config.MirrorConfig{
+ Schedules: []config.MirrorSchedule{{Images: []string{"pkg:docker/alpine@3"}, Interval: "6h", Platforms: []string{"linux/amd64"}}},
+ Refresh: &config.MirrorRefresh{Interval: "24h", Top: 20, Since: "7d", MaxVersions: 100, MaxBytes: "1GB"},
+ }
+ if err := cfg.Validate(); err != nil {
+ t.Fatal(err)
+ }
+ want := []mirror.Schedule{
+ {PURLs: []string{"pkg:docker/alpine@3"}, Platforms: []string{"linux/amd64"}, Interval: 6 * time.Hour},
+ {Refresh: &mirror.Refresh{Top: 20, Since: 7 * 24 * time.Hour, MaxVersions: 100, MaxBytes: 1 << 30}, Interval: 24 * time.Hour},
+ }
+ if got := mirrorSchedules(cfg); !reflect.DeepEqual(got, want) {
+ t.Errorf("mirrorSchedules =\n%+v\nwant\n%+v", got, want)
+ }
+ if got := mirrorSchedules(config.MirrorConfig{}); len(got) != 0 {
+ t.Errorf("mirrorSchedules of an empty config = %+v, want none", got)
+ }
+}