Mirror Alpine packages

docs/configuration.md

@@ -1726,6 +1726,8 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check
 
 A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either.
 
+An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one.
+
 | Flag | Default | Description |
 |------|---------|-------------|
 | `--sbom` | | Path to CycloneDX or SPDX SBOM file |

internal/handler/apk.go

@@ -60,18 +60,31 @@ type APKHandler struct {
 // When repositories is empty, a single repository named "alpine" pointing at
 // the official Alpine mirror is used.
 func NewAPKHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *APKHandler {
-	h := &APKHandler{
+	return &APKHandler{
 		proxy:        proxy,
 		proxyURL:     strings.TrimSuffix(proxyURL, "/"),
-		repositories: make(map[string]string, len(repositories)),
+		repositories: APKRepositories(repositories),
 	}
-	for name, repositoryURL := range repositories {
-		h.repositories[name] = strings.TrimSuffix(repositoryURL, "/")
+}
+
+// APKRepositories returns the upstream.apk repositories by name without
+// trailing slashes, or the official Alpine mirror as "alpine" when none are
+// configured.
+func APKRepositories(configured map[string]string) map[string]string {
+	repositories := make(map[string]string, len(configured))
+	for name, repositoryURL := range configured {
+		repositories[name] = strings.TrimSuffix(repositoryURL, "/")
 	}
-	if len(h.repositories) == 0 {
-		h.repositories[defaultAPKRepositoryName] = defaultAPKUpstream
+	if len(repositories) == 0 {
+		repositories[defaultAPKRepositoryName] = defaultAPKUpstream
 	}
-	return h
+	return repositories
+}
+
+// APKCacheFilename returns the filename a package at path
+// ({release}/{repo}/{arch}/{file}) of repository is cached under.
+func APKCacheFilename(repository, path string) string {
+	return repository + "/" + path
 }
 
 // Routes returns the HTTP handler for APK requests.
@@ -145,7 +158,7 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques
 	}
 
 	downloadURL := upstreamURL + "/" + path
-	cacheFilename := repository + "/" + path
+	cacheFilename := APKCacheFilename(repository, path)
 
 	h.proxy.Logger.Info("apk package download",
 		"repository", repository, "name", name, "version", version, "arch", arch)

internal/mirror/apk.go

@@ -0,0 +1,154 @@
+package mirror
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/git-pkgs/proxy/internal/handler"
+	"github.com/git-pkgs/registries"
+)
+
+// apkIndexTTL is how long a repository's APKINDEX is reused, so mirroring
+// the packages of an image reads each index once.
+const apkIndexTTL = 5 * time.Minute
+
+var (
+	// apkDistroPattern takes the release from syft's distro qualifier, such
+	// as alpine-3.20.3 for v3.20. Edge, as alpine-3.21.0_alpha20240807,
+	// isn't a release and doesn't match.
+	apkDistroPattern = regexp.MustCompile(`^alpine-(\d+)\.(\d+)(?:\.\d+)?$`)
+	apkArchPattern   = regexp.MustCompile(`^[a-z0-9_]+$`)
+	// apkComponents are the repositories of a release a package is looked
+	// up in, in order.
+	apkComponents = []string{"main", "community"}
+)
+
+// apkIndexes keeps one registries client per repository directory, which
+// holds the parsed APKINDEX.
+type apkIndexes struct {
+	mu   sync.Mutex
+	regs map[string]apkIndex
+}
+
+type apkIndex struct {
+	reg     registries.Registry
+	fetched time.Time
+}
+
+// mirrorAPK mirrors an Alpine package under the path the handler caches it
+// at, {repository}/v{release}/{component}/{arch}/{file}. The PURL namespace
+// names the upstream.apk repository; the distro and arch qualifiers give
+// the release and architecture, and the package is looked up in the main
+// and community indexes of that release.
+func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *progressTracker) {
+	fail := func(err error) { m.recordFailure(tracker, pv, err) }
+	repository, name, ok := strings.Cut(pv.Name, "/")
+	if !ok {
+		repository, name = "alpine", pv.Name
+	}
+	repositoryURL, ok := handler.APKRepositories(m.APKRepositories)[repository]
+	if !ok {
+		fail(fmt.Errorf("unknown APK repository %q: it isn't in upstream.apk", repository))
+		return
+	}
+	match := apkDistroPattern.FindStringSubmatch(pv.Qualifiers["distro"])
+	if match == nil {
+		fail(fmt.Errorf("apk packages need a distro qualifier with the Alpine release, such as distro=alpine-3.20.3, not %q", pv.Qualifiers["distro"]))
+		return
+	}
+	release := "v" + match[1] + "." + match[2]
+	arch := pv.Qualifiers["arch"]
+	if !apkArchPattern.MatchString(arch) {
+		fail(fmt.Errorf("apk packages need an arch qualifier, such as arch=x86_64, not %q", arch))
+		return
+	}
+
+	for _, component := range apkComponents {
+		dir := release + "/" + component + "/" + arch
+		artifact, found, err := m.apkArtifact(ctx, repositoryURL+"/"+dir, name, pv.Version)
+		if err != nil {
+			fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", dir, err))
+			return
+		}
+		if !found {
+			continue
+		}
+		cacheFilename := handler.APKCacheFilename(repository, dir+"/"+artifact.Filename)
+		result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, pv.Ecosystem, name, pv.Version, cacheFilename, artifact.URL)
+		if err != nil {
+			fail(err)
+			return
+		}
+		m.recordResult(tracker, pv, result)
+		return
+	}
+	fail(fmt.Errorf("%s %s isn't in %s of %s for %s", name, pv.Version, strings.Join(apkComponents, " or "), release, arch))
+}
+
+// apkArtifact returns the file of version of name that the index in dirURL
+// lists, and whether it lists one.
+func (m *Mirror) apkArtifact(ctx context.Context, dirURL, name, version string) (registries.Artifact, bool, error) {
+	reg, err := m.apkRegistry(dirURL)
+	if err != nil {
+		return registries.Artifact{}, false, err
+	}
+	versions, err := reg.FetchVersions(ctx, name)
+	if isNotFound(err) {
+		return registries.Artifact{}, false, nil
+	}
+	if err != nil {
+		m.apk.forget(dirURL)
+		return registries.Artifact{}, false, err
+	}
+	for _, v := range versions {
+		if v.Number == version && len(v.Artifacts) > 0 {
+			return v.Artifacts[0], true, nil
+		}
+	}
+	return registries.Artifact{}, false, nil
+}
+
+// apkRegistry returns the client for the repository directory at dirURL,
+// reusing it, and the index it read, for apkIndexTTL.
+func (m *Mirror) apkRegistry(dirURL string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface
+	m.apk.mu.Lock()
+	defer m.apk.mu.Unlock()
+	if idx, ok := m.apk.regs[dirURL]; ok && time.Since(idx.fetched) < apkIndexTTL {
+		return idx.reg, nil
+	}
+	r, _ := m.Versions.(RegistryVersions)
+	// Naming the v2 index reads it also where Packages.adb is served beside
+	// it, which the client would try first and refuse.
+	reg, err := registries.New("apk", dirURL+"/APKINDEX.tar.gz", r.client())
+	if err != nil {
+		return nil, err
+	}
+	if m.apk.regs == nil {
+		m.apk.regs = map[string]apkIndex{}
+	}
+	m.apk.regs[dirURL] = apkIndex{reg: reg, fetched: time.Now()}
+	return reg, nil
+}
+
+// forget drops the client for dirURL, so an index that failed to load is
+// read again next time.
+func (a *apkIndexes) forget(dirURL string) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	delete(a.regs, dirURL)
+}
+
+// isNotFound reports whether err means a package or index doesn't exist,
+// such as a release without a community repository for an architecture.
+func isNotFound(err error) bool {
+	if errors.Is(err, registries.ErrNotFound) {
+		return true
+	}
+	var httpErr *registries.HTTPError
+	return errors.As(err, &httpErr) && httpErr.IsNotFound()
+}

internal/mirror/mirror.go

@@ -45,11 +45,16 @@ type Mirror struct {
 	// uses https://pkg.julialang.org.
 	JuliaUpstream string
 
+	// APKRepositories are the upstream.apk repositories by name. Nil uses
+	// the official Alpine mirror as "alpine".
+	APKRepositories map[string]string
+
 	// RefusePackages, when set, fails every item Images does not take with
 	// this reason instead of fetching it, so nothing is cached.
 	RefusePackages string
 
 	cran cranIndex
+	apk  apkIndexes
 }
 
 // New creates a new Mirror with the given dependencies.
@@ -270,6 +275,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog
 		m.mirrorCRAN(ctx, pv, tracker)
 		return
 	}
+	if pv.Ecosystem == "alpine" {
+		m.mirrorAPK(ctx, pv, tracker)
+		return
+	}
 	result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "")
 	if err != nil {
 		m.recordFailure(tracker, pv, err)

internal/mirror/source.go

@@ -148,8 +148,11 @@ func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]s
 		name = p.Namespace + "/" + p.Name
 	}
 	qualifiers = fileQualifiers(ecosystem, p.Qualifiers.Map())
-	if ecosystem == "julia" {
+	switch ecosystem {
+	case "julia":
 		qualifiers = map[string]string{"uuid": p.Qualifier("uuid")}
+	case "alpine":
+		qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro")}
 	}
 	return ecosystem, name, qualifiers
 }