diff --git a/docs/configuration.md b/docs/configuration.md index ae2ae796..73a0b92c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1726,6 +1726,8 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. +An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 5de909a5..045dbac6 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -60,18 +60,31 @@ type APKHandler struct { // When repositories is empty, a single repository named "alpine" pointing at // the official Alpine mirror is used. func NewAPKHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *APKHandler { - h := &APKHandler{ + return &APKHandler{ proxy: proxy, proxyURL: strings.TrimSuffix(proxyURL, "/"), - repositories: make(map[string]string, len(repositories)), + repositories: APKRepositories(repositories), } - for name, repositoryURL := range repositories { - h.repositories[name] = strings.TrimSuffix(repositoryURL, "/") +} + +// APKRepositories returns the upstream.apk repositories by name without +// trailing slashes, or the official Alpine mirror as "alpine" when none are +// configured. +func APKRepositories(configured map[string]string) map[string]string { + repositories := make(map[string]string, len(configured)) + for name, repositoryURL := range configured { + repositories[name] = strings.TrimSuffix(repositoryURL, "/") } - if len(h.repositories) == 0 { - h.repositories[defaultAPKRepositoryName] = defaultAPKUpstream + if len(repositories) == 0 { + repositories[defaultAPKRepositoryName] = defaultAPKUpstream } - return h + return repositories +} + +// APKCacheFilename returns the filename a package at path +// ({release}/{repo}/{arch}/{file}) of repository is cached under. +func APKCacheFilename(repository, path string) string { + return repository + "/" + path } // Routes returns the HTTP handler for APK requests. @@ -145,7 +158,7 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques } downloadURL := upstreamURL + "/" + path - cacheFilename := repository + "/" + path + cacheFilename := APKCacheFilename(repository, path) h.proxy.Logger.Info("apk package download", "repository", repository, "name", name, "version", version, "arch", arch) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go new file mode 100644 index 00000000..3e08572a --- /dev/null +++ b/internal/mirror/apk.go @@ -0,0 +1,154 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + "sync" + "time" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// apkIndexTTL is how long a repository's APKINDEX is reused, so mirroring +// the packages of an image reads each index once. +const apkIndexTTL = 5 * time.Minute + +var ( + // apkDistroPattern takes the release from syft's distro qualifier, such + // as alpine-3.20.3 for v3.20. Edge, as alpine-3.21.0_alpha20240807, + // isn't a release and doesn't match. + apkDistroPattern = regexp.MustCompile(`^alpine-(\d+)\.(\d+)(?:\.\d+)?$`) + apkArchPattern = regexp.MustCompile(`^[a-z0-9_]+$`) + // apkComponents are the repositories of a release a package is looked + // up in, in order. + apkComponents = []string{"main", "community"} +) + +// apkIndexes keeps one registries client per repository directory, which +// holds the parsed APKINDEX. +type apkIndexes struct { + mu sync.Mutex + regs map[string]apkIndex +} + +type apkIndex struct { + reg registries.Registry + fetched time.Time +} + +// mirrorAPK mirrors an Alpine package under the path the handler caches it +// at, {repository}/v{release}/{component}/{arch}/{file}. The PURL namespace +// names the upstream.apk repository; the distro and arch qualifiers give +// the release and architecture, and the package is looked up in the main +// and community indexes of that release. +func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + repository, name, ok := strings.Cut(pv.Name, "/") + if !ok { + repository, name = "alpine", pv.Name + } + repositoryURL, ok := handler.APKRepositories(m.APKRepositories)[repository] + if !ok { + fail(fmt.Errorf("unknown APK repository %q: it isn't in upstream.apk", repository)) + return + } + match := apkDistroPattern.FindStringSubmatch(pv.Qualifiers["distro"]) + if match == nil { + fail(fmt.Errorf("apk packages need a distro qualifier with the Alpine release, such as distro=alpine-3.20.3, not %q", pv.Qualifiers["distro"])) + return + } + release := "v" + match[1] + "." + match[2] + arch := pv.Qualifiers["arch"] + if !apkArchPattern.MatchString(arch) { + fail(fmt.Errorf("apk packages need an arch qualifier, such as arch=x86_64, not %q", arch)) + return + } + + for _, component := range apkComponents { + dir := release + "/" + component + "/" + arch + artifact, found, err := m.apkArtifact(ctx, repositoryURL+"/"+dir, name, pv.Version) + if err != nil { + fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", dir, err)) + return + } + if !found { + continue + } + cacheFilename := handler.APKCacheFilename(repository, dir+"/"+artifact.Filename) + result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, pv.Ecosystem, name, pv.Version, cacheFilename, artifact.URL) + if err != nil { + fail(err) + return + } + m.recordResult(tracker, pv, result) + return + } + fail(fmt.Errorf("%s %s isn't in %s of %s for %s", name, pv.Version, strings.Join(apkComponents, " or "), release, arch)) +} + +// apkArtifact returns the file of version of name that the index in dirURL +// lists, and whether it lists one. +func (m *Mirror) apkArtifact(ctx context.Context, dirURL, name, version string) (registries.Artifact, bool, error) { + reg, err := m.apkRegistry(dirURL) + if err != nil { + return registries.Artifact{}, false, err + } + versions, err := reg.FetchVersions(ctx, name) + if isNotFound(err) { + return registries.Artifact{}, false, nil + } + if err != nil { + m.apk.forget(dirURL) + return registries.Artifact{}, false, err + } + for _, v := range versions { + if v.Number == version && len(v.Artifacts) > 0 { + return v.Artifacts[0], true, nil + } + } + return registries.Artifact{}, false, nil +} + +// apkRegistry returns the client for the repository directory at dirURL, +// reusing it, and the index it read, for apkIndexTTL. +func (m *Mirror) apkRegistry(dirURL string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface + m.apk.mu.Lock() + defer m.apk.mu.Unlock() + if idx, ok := m.apk.regs[dirURL]; ok && time.Since(idx.fetched) < apkIndexTTL { + return idx.reg, nil + } + r, _ := m.Versions.(RegistryVersions) + // Naming the v2 index reads it also where Packages.adb is served beside + // it, which the client would try first and refuse. + reg, err := registries.New("apk", dirURL+"/APKINDEX.tar.gz", r.client()) + if err != nil { + return nil, err + } + if m.apk.regs == nil { + m.apk.regs = map[string]apkIndex{} + } + m.apk.regs[dirURL] = apkIndex{reg: reg, fetched: time.Now()} + return reg, nil +} + +// forget drops the client for dirURL, so an index that failed to load is +// read again next time. +func (a *apkIndexes) forget(dirURL string) { + a.mu.Lock() + defer a.mu.Unlock() + delete(a.regs, dirURL) +} + +// isNotFound reports whether err means a package or index doesn't exist, +// such as a release without a community repository for an architecture. +func isNotFound(err error) bool { + if errors.Is(err, registries.ErrNotFound) { + return true + } + var httpErr *registries.HTTPError + return errors.As(err, &httpErr) && httpErr.IsNotFound() +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index ac20a4ee..d5ec9a59 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -45,11 +45,16 @@ type Mirror struct { // uses https://pkg.julialang.org. JuliaUpstream string + // APKRepositories are the upstream.apk repositories by name. Nil uses + // the official Alpine mirror as "alpine". + APKRepositories map[string]string + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string cran cranIndex + apk apkIndexes } // New creates a new Mirror with the given dependencies. @@ -270,6 +275,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorCRAN(ctx, pv, tracker) return } + if pv.Ecosystem == "alpine" { + m.mirrorAPK(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index ecc3a0c5..1d3954cd 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -148,8 +148,11 @@ func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]s name = p.Namespace + "/" + p.Name } qualifiers = fileQualifiers(ecosystem, p.Qualifiers.Map()) - if ecosystem == "julia" { + switch ecosystem { + case "julia": qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} + case "alpine": + qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro")} } return ecosystem, name, qualifiers } diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go new file mode 100644 index 00000000..d5a8c0d6 --- /dev/null +++ b/internal/server/mirror_apk_test.go @@ -0,0 +1,158 @@ +package server + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +const ( + apkBusyboxPath = "/v3.20/main/x86_64/busybox-1.36.1-r29.apk" + apkHtopPath = "/v3.20/community/x86_64/htop-3.3.0-r0.apk" + apkMainIndex = "/v3.20/main/x86_64/APKINDEX.tar.gz" + apkCommIndex = "/v3.20/community/x86_64/APKINDEX.tar.gz" +) + +// apkUpstream serves an Alpine mirror with busybox in v3.20/main and htop in +// v3.20/community, for x86_64. It counts requests by path. +type apkUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string]string + hits map[string]int +} + +func apkIndexArchive(t *testing.T, index string) string { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + if err := tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(index))}); err != nil { + t.Fatal(err) + } + _, _ = io.WriteString(tw, index) + _ = tw.Close() + _ = zw.Close() + return buf.String() +} + +func newAPKUpstream(t *testing.T) *apkUpstream { + t.Helper() + u := &apkUpstream{ + files: map[string]string{ + apkMainIndex: apkIndexArchive(t, "P:busybox\nV:1.36.1-r29\nA:x86_64\nS:18\n\nP:musl\nV:1.2.5-r0\nA:x86_64\nS:15\n\n"), + apkCommIndex: apkIndexArchive(t, "P:htop\nV:3.3.0-r0\nA:x86_64\nS:15\n\n"), + apkBusyboxPath: "busybox 1.36.1-r29", + apkHtopPath: "htop 3.3.0-r0 x86", + "/v3.20/main/x86_64/musl-1.2.5-r0.apk": "musl 1.2.5-r0", + }, + hits: map[string]int{}, + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + body, ok := u.files[r.URL.Path] + u.hits[r.URL.Path]++ + u.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *apkUpstream) hitsFor(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +func apkMirrorConfig(upstream *apkUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.APK = map[string]string{"alpine": upstream.URL} + return cfg +} + +func getAPK(proxy *handler.Proxy, upstream *apkUpstream, path string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + h := handler.NewAPKHandler(proxy, "http://proxy.test", map[string]string{"alpine": upstream.URL}) + h.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/alpine"+path, nil)) + return rec +} + +func TestMirrorAPKPackagesAreServedFromCache(t *testing.T) { + upstream := newAPKUpstream(t) + + progress, proxy := mirrorWithConfig(t, apkMirrorConfig(upstream), + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3", + "pkg:apk/alpine/htop@3.3.0-r0?arch=x86_64&distro=alpine-3.20.3", + "pkg:apk/alpine/musl@1.2.5-r0?arch=x86_64&distro=alpine-3.20.3") + + if progress.Completed != 3 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{apkBusyboxPath, apkHtopPath, apkMainIndex, apkCommIndex} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } + + // apk gets the packages from the cache once the upstream is down. + upstream.Close() + for path, want := range map[string]string{ + apkBusyboxPath: "busybox 1.36.1-r29", + apkHtopPath: "htop 3.3.0-r0 x86", + } { + if rec := getAPK(proxy, upstream, path); rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", path, rec.Code, rec.Body, want) + } + } +} + +func TestMirrorAPKReadsV2IndexBesideV3(t *testing.T) { + // A repository moving to apk v3 serves Packages.adb next to + // APKINDEX.tar.gz; the mirror reads the v2 index. + upstream := newAPKUpstream(t) + upstream.files["/v3.20/main/x86_64/Packages.adb"] = "ADB.v3-index" + + progress, _ := mirrorWithConfig(t, apkMirrorConfig(upstream), + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3") + + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } +} + +func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { + upstream := newAPKUpstream(t) + + tests := map[string]string{ + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64": "distro", + "pkg:apk/alpine/busybox@1.36.1-r29?distro=alpine-3.20.3": "arch", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20-evil": "distro", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.21.0_alpha20240807": "distro", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=debian-12": "distro", + "pkg:apk/other/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3": "upstream.apk", + "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "9.9.9-r0", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=../../etc&distro=alpine-3.20.3": "arch", + } + for p, want := range tests { + t.Run(p, func(t *testing.T) { + progress, _ := mirrorWithConfig(t, apkMirrorConfig(upstream), p) + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, want) { + t.Errorf("failed = %d, errors = %v; want one error mentioning %q", progress.Failed, progress.Errors, want) + } + }) + } +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index dba27d30..3e2cebb6 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -64,6 +64,7 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm) m.Versions = versions m.JuliaUpstream = cfg.Upstream.Julia + m.APKRepositories = cfg.Upstream.APK if versions == nil { m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) } diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 27b3d519..6d4599e7 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -37,11 +37,12 @@ func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { } // UseMirrorUpstreams points m at the configured upstreams: it lists -// versions from them, through client, and fetches Julia packages from -// upstream.julia. +// versions from them, through client, fetches Julia packages from +// upstream.julia and Alpine packages from upstream.apk. func UseMirrorUpstreams(m *mirror.Mirror, upstream config.UpstreamConfig, client *http.Client) { m.Versions = NewMirrorVersions(upstream, client) m.JuliaUpstream = upstream.Julia + m.APKRepositories = upstream.APK } // mirrorVersionsURL returns the registry base URL to list a package's