From 0bc0e903f872f5b49328811e6d8f30c20820dcd2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 19:30:34 +0100 Subject: [PATCH 01/19] Add a project inventory built from uploaded SBOMs Repositories upload an SBOM of their default branch, for example from syft in CI, to PUT /api/projects/{project}/sbom. The proxy records the package versions it lists, keyed by the same unversioned PURL the handlers record packages under, so the inventory joins with what the proxy has cached and the vulnerabilities it holds. That answers which repository uses which version of a package: the new /ui/projects pages list each project's package versions with outdated and vulnerability markers, package pages show the projects per version, the dashboard lists vulnerable versions in use, and GET /api/projects?purl= does the same for scripts. The feature is off by default. Uploads and deletes need projects.token; an upload replaces the project's components in one transaction. projects.max_age removes projects that stopped uploading. --- README.md | 12 + cmd/proxy/main.go | 5 +- config.example.yaml | 9 + docs/architecture.md | 4 + docs/configuration.md | 82 ++++ docs/swagger/docs.go | 288 +++++++++++++- docs/swagger/swagger.json | 288 +++++++++++++- go.mod | 2 +- internal/config/config.go | 10 + internal/config/projects.go | 53 +++ internal/config/projects_test.go | 69 ++++ internal/database/database_test.go | 2 +- internal/database/projects.go | 257 +++++++++++++ internal/database/projects_test.go | 228 +++++++++++ internal/database/schema.go | 76 ++++ internal/inventory/inventory.go | 185 +++++++++ internal/inventory/inventory_test.go | 133 +++++++ .../inventory/testdata/syft-cyclonedx.json | 1 + internal/server/dashboard.go | 6 + internal/server/layout.go | 3 + internal/server/projects_api.go | 335 +++++++++++++++++ internal/server/projects_api_test.go | 354 ++++++++++++++++++ internal/server/projects_prune.go | 43 +++ internal/server/projects_prune_test.go | 48 +++ internal/server/projects_ui.go | 287 ++++++++++++++ internal/server/projects_ui_test.go | 137 +++++++ internal/server/server.go | 14 + .../templates/components/severity_badge.html | 3 + internal/server/templates/layout/header.html | 1 + .../server/templates/pages/dashboard.html | 27 ++ .../server/templates/pages/package_show.html | 28 ++ .../server/templates/pages/project_show.html | 53 +++ .../server/templates/pages/projects_list.html | 33 ++ 33 files changed, 3071 insertions(+), 5 deletions(-) create mode 100644 internal/config/projects.go create mode 100644 internal/config/projects_test.go create mode 100644 internal/database/projects.go create mode 100644 internal/database/projects_test.go create mode 100644 internal/inventory/inventory.go create mode 100644 internal/inventory/inventory_test.go create mode 100644 internal/inventory/testdata/syft-cyclonedx.json create mode 100644 internal/server/projects_api.go create mode 100644 internal/server/projects_api_test.go create mode 100644 internal/server/projects_prune.go create mode 100644 internal/server/projects_prune_test.go create mode 100644 internal/server/projects_ui.go create mode 100644 internal/server/projects_ui_test.go create mode 100644 internal/server/templates/components/severity_badge.html create mode 100644 internal/server/templates/pages/project_show.html create mode 100644 internal/server/templates/pages/projects_list.html diff --git a/README.md b/README.md index 20c6d5db..61edf9dd 100644 --- a/README.md +++ b/README.md @@ -1300,6 +1300,17 @@ Registered when `hosted.enabled` is set. Pushing and deleting require `Authoriza | `GET /api/hosted/{ecosystem}/{name}` | List the archives pushed for a package | | `DELETE /api/hosted/{ecosystem}/{name}/{version}` | Delete a pushed version | +### Projects API + +Registered when `projects.enabled` is set. Uploading and deleting require `Authorization: Bearer `. See [Project inventory](docs/configuration.md#project-inventory). + +| Endpoint | Description | +|----------|-------------| +| `PUT /api/projects/{project}/sbom` | Replace a project's recorded package versions with the ones the SBOM in the body lists | +| `GET /api/projects` | List projects, or with `?purl=pkg:npm/lodash@4.17.20` the projects using a package or version | +| `GET /api/projects/{project}` | A project's recorded package versions | +| `DELETE /api/projects/{project}` | Delete a project | + ### Cache API Registered when `cache_delete` is set. It has no authentication of its own. @@ -1503,6 +1514,7 @@ The proxy serves a web UI under `/ui`. No separate frontend build is needed -- t - **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts. - **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews. - **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files. +- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated or vulnerable. Package pages then show which projects use each version, and the dashboard lists vulnerable versions in use. ## Monitoring diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index b19f6893..728a579f 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -8,7 +8,7 @@ // @securityDefinitions.apikey BearerAuth // @in header // @name Authorization -// @description "Bearer " followed by hosted.token +// @description "Bearer " followed by hosted.token, or projects.token for /api/projects // // The proxy caches package artifacts from upstream registries (npm, cargo, etc.) // providing faster, more reliable access for development teams. @@ -93,6 +93,9 @@ // PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB") // PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only // PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only +// PROXY_PROJECTS_ENABLED - Enable the project inventory (/api/projects, /ui/projects) +// PROXY_PROJECTS_TOKEN - Bearer token for uploading and deleting project SBOMs +// PROXY_PROJECTS_MAX_AGE - Remove projects not uploaded for this long (e.g. "90d") // PROXY_URL - Proxy URL for proxy hosted // // Example: diff --git a/config.example.yaml b/config.example.yaml index d3b6da49..a8753223 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -442,6 +442,15 @@ gradle: # npm_scopes: ["@acme"] # composer_vendors: ["acme"] +# Project inventory (disabled by default). Repositories upload an SBOM to +# PUT /api/projects/{project}/sbom, and the UI shows which project uses +# which package versions. Project names are visible on the unauthenticated UI. +# projects: +# enabled: true +# token: "${PROXY_PROJECTS_TOKEN}" +# # Drop projects that have not uploaded for this long ("" keeps them) +# max_age: "90d" + # Re-mirror container image tags on a schedule, so a moved tag is picked up # even when nobody pulls it. An unchanged tag costs one HEAD request. # mirror: diff --git a/docs/architecture.md b/docs/architecture.md index 660d8102..0105d3f5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -363,6 +363,10 @@ Selective package mirroring for pre-populating the proxy cache. Supports multipl The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. +### `internal/inventory` + +Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated and vulnerability markers on the project pages. + ### `internal/config` Configuration loading. diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b23..c5c6f7a0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1504,6 +1504,88 @@ Generic files never come from an upstream: anything not pushed answers 404. Whil Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones. +## Project inventory + +The project inventory answers "which repository uses which version of a package?". Each repository uploads an SBOM of its default branch, and the proxy records the package versions it lists. The UI then shows: + +- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version or vulnerabilities for that version. +- On a package page: which projects use each version, including versions the proxy hasn't cached. +- On the dashboard: vulnerable versions in use and the projects using them. + +It is disabled by default. Uploads and deletes require a token: + +```yaml +projects: + enabled: true + token: "${PROXY_PROJECTS_TOKEN}" + max_age: "90d" +``` + +| Config | Environment | Description | +|--------|-------------|-------------| +| `projects.enabled` | `PROXY_PROJECTS_ENABLED` | Register the `/api/projects` endpoints and the UI pages (default `false`) | +| `projects.token` | `PROXY_PROJECTS_TOKEN` | Bearer token required to upload and delete; required when enabled, supports `${VAR}` | +| `projects.max_age` | `PROXY_PROJECTS_MAX_AGE` | Remove projects that haven't uploaded for this long, e.g. `90d` or `720h`; empty or `0` keeps them (default) | + +Removing old projects runs hourly. With several instances on one database, one of them does it. + +### Uploading an SBOM + +`PUT /api/projects/{project}/sbom` takes a CycloneDX (JSON or XML) or SPDX (JSON or tag-value) document of up to 32 MiB and replaces everything recorded for the project with what it lists. The optional `ref` and `branch` query parameters are stored and shown with the project. + +The project name is a path such as `group/sub/repo`: segments of letters, digits, `.`, `_` and `-`, at most 255 characters. It can be sent as path segments (`/api/projects/group/sub/repo/sbom`) or URL-encoded as one segment (`/api/projects/group%2Fsub%2Frepo/sbom`). + +```bash +syft dir:. -o cyclonedx-json > sbom.json +curl -X PUT --data-binary @sbom.json \ + -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ + "https://proxy.example.com/api/projects/acme/shop/web/sbom?ref=$(git rev-parse HEAD)&branch=main" +``` + +The response counts the recorded package versions and the skipped entries: + +```json +{"project":"acme/shop/web","components":10,"skipped":5} +``` + +Entries are recorded by their package URL. Entries without one (files, the operating system) and package types the proxy doesn't serve (such as `pkg:github`) are skipped. Duplicates are recorded once. + +A GitLab CI job on the default branch could look like this: + +```yaml +sbom: + image: alpine:3.20 + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + script: + - apk add --no-cache curl + - curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin + - syft dir:. -o cyclonedx-json > sbom.json + - > + curl -sSf -X PUT --data-binary @sbom.json + -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" + "https://proxy.example.com/api/projects/$CI_PROJECT_PATH/sbom?ref=$CI_COMMIT_SHA&branch=$CI_COMMIT_BRANCH" +``` + +### Querying + +```bash +curl https://proxy.example.com/api/projects # all projects +curl https://proxy.example.com/api/projects/acme/shop/web # one project's package versions +curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash' # projects using lodash +curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash@4.17.20' # projects using lodash 4.17.20 +curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ + https://proxy.example.com/api/projects/acme/shop/web +``` + +Reading needs no token, like the rest of `/api`. + +### What it shows + +The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. Outdated and vulnerability markers use what the proxy knows about a package: its recorded latest version and the vulnerabilities recorded for it, matched against the version a project uses by the vulnerability's fixed version. A package the proxy has no record of has neither. + +Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy. + ## Mirror API The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP: diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index e35a86f3..7ddf6301 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -460,6 +460,192 @@ const docTemplate = `{ } } }, + "/api/projects": { + "get": { + "description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "List projects", + "parameters": [ + { + "type": "string", + "description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20", + "name": "purl", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}": { + "get": { + "description": "Returns the project with the package versions recorded for it.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Get a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the project and the package versions recorded for it.", + "tags": [ + "projects" + ], + "summary": "Delete a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}/sbom": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Upload a project's SBOM", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Commit the SBOM was generated from", + "name": "ref", + "in": "query" + }, + { + "type": "string", + "description": "Branch the SBOM was generated from", + "name": "branch", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectUploadResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/search": { "get": { "produces": [ @@ -1120,6 +1306,106 @@ const docTemplate = `{ } } }, + "server.ProjectComponent": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + }, + "purl": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "server.ProjectListResponse": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectSummary" + } + } + } + }, + "server.ProjectResponse": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectComponent" + } + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectSummary": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectUploadResponse": { + "type": "object", + "properties": { + "components": { + "description": "Components is the number of package versions recorded.", + "type": "integer" + }, + "project": { + "type": "string" + }, + "skipped": { + "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "type": "integer" + } + } + }, "server.SearchPackageResult": { "type": "object", "properties": { @@ -1206,7 +1492,7 @@ const docTemplate = `{ }, "securityDefinitions": { "BearerAuth": { - "description": "\"Bearer \" followed by hosted.token", + "description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects", "type": "apiKey", "name": "Authorization", "in": "header" diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 09e4f11e..eb6f3b9d 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -453,6 +453,192 @@ } } }, + "/api/projects": { + "get": { + "description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "List projects", + "parameters": [ + { + "type": "string", + "description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20", + "name": "purl", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}": { + "get": { + "description": "Returns the project with the package versions recorded for it.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Get a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the project and the package versions recorded for it.", + "tags": [ + "projects" + ], + "summary": "Delete a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}/sbom": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Upload a project's SBOM", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Commit the SBOM was generated from", + "name": "ref", + "in": "query" + }, + { + "type": "string", + "description": "Branch the SBOM was generated from", + "name": "branch", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectUploadResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/search": { "get": { "produces": [ @@ -1113,6 +1299,106 @@ } } }, + "server.ProjectComponent": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + }, + "purl": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "server.ProjectListResponse": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectSummary" + } + } + } + }, + "server.ProjectResponse": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectComponent" + } + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectSummary": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectUploadResponse": { + "type": "object", + "properties": { + "components": { + "description": "Components is the number of package versions recorded.", + "type": "integer" + }, + "project": { + "type": "string" + }, + "skipped": { + "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "type": "integer" + } + } + }, "server.SearchPackageResult": { "type": "object", "properties": { @@ -1199,7 +1485,7 @@ }, "securityDefinitions": { "BearerAuth": { - "description": "\"Bearer \" followed by hosted.token", + "description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects", "type": "apiKey", "name": "Authorization", "in": "header" diff --git a/go.mod b/go.mod index 94fbe6c3..2213d684 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/git-pkgs/registries v0.11.1 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 + github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 github.com/klauspost/compress v1.20.1 @@ -76,7 +77,6 @@ require ( github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/pom v0.1.8 // indirect - github.com/git-pkgs/vers v0.7.2 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index a6606994..42a4b9e5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -211,6 +211,9 @@ type Config struct { // Hosted configures pushing packages to the proxy (/api/hosted). Hosted HostedConfig `json:"hosted" yaml:"hosted"` + // Projects configures the project inventory (/api/projects). + Projects ProjectsConfig `json:"projects" yaml:"projects"` + // Gradle configures Gradle HttpBuildCache behavior. Gradle GradleConfig `json:"gradle" yaml:"gradle"` @@ -1448,6 +1451,9 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES") setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS") + setEnvBool(&c.Projects.Enabled, "PROXY_PROJECTS_ENABLED") + setEnvString(&c.Projects.Token, "PROXY_PROJECTS_TOKEN") + setEnvString(&c.Projects.MaxAge, "PROXY_PROJECTS_MAX_AGE") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") @@ -1622,6 +1628,10 @@ func (c *Config) validateComponents() error { HostedGenericRepository, HostedGenericRepository) } + if err := c.Projects.Validate(); err != nil { + return err + } + if err := c.Mirror.Validate(); err != nil { return err } diff --git a/internal/config/projects.go b/internal/config/projects.go new file mode 100644 index 00000000..f4160077 --- /dev/null +++ b/internal/config/projects.go @@ -0,0 +1,53 @@ +package config + +import ( + "fmt" + "time" +) + +// ProjectsConfig configures the project inventory: repositories upload an +// SBOM to /api/projects, and the proxy shows which of them use which package +// versions. +type ProjectsConfig struct { + // Enabled registers the /api/projects endpoints and the UI pages. Off by + // default. + Enabled bool `json:"enabled" yaml:"enabled"` + + // Token is the bearer token required to upload and delete projects. + // Required when Enabled. Supports ${VAR_NAME} expansion like AuthConfig + // fields. + Token string `json:"token" yaml:"token"` + + // MaxAge removes projects that have not uploaded an SBOM for this long + // (e.g., "90d"). Empty or "0" keeps them until they are deleted. + MaxAge string `json:"max_age" yaml:"max_age"` + + maxAge time.Duration +} + +// TokenExpanded returns Token with ${VAR_NAME} references expanded. +func (p *ProjectsConfig) TokenExpanded() string { + return expandEnv(p.Token) +} + +// MaxAgeDuration returns the parsed MaxAge, 0 to keep projects. Validate +// must run first. +func (p *ProjectsConfig) MaxAgeDuration() time.Duration { + return p.maxAge +} + +// Validate checks the project inventory settings. +func (p *ProjectsConfig) Validate() error { + p.maxAge = 0 + if p.MaxAge != "" { + d, err := parseRetentionDuration("projects.max_age", p.MaxAge) + if err != nil { + return err + } + p.maxAge = d + } + if p.Enabled && p.TokenExpanded() == "" { + return fmt.Errorf("projects.token is required when projects.enabled is true") + } + return nil +} diff --git a/internal/config/projects_test.go b/internal/config/projects_test.go new file mode 100644 index 00000000..ce3c27ce --- /dev/null +++ b/internal/config/projects_test.go @@ -0,0 +1,69 @@ +package config + +import ( + "testing" + "time" +) + +func TestProjectsConfigValidate(t *testing.T) { + t.Setenv("PROJECTS_TEST_TOKEN", "from-env") + + tests := []struct { + name string + projects ProjectsConfig + wantErr bool + maxAge time.Duration + }{ + {name: "disabled without token", projects: ProjectsConfig{}}, + {name: "enabled without token", projects: ProjectsConfig{Enabled: true}, wantErr: true}, + {name: "enabled with token", projects: ProjectsConfig{Enabled: true, Token: "s3cret"}}, + {name: "token from env reference", projects: ProjectsConfig{Enabled: true, Token: "${PROJECTS_TEST_TOKEN}"}}, + {name: "token env reference unset", projects: ProjectsConfig{Enabled: true, Token: "${PROJECTS_TEST_UNSET}"}, wantErr: true}, + {name: "max age in days", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "90d"}, maxAge: 90 * 24 * time.Hour}, + {name: "max age in hours", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "36h"}, maxAge: 36 * time.Hour}, + {name: "max age zero keeps projects", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "0"}}, + {name: "invalid max age", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "soon"}, wantErr: true}, + {name: "negative max age", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "-1d"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.projects.Validate() + if (err != nil) != tt.wantErr { + t.Fatalf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + if got := tt.projects.MaxAgeDuration(); err == nil && got != tt.maxAge { + t.Errorf("MaxAgeDuration() = %v, want %v", got, tt.maxAge) + } + }) + } +} + +func TestLoadFromEnvProjects(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_PROJECTS_ENABLED", "true") + t.Setenv("PROXY_PROJECTS_TOKEN", "env-token") + t.Setenv("PROXY_PROJECTS_MAX_AGE", "30d") + + cfg.LoadFromEnv() + + if !cfg.Projects.Enabled { + t.Error("Projects.Enabled not loaded from PROXY_PROJECTS_ENABLED") + } + if cfg.Projects.TokenExpanded() != "env-token" { + t.Errorf("Projects.Token = %q, want env-token", cfg.Projects.Token) + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := cfg.Projects.MaxAgeDuration(); got != 30*24*time.Hour { + t.Errorf("MaxAgeDuration() = %v, want 30 days", got) + } +} + +func TestValidateRejectsProjectsWithoutToken(t *testing.T) { + cfg := Default() + cfg.Projects = ProjectsConfig{Enabled: true} + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted projects.enabled without a token") + } +} diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a63..ea3b349e 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB { // Drop and recreate every table CreateSchema creates for clean test state; // leftover migration records make the next CreateSchema fail on the // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "migrations", "schema_info"} + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "projects", "project_components", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } diff --git a/internal/database/projects.go b/internal/database/projects.go new file mode 100644 index 00000000..b00596b9 --- /dev/null +++ b/internal/database/projects.go @@ -0,0 +1,257 @@ +package database + +import ( + "database/sql" + "errors" + "fmt" + "time" +) + +// Project is a repository that uploaded an SBOM to the project inventory. +type Project struct { + ID int64 `db:"id"` + Name string `db:"name"` + Ref sql.NullString `db:"ref"` + Branch sql.NullString `db:"branch"` + UploadedAt time.Time `db:"uploaded_at"` + // Components is the number of package versions recorded for it. + Components int64 `db:"components"` +} + +// ProjectComponent is a package version a project uses. PackagePURL is the +// unversioned package URL, as packages.purl records it. +type ProjectComponent struct { + PackagePURL string `db:"package_purl"` + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + Version string `db:"version"` +} + +// ProjectComponentInfo is a project's component together with the package +// record the proxy holds for it, if any. +type ProjectComponentInfo struct { + ProjectComponent + CachedEcosystem sql.NullString `db:"cached_ecosystem"` + CachedName sql.NullString `db:"cached_name"` + LatestVersion sql.NullString `db:"latest_version"` +} + +// ProjectUsage is a project using a version of a package. +type ProjectUsage struct { + Project + Version string `db:"version"` +} + +// ComponentVulnerability is a vulnerability recorded for the package of a +// component a project uses. It is not yet filtered by version: whether it +// affects Version depends on FixedVersion. +type ComponentVulnerability struct { + Project string `db:"project"` + ProjectComponent + CachedEcosystem string `db:"cached_ecosystem"` + CachedName string `db:"cached_name"` + VulnID string `db:"vuln_id"` + Severity sql.NullString `db:"severity"` + FixedVersion sql.NullString `db:"fixed_version"` + CVSSScore sql.NullFloat64 `db:"cvss_score"` +} + +const projectColumns = ` + SELECT pr.id, pr.name, pr.ref, pr.branch, pr.uploaded_at, + (SELECT COUNT(*) FROM project_components pc WHERE pc.project_id = pr.id) AS components` + +// SaveProject records the components of a project, replacing everything +// recorded for it before, in one transaction. Readers see either the old or +// the new set, never a mix. +func (db *DB) SaveProject(name, ref, branch string, uploadedAt time.Time, components []ProjectComponent) error { + tx, err := db.Beginx() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + // The upsert locks the project row, so concurrent uploads of the same + // project and pruning take turns. + if _, err := tx.Exec(db.Rebind(` + INSERT INTO projects (name, ref, branch, uploaded_at) VALUES (?, ?, ?, ?) + ON CONFLICT(name) DO UPDATE SET ref = excluded.ref, branch = excluded.branch, uploaded_at = excluded.uploaded_at + `), name, nullString(ref), nullString(branch), uploadedAt.UTC()); err != nil { + return fmt.Errorf("recording project: %w", err) + } + var id int64 + if err := tx.Get(&id, db.Rebind(`SELECT id FROM projects WHERE name = ?`), name); err != nil { + return fmt.Errorf("reading project id: %w", err) + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return fmt.Errorf("removing old components: %w", err) + } + + insert, err := tx.Preparex(db.Rebind(` + INSERT INTO project_components (project_id, package_purl, ecosystem, name, version) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(project_id, package_purl, version) DO NOTHING + `)) + if err != nil { + return err + } + defer func() { _ = insert.Close() }() + for _, c := range components { + if _, err := insert.Exec(id, c.PackagePURL, c.Ecosystem, c.Name, c.Version); err != nil { + return fmt.Errorf("recording component %s@%s: %w", c.PackagePURL, c.Version, err) + } + } + return tx.Commit() +} + +// DeleteProject removes a project and its components. It reports whether the +// project existed. +func (db *DB) DeleteProject(name string) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + var id int64 + err = tx.Get(&id, db.Rebind(`DELETE FROM projects WHERE name = ? RETURNING id`), name) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("deleting project: %w", err) + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return false, fmt.Errorf("deleting project components: %w", err) + } + return true, tx.Commit() +} + +// PruneProjects removes the projects last uploaded before cutoff, with their +// components, and returns how many it removed. A project uploaded again while +// the prune runs is kept. +func (db *DB) PruneProjects(cutoff time.Time) (int, error) { + var ids []int64 + if err := db.Select(&ids, db.Rebind(`SELECT id FROM projects WHERE uploaded_at < ?`), cutoff.UTC()); err != nil { + return 0, fmt.Errorf("listing expired projects: %w", err) + } + pruned := 0 + for _, id := range ids { + ok, err := db.pruneProject(id, cutoff) + if err != nil { + return pruned, err + } + if ok { + pruned++ + } + } + return pruned, nil +} + +func (db *DB) pruneProject(id int64, cutoff time.Time) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + res, err := tx.Exec(db.Rebind(`DELETE FROM projects WHERE id = ? AND uploaded_at < ?`), id, cutoff.UTC()) + if err != nil { + return false, fmt.Errorf("pruning project: %w", err) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return false, err + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return false, fmt.Errorf("pruning project components: %w", err) + } + return true, tx.Commit() +} + +// ListProjects returns all projects ordered by name. +func (db *DB) ListProjects() ([]Project, error) { + projects := []Project{} + if err := db.Select(&projects, projectColumns+` FROM projects pr ORDER BY pr.name`); err != nil { + return nil, err + } + return projects, nil +} + +// GetProject returns the named project, or nil if there is none. +func (db *DB) GetProject(name string) (*Project, error) { + var p Project + err := db.Get(&p, db.Rebind(projectColumns+` FROM projects pr WHERE pr.name = ?`), name) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &p, nil +} + +// GetProjectComponents returns the components of a project, ordered by +// ecosystem, name and version, with the package record the proxy holds for +// each. +func (db *DB) GetProjectComponents(projectID int64) ([]ProjectComponentInfo, error) { + components := []ProjectComponentInfo{} + err := db.Select(&components, db.Rebind(` + SELECT c.package_purl, c.ecosystem, c.name, c.version, + p.ecosystem AS cached_ecosystem, p.name AS cached_name, p.latest_version + FROM project_components c + LEFT JOIN packages p ON p.purl = c.package_purl + WHERE c.project_id = ? + ORDER BY c.ecosystem, c.name, c.version + `), projectID) + if err != nil { + return nil, err + } + return components, nil +} + +// GetProjectsUsingPackage returns the projects using a package, or only +// those using version when it is not empty, ordered by project and version. +func (db *DB) GetProjectsUsingPackage(packagePURL, version string) ([]ProjectUsage, error) { + query := projectColumns + `, c.version + FROM project_components c + JOIN projects pr ON pr.id = c.project_id + WHERE c.package_purl = ?` + args := []any{packagePURL} + if version != "" { + query += ` AND c.version = ?` + args = append(args, version) + } + usages := []ProjectUsage{} + if err := db.Select(&usages, db.Rebind(query+` ORDER BY pr.name, c.version`), args...); err != nil { + return nil, err + } + return usages, nil +} + +// GetComponentVulnerabilities returns the vulnerabilities the proxy holds for +// packages that projects use, for one project or, with an empty name, for all +// of them. Vulnerabilities are recorded per package, so only packages the +// proxy has a record of are matched. +func (db *DB) GetComponentVulnerabilities(project string) ([]ComponentVulnerability, error) { + query := ` + SELECT pr.name AS project, c.package_purl, c.ecosystem, c.name, c.version, + p.ecosystem AS cached_ecosystem, p.name AS cached_name, + v.vuln_id, v.severity, v.fixed_version, v.cvss_score + FROM project_components c + JOIN projects pr ON pr.id = c.project_id + JOIN packages p ON p.purl = c.package_purl + JOIN vulnerabilities v ON v.ecosystem = p.ecosystem AND v.package_name = p.name` + var args []any + if project != "" { + query += ` WHERE pr.name = ?` + args = append(args, project) + } + vulns := []ComponentVulnerability{} + if err := db.Select(&vulns, db.Rebind(query+` ORDER BY c.package_purl, c.version, pr.name, v.vuln_id`), args...); err != nil { + return nil, err + } + return vulns, nil +} + +func nullString(s string) sql.NullString { + return sql.NullString{String: s, Valid: s != ""} +} diff --git a/internal/database/projects_test.go b/internal/database/projects_test.go new file mode 100644 index 00000000..4eaaf500 --- /dev/null +++ b/internal/database/projects_test.go @@ -0,0 +1,228 @@ +package database + +import ( + "database/sql" + "slices" + "testing" + "time" +) + +var ( + lodash20 = ProjectComponent{PackagePURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", Version: "4.17.20"} + lodash21 = ProjectComponent{PackagePURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", Version: "4.17.21"} + console = ProjectComponent{PackagePURL: "pkg:composer/symfony/console", Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1"} +) + +func saveProject(t *testing.T, db *DB, name string, at time.Time, components ...ProjectComponent) { + t.Helper() + if err := db.SaveProject(name, "", "", at, components); err != nil { + t.Fatalf("SaveProject(%s): %v", name, err) + } +} + +func usageNames(usages []ProjectUsage) []string { + var names []string + for _, u := range usages { + names = append(names, u.Name+"@"+u.Version) + } + return names +} + +func TestSaveProjectReplacesComponents(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now().UTC().Truncate(time.Second) + if err := db.SaveProject("acme/web", "abc123", "main", now, []ProjectComponent{lodash20, console, lodash20}); err != nil { + t.Fatal(err) + } + saveProject(t, db, "acme/web", now.Add(time.Minute), lodash21) + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + if project.Components != 1 || project.Ref.Valid || project.Branch.Valid { + t.Errorf("project = %+v, want one component and ref and branch cleared", project) + } + if !project.UploadedAt.Equal(now.Add(time.Minute)) { + t.Errorf("uploaded_at = %v, want %v", project.UploadedAt, now.Add(time.Minute)) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].ProjectComponent != lodash21 { + t.Errorf("components = %+v, want only lodash 4.17.21", components) + } + + missing, err := db.GetProject("acme/other") + if err != nil || missing != nil { + t.Errorf("GetProject(missing) = %v, %v", missing, err) + } + }) +} + +func TestGetProjectsUsingPackage(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/web", now, lodash20, console) + saveProject(t, db, "acme/api", now, lodash21) + saveProject(t, db, "acme/cli", now, console) + + all, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(all), []string{"acme/api@4.17.21", "acme/web@4.17.20"}; !slices.Equal(got, want) { + t.Errorf("any version = %v, want %v", got, want) + } + one, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "4.17.20") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(one), []string{"acme/web@4.17.20"}; !slices.Equal(got, want) { + t.Errorf("4.17.20 = %v, want %v", got, want) + } + none, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "1.0.0") + if err != nil || len(none) != 0 { + t.Errorf("unused version = %v, %v", none, err) + } + }) +} + +func TestListAndDeleteProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/web", now, lodash20, console) + saveProject(t, db, "acme/api", now) + + projects, err := db.ListProjects() + if err != nil { + t.Fatal(err) + } + if len(projects) != 2 || projects[0].Name != "acme/api" || projects[0].Components != 0 || projects[1].Components != 2 { + t.Fatalf("projects = %+v", projects) + } + + deleted, err := db.DeleteProject("acme/web") + if err != nil || !deleted { + t.Fatalf("DeleteProject = %v, %v", deleted, err) + } + deleted, err = db.DeleteProject("acme/web") + if err != nil || deleted { + t.Errorf("second DeleteProject = %v, %v, want false", deleted, err) + } + var rows int + if err := db.Get(&rows, `SELECT COUNT(*) FROM project_components`); err != nil { + t.Fatal(err) + } + if rows != 0 { + t.Errorf("%d component rows left after the delete", rows) + } + }) +} + +func TestPruneProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/old", now.Add(-100*24*time.Hour), lodash20) + saveProject(t, db, "acme/new", now.Add(-time.Hour), lodash21) + + pruned, err := db.PruneProjects(now.Add(-90 * 24 * time.Hour)) + if err != nil { + t.Fatal(err) + } + if pruned != 1 { + t.Errorf("pruned = %d, want 1", pruned) + } + projects, err := db.ListProjects() + if err != nil { + t.Fatal(err) + } + if len(projects) != 1 || projects[0].Name != "acme/new" { + t.Errorf("projects after prune = %+v", projects) + } + usages, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(usages), []string{"acme/new@4.17.21"}; !slices.Equal(got, want) { + t.Errorf("usages after prune = %v, want %v", got, want) + } + }) +} + +func TestProjectComponentsJoinPackages(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + if err := db.UpsertPackage(&Package{ + PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", + LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } + if err := db.UpsertVulnerability(&Vulnerability{ + VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", + Severity: sql.NullString{String: "high", Valid: true}, + FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } + saveProject(t, db, "acme/web", time.Now(), lodash20, console) + saveProject(t, db, "acme/api", time.Now(), lodash21) + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) != 2 { + t.Fatalf("components = %+v", components) + } + // Ordered by ecosystem: composer first, which the proxy has no record of. + if components[0].CachedName.Valid { + t.Errorf("uncached component has a package record: %+v", components[0]) + } + if components[1].CachedName.String != "lodash" || components[1].LatestVersion.String != "4.17.21" { + t.Errorf("cached component = %+v", components[1]) + } + + vulns, err := db.GetComponentVulnerabilities("acme/web") + if err != nil { + t.Fatal(err) + } + if len(vulns) != 1 || vulns[0].Version != "4.17.20" || vulns[0].VulnID != "GHSA-35jh-r3h4-6jhm" { + t.Errorf("acme/web vulnerabilities = %+v", vulns) + } + all, err := db.GetComponentVulnerabilities("") + if err != nil { + t.Fatal(err) + } + if len(all) != 2 { + t.Errorf("all vulnerabilities = %+v, want one row per project", all) + } + }) +} + +func TestMigrateAddsProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + // A database from before the project inventory. + for _, table := range []string{"project_components", "projects"} { + if _, err := db.Exec("DROP TABLE " + table); err != nil { + t.Fatalf("dropping %s: %v", table, err) + } + } + if _, err := db.Exec(db.Rebind("DELETE FROM migrations WHERE name = ?"), "020_add_projects"); err != nil { + t.Fatalf("forgetting the migration: %v", err) + } + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + saveProject(t, db, "acme/web", time.Now(), lodash20) + usages, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "4.17.20") + if err != nil || len(usages) != 1 { + t.Fatalf("GetProjectsUsingPackage after migration = %v, %v", usages, err) + } + }) +} diff --git a/internal/database/schema.go b/internal/database/schema.go index f0ec6516..861429ec 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -154,6 +154,26 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS projects ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at DATETIME NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name); +CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at); + +CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) +); +CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at DATETIME NOT NULL @@ -297,6 +317,26 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms BIGINT NOT NULL ); +CREATE TABLE IF NOT EXISTS projects ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at TIMESTAMP NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name); +CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at); + +CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) +); +CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at TIMESTAMP NOT NULL @@ -468,6 +508,7 @@ var migrations = []migration{ {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, + {"020_add_projects", migrateAddProjects}, } // isTableNotFound returns true if the error indicates a missing table. @@ -820,6 +861,41 @@ func migrateAddUpstreamHolds(db *DB) error { return nil } +// migrateAddProjects creates the tables of the project inventory: the +// projects that uploaded an SBOM and the package versions each one uses. +func migrateAddProjects(db *DB) error { + id, ts := "INTEGER PRIMARY KEY", sqliteDatetime + if db.dialect == DialectPostgres { + id, ts = "SERIAL PRIMARY KEY", postgresTimestamp + } + statements := []string{ + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS projects ( + id %s, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at %s NOT NULL + )`, id, ts), + `CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name)`, + `CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at)`, + `CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) + )`, + `CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version)`, + } + for _, stmt := range statements { + if _, err := db.Exec(stmt); err != nil { + return fmt.Errorf("creating project tables: %w", err) + } + } + return nil +} + // migrateAddPendingDeletesQueuedAtIndex indexes the column reclaim orders // the queue by, which grows large after a retention sweep. func migrateAddPendingDeletesQueuedAtIndex(db *DB) error { diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go new file mode 100644 index 00000000..954d46e4 --- /dev/null +++ b/internal/inventory/inventory.go @@ -0,0 +1,185 @@ +// Package inventory turns SBOMs uploaded for a project into the package +// versions the proxy records for it, named the way the proxy's handlers name +// the packages they cache, so the two can be joined. +package inventory + +import ( + "errors" + "fmt" + "strings" + + "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/purl" + "github.com/git-pkgs/sbom" + "github.com/git-pkgs/vers" +) + +// MaxProjectNameLength bounds a project name, such as a GitLab project path. +const MaxProjectNameLength = 255 + +// Component is one package version a project uses. +type Component struct { + // Ecosystem is the name the proxy's handlers store the package under, + // e.g. "npm", "composer", "golang" or "alpine". + Ecosystem string + // Name is the ecosystem-native package name, e.g. "@babel/core", + // "symfony/console" or "org.slf4j:slf4j-api". + Name string + // Version is the version as the SBOM gives it; it may be empty. + Version string + // PackagePURL is the unversioned package URL the proxy records the + // package under, so it matches packages.purl. + PackagePURL string +} + +// ecosystems maps the PURL types the proxy serves to the ecosystem name its +// handlers store packages under. +var ecosystems = map[string]string{ + "alpm": "arch", + "apk": "alpine", + "bsd": "freebsd", + "cargo": "cargo", + "chef": "chef", + "composer": "composer", + "conan": "conan", + "conda": "conda", + "cran": "cran", + "deb": "deb", + "gem": "gem", + "golang": "golang", + "hex": "hex", + "julia": "julia", + "maven": "maven", + "npm": "npm", + "nuget": "nuget", + "pub": "pub", + "pypi": "pypi", + "rpm": "rpm", + "swift": "swift", +} + +// distroTypes are PURL types whose namespace names the distribution +// ("debian", "alpine"), not part of the package name the proxy stores. +var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} + +// ErrUnsupportedPURL is returned for a package URL whose type the proxy +// doesn't serve. +var ErrUnsupportedPURL = errors.New("package type is not served by the proxy") + +// FromPURL maps a package URL to a component. Qualifiers and subpath are +// dropped; the version is kept as given. +func FromPURL(s string) (Component, error) { + p, err := purl.Parse(s) + if err != nil { + return Component{}, err + } + ecosystem, ok := ecosystems[p.Type] + if !ok { + return Component{}, fmt.Errorf("%w: %s", ErrUnsupportedPURL, p.Type) + } + name := p.FullName() + switch { + case distroTypes[p.Type]: + name = p.Name + case p.Type == "conda" && p.Namespace == "" && p.Qualifier("channel") != "": + name = p.Qualifier("channel") + "/" + p.Name + } + packagePURL, _ := packageurl.MakeCacheStrings(ecosystem, name, "") + if packagePURL == "" { + return Component{}, fmt.Errorf("package URL %q has no valid package name", s) + } + return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil +} + +// Parse reads a CycloneDX (JSON or XML) or SPDX SBOM and returns the package +// versions it lists, without duplicates. Packages without a package URL, or +// of a type the proxy doesn't serve, are counted in skipped. +func Parse(data []byte) (components []Component, skipped int, err error) { + doc, err := sbom.Parse(data) + if err != nil { + return nil, 0, err + } + seen := make(map[[2]string]bool, len(doc.Packages)) + for _, pkg := range doc.Packages { + c, ok := componentOf(pkg) + if !ok { + skipped++ + continue + } + key := [2]string{c.PackagePURL, c.Version} + if seen[key] { + continue + } + seen[key] = true + components = append(components, c) + } + return components, skipped, nil +} + +func componentOf(pkg sbom.Package) (Component, bool) { + locator := pkg.PURL() + if locator == "" { + return Component{}, false + } + c, err := FromPURL(locator) + if err != nil { + return Component{}, false + } + if c.Version == "" { + c.Version = pkg.Version + } + return c, true +} + +// ValidateProjectName checks a project name such as "group/sub/repo": one or +// more segments of letters, digits, ".", "_" and "-", none of them "." or +// "..". +func ValidateProjectName(name string) error { + if name == "" { + return errors.New("project name is empty") + } + if len(name) > MaxProjectNameLength { + return fmt.Errorf("project name is longer than %d characters", MaxProjectNameLength) + } + for segment := range strings.SplitSeq(name, "/") { + if segment == "" || segment == "." || segment == ".." { + return fmt.Errorf("project name %q has an empty, \".\" or \"..\" segment", name) + } + for _, r := range segment { + if !validNameRune(r) { + return fmt.Errorf("project name %q may only contain letters, digits, '.', '_', '-' and '/'", name) + } + } + } + return nil +} + +func validNameRune(r rune) bool { + return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '.' || r == '_' || r == '-' +} + +// CompareVersions compares two versions of a package in ecosystem with the +// ecosystem's version rules, returning -1, 0 or 1. +func CompareVersions(ecosystem, a, b string) int { + return vers.CompareWithScheme(a, b, purl.EcosystemToPURLType(ecosystem)) +} + +// Outdated reports whether version is lower than latest. +func Outdated(ecosystem, version, latest string) bool { + if version == "" || latest == "" || version == latest { + return false + } + return CompareVersions(ecosystem, version, latest) < 0 +} + +// Affected reports whether a vulnerability fixed in fixedVersion affects +// version. A vulnerability without a fixed version affects every version. +func Affected(ecosystem, version, fixedVersion string) bool { + if fixedVersion == "" { + return true + } + if version == "" { + return false + } + return CompareVersions(ecosystem, version, fixedVersion) < 0 +} diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go new file mode 100644 index 00000000..3d55d506 --- /dev/null +++ b/internal/inventory/inventory_test.go @@ -0,0 +1,133 @@ +package inventory + +import ( + "errors" + "os" + "slices" + "strings" + "testing" +) + +func TestParseSyftCycloneDX(t *testing.T) { + data, err := os.ReadFile("testdata/syft-cyclonedx.json") + if err != nil { + t.Fatal(err) + } + components, skipped, err := Parse(data) + if err != nil { + t.Fatalf("Parse: %v", err) + } + want := []Component{ + {Ecosystem: "npm", Name: "@babel/core", Version: "7.23.0", PackagePURL: "pkg:npm/%40babel/core"}, + {Ecosystem: "alpine", Name: "busybox", Version: "1.36.1-r15", PackagePURL: "pkg:apk/alpine/busybox"}, + {Ecosystem: "golang", Name: "github.com/inconshreveable/mousetrap", Version: "v1.1.0", PackagePURL: "pkg:golang/github.com/inconshreveable/mousetrap"}, + {Ecosystem: "golang", Name: "github.com/spf13/cobra", Version: "v1.8.0", PackagePURL: "pkg:golang/github.com/spf13/cobra"}, + {Ecosystem: "golang", Name: "github.com/spf13/pflag", Version: "v1.0.5", PackagePURL: "pkg:golang/github.com/spf13/pflag"}, + {Ecosystem: "npm", Name: "lodash", Version: "4.17.20", PackagePURL: "pkg:npm/lodash"}, + {Ecosystem: "composer", Name: "monolog/monolog", Version: "3.5.0", PackagePURL: "pkg:composer/monolog/monolog"}, + {Ecosystem: "alpine", Name: "musl", Version: "1.2.4-r2", PackagePURL: "pkg:apk/alpine/musl"}, + {Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1", PackagePURL: "pkg:composer/symfony/console"}, + {Ecosystem: "npm", Name: "webapp", Version: "1.0.0", PackagePURL: "pkg:npm/webapp"}, + } + if !slices.Equal(components, want) { + t.Errorf("components =\n%v\nwant\n%v", components, want) + } + // The operating system and the four lock files syft lists carry no PURL. + if skipped != 5 { + t.Errorf("skipped = %d, want 5", skipped) + } +} + +func TestParseDeduplicatesAndSkipsUnknownTypes(t *testing.T) { + doc := `{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}, + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}, + {"type":"library","name":"lodash","version":"4.17.20","purl":"pkg:npm/lodash@4.17.20"}, + {"type":"library","name":"checkout","version":"v4","purl":"pkg:github/actions/checkout@v4"}, + {"type":"library","name":"Django","version":"4.2.0","purl":"pkg:pypi/Django"}, + {"type":"library","name":"guava","version":"32.1.3-jre","purl":"pkg:maven/com.google.guava/guava@32.1.3-jre?type=jar"}, + {"type":"library","name":"libc6","version":"2.36-9","purl":"pkg:deb/debian/libc6@2.36-9?arch=amd64&distro=debian-12"}, + {"type":"library","name":"broken","purl":"not a purl"} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + want := []Component{ + {Ecosystem: "npm", Name: "lodash", Version: "4.17.21", PackagePURL: "pkg:npm/lodash"}, + {Ecosystem: "npm", Name: "lodash", Version: "4.17.20", PackagePURL: "pkg:npm/lodash"}, + {Ecosystem: "pypi", Name: "django", Version: "4.2.0", PackagePURL: "pkg:pypi/django"}, + {Ecosystem: "maven", Name: "com.google.guava:guava", Version: "32.1.3-jre", PackagePURL: "pkg:maven/com.google.guava/guava"}, + {Ecosystem: "deb", Name: "libc6", Version: "2.36-9", PackagePURL: "pkg:deb/libc6"}, + } + if !slices.Equal(components, want) { + t.Errorf("components =\n%v\nwant\n%v", components, want) + } + if skipped != 2 { + t.Errorf("skipped = %d, want 2", skipped) + } +} + +func TestParseRejectsNonSBOM(t *testing.T) { + if _, _, err := Parse([]byte(`{"hello":"world"}`)); err == nil { + t.Error("Parse accepted a document that is not an SBOM") + } +} + +func TestFromPURLUnsupported(t *testing.T) { + _, err := FromPURL("pkg:github/actions/checkout") + if !errors.Is(err, ErrUnsupportedPURL) { + t.Errorf("err = %v, want ErrUnsupportedPURL", err) + } +} + +func TestValidateProjectName(t *testing.T) { + valid := []string{"repo", "group/repo", "group/sub/my_repo-2.0", strings.Repeat("a", MaxProjectNameLength)} + for _, name := range valid { + if err := ValidateProjectName(name); err != nil { + t.Errorf("ValidateProjectName(%q) = %v, want nil", name, err) + } + } + invalid := []string{ + "", "/repo", "repo/", "group//repo", ".", "..", "group/../repo", "group/./repo", + "group/re po", "group/repo?x", "group\\repo", "grüppe/repo", "group/repo%2F", + strings.Repeat("a", MaxProjectNameLength+1), + } + for _, name := range invalid { + if err := ValidateProjectName(name); err == nil { + t.Errorf("ValidateProjectName(%q) = nil, want an error", name) + } + } +} + +func TestOutdatedAndAffected(t *testing.T) { + tests := []struct { + name string + ecosystem string + version string + other string + outdated bool + affected bool + }{ + {"npm older", "npm", "4.17.20", "4.17.21", true, true}, + {"npm same", "npm", "4.17.21", "4.17.21", false, false}, + {"npm newer", "npm", "4.18.0", "4.17.21", false, false}, + {"composer v prefix", "composer", "v6.4.1", "v6.4.2", true, true}, + {"golang", "golang", "v1.8.0", "v1.10.0", true, true}, + {"alpine revisions", "alpine", "1.36.1-r15", "1.36.1-r2", false, false}, + {"empty version", "npm", "", "1.0.0", false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Outdated(tt.ecosystem, tt.version, tt.other); got != tt.outdated { + t.Errorf("Outdated = %v, want %v", got, tt.outdated) + } + if got := Affected(tt.ecosystem, tt.version, tt.other); got != tt.affected { + t.Errorf("Affected = %v, want %v", got, tt.affected) + } + }) + } + if !Affected("npm", "1.0.0", "") { + t.Error("a vulnerability without a fix should affect every version") + } +} diff --git a/internal/inventory/testdata/syft-cyclonedx.json b/internal/inventory/testdata/syft-cyclonedx.json new file mode 100644 index 00000000..7647d689 --- /dev/null +++ b/internal/inventory/testdata/syft-cyclonedx.json @@ -0,0 +1 @@ +{"$schema":"http://cyclonedx.org/schema/bom-1.7.schema.json","bomFormat":"CycloneDX","specVersion":"1.7","serialNumber":"urn:uuid:d69cf664-a43e-4d65-93e3-e43e24d33c0e","version":1,"metadata":{"timestamp":"2026-10-27T19:17:53+01:00","tools":{"components":[{"type":"application","author":"anchore","name":"syft","version":"1.55.0"}]},"component":{"bom-ref":"af63bd4c8601b7f1","type":"file","name":"."}},"components":[{"bom-ref":"pkg:npm/%40babel/core@7.23.0?package-id=7b01fd7a9ce63128","type":"library","name":"@babel/core","version":"7.23.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:\\@babel\\/core:\\@babel\\/core:7.23.0:*:*:*:*:*:*:*","purl":"pkg:npm/%40babel/core@7.23.0","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64&distro=alpine-3.19.1&package-id=464f552f6c89c192","type":"library","publisher":"Sören Tempel ","name":"busybox","version":"1.36.1-r15","description":"Size optimized toolbox of many common UNIX utilities","licenses":[{"license":{"id":"GPL-2.0-only"}}],"cpe":"cpe:2.3:a:busybox:busybox:1.36.1-r15:*:*:*:*:*:*:*","purl":"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64&distro=alpine-3.19.1","externalReferences":[{"url":"https://busybox.net/","type":"distribution"}],"properties":[{"name":"syft:package:foundBy","value":"apk-db-cataloger"},{"name":"syft:package:type","value":"apk"},{"name":"syft:package:metadataType","value":"apk-db-entry"},{"name":"syft:location:0:path","value":"/lib/apk/db/installed"},{"name":"syft:metadata:gitCommitOfApkPort","value":"1b0b2f6e7d0e6a5b4c3d2e1f0a9b8c7d6e5f4a3b"},{"name":"syft:metadata:installedSize","value":"950272"},{"name":"syft:metadata:originPackage","value":"busybox"},{"name":"syft:metadata:pullChecksum","value":"Q1Xy0qB0j7Y7uQ2kGmWbN8vT6Ff0A4E="},{"name":"syft:metadata:pullDependencies:0","value":"so:libc.musl-x86_64.so.1"},{"name":"syft:metadata:size","value":"509268"}]},{"bom-ref":"pkg:golang/github.com/inconshreveable/mousetrap@v1.1.0?package-id=f6300927a9f6b106","type":"library","name":"github.com/inconshreveable/mousetrap","version":"v1.1.0","cpe":"cpe:2.3:a:inconshreveable:mousetrap:v1.1.0:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/inconshreveable/mousetrap@v1.1.0","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8="}]},{"bom-ref":"pkg:golang/github.com/spf13/cobra@v1.8.0?package-id=81b06fbcbe87f414","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","cpe":"cpe:2.3:a:spf13:cobra:v1.8.0:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0="}]},{"bom-ref":"pkg:golang/github.com/spf13/pflag@v1.0.5?package-id=163f67459b76e534","type":"library","name":"github.com/spf13/pflag","version":"v1.0.5","cpe":"cpe:2.3:a:spf13:pflag:v1.0.5:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/spf13/pflag@v1.0.5","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRPuqXzS0rkr6qZw="}]},{"bom-ref":"pkg:npm/lodash@4.17.20?package-id=ece3658907628a6b","type":"library","name":"lodash","version":"4.17.20","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:lodash:lodash:4.17.20:*:*:*:*:node.js:*:*","purl":"pkg:npm/lodash@4.17.20","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"pkg:composer/monolog/monolog@3.5.0?package-id=3abfdcdd00fd8d2f","type":"library","name":"monolog/monolog","version":"3.5.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:monolog\\/monolog:monolog\\/monolog:3.5.0:*:*:*:*:*:*:*","purl":"pkg:composer/monolog/monolog@3.5.0","properties":[{"name":"syft:package:foundBy","value":"php-composer-lock-cataloger"},{"name":"syft:package:language","value":"php"},{"name":"syft:package:type","value":"php-composer"},{"name":"syft:package:metadataType","value":"php-composer-lock-entry"},{"name":"syft:location:0:path","value":"/composer.lock"}]},{"bom-ref":"pkg:apk/alpine/musl@1.2.4-r2?arch=x86_64&distro=alpine-3.19.1&package-id=9f0e3d80c35c10bc","type":"library","publisher":"Timo Teräs ","name":"musl","version":"1.2.4-r2","description":"the musl c library (libc) implementation","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:musl-libc:musl:1.2.4-r2:*:*:*:*:*:*:*","purl":"pkg:apk/alpine/musl@1.2.4-r2?arch=x86_64&distro=alpine-3.19.1","externalReferences":[{"url":"https://musl.libc.org/","type":"distribution"}],"properties":[{"name":"syft:package:foundBy","value":"apk-db-cataloger"},{"name":"syft:package:type","value":"apk"},{"name":"syft:package:metadataType","value":"apk-db-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:musl_libc:musl:1.2.4-r2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:musl:musl:1.2.4-r2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/lib/apk/db/installed"},{"name":"syft:metadata:gitCommitOfApkPort","value":"4b2f0d7b4e0e8a4b0e1b1b4d2b8a7a3d1a2f8e0c"},{"name":"syft:metadata:installedSize","value":"647168"},{"name":"syft:metadata:originPackage","value":"musl"},{"name":"syft:metadata:pullChecksum","value":"Q1N9CgYbNRm8SgkqbXqJhCAc6iqXo="},{"name":"syft:metadata:size","value":"407383"}]},{"bom-ref":"pkg:composer/symfony/console@v6.4.1?package-id=767524bf09da7da7","type":"library","name":"symfony/console","version":"v6.4.1","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:symfony\\/console:symfony\\/console:v6.4.1:*:*:*:*:*:*:*","purl":"pkg:composer/symfony/console@v6.4.1","properties":[{"name":"syft:package:foundBy","value":"php-composer-lock-cataloger"},{"name":"syft:package:language","value":"php"},{"name":"syft:package:type","value":"php-composer"},{"name":"syft:package:metadataType","value":"php-composer-lock-entry"},{"name":"syft:location:0:path","value":"/composer.lock"}]},{"bom-ref":"pkg:npm/webapp@1.0.0?package-id=982a05090388c92d","type":"library","name":"webapp","version":"1.0.0","cpe":"cpe:2.3:a:webapp:webapp:1.0.0:*:*:*:*:*:*:*","purl":"pkg:npm/webapp@1.0.0","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"os:alpine@3.19.1","type":"operating-system","name":"alpine","version":"3.19.1","description":"Alpine Linux v3.19","swid":{"tagId":"alpine","name":"alpine","version":"3.19.1"},"properties":[{"name":"syft:distro:extendedSupport","value":"false"},{"name":"syft:distro:id","value":"alpine"},{"name":"syft:distro:prettyName","value":"Alpine Linux v3.19"},{"name":"syft:distro:versionID","value":"3.19.1"}]},{"bom-ref":"c8bb62e6bded633e","type":"file","name":"/builds/acme/webapp/composer.lock","hashes":[{"alg":"SHA-1","content":"53c59ca459a8ff42918b5ed4fdcea1d8ba98432f"},{"alg":"SHA-256","content":"63738301cc7186a6645f00ae0a63ae6347e04552489f3cc70e92200463f5b4ac"}]},{"bom-ref":"3fc5a8d3d86e9790","type":"file","name":"/builds/acme/webapp/go.mod","hashes":[{"alg":"SHA-1","content":"c67c41265c415c41049b31f850b4497898eff1fd"},{"alg":"SHA-256","content":"f2303d4139e491238d8cb9a6c448e4c366b82bae3a646164465387df76e44f32"}]},{"bom-ref":"9f5aca292136191a","type":"file","name":"/builds/acme/webapp/lib/apk/db/installed","hashes":[{"alg":"SHA-1","content":"b2ebf78ea604d3a76359a262afd401ab966a36a1"},{"alg":"SHA-256","content":"d363cb96812dc6eb1d6f4c00879c2564cdbff238b6c1d0a37701d2adedd326e0"}]},{"bom-ref":"fd71c2238fc07657","type":"file","name":"/builds/acme/webapp/package-lock.json","hashes":[{"alg":"SHA-1","content":"fc26eadd11405354163eb0f53ffd2bb8cb56c5e7"},{"alg":"SHA-256","content":"69094786dd997053d1c7a81d4349bb5b608c08b576e67360f07cdc2726a041ea"}]}],"dependencies":[{"ref":"pkg:npm/webapp@1.0.0?package-id=982a05090388c92d","dependsOn":["pkg:npm/%40babel/core@7.23.0?package-id=7b01fd7a9ce63128","pkg:npm/lodash@4.17.20?package-id=ece3658907628a6b"]}]} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index da866a6a..ebcc3331 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -15,6 +15,9 @@ type DashboardData struct { EnrichmentStats EnrichmentStatsView RecentPackages []PackageInfo PopularPackages []PackageInfo + // VulnerableInUse lists vulnerable package versions projects use, when + // the project inventory is enabled. + VulnerableInUse []VulnerableVersion } // DashboardStats contains cache statistics for the dashboard. @@ -68,6 +71,9 @@ type PackageShowData struct { Versions []database.Version Vulnerabilities []database.Vulnerability LicenseCategory string + // UsedBy lists the projects using each version, when the project + // inventory is enabled. + UsedBy []VersionUsage } // VersionShowData contains data for rendering the version show page. diff --git a/internal/server/layout.go b/internal/server/layout.go index 3a518768..1a63d2ff 100644 --- a/internal/server/layout.go +++ b/internal/server/layout.go @@ -17,6 +17,8 @@ type Layout struct { CanonicalPath string // CacheDelete shows the controls for deleting cached artifacts. CacheDelete bool + // Projects shows the project inventory pages. + Projects bool } func (s *Server) layoutFor(r *http.Request) Layout { @@ -25,5 +27,6 @@ func (s *Server) layoutFor(r *http.Request) Layout { UIBaseURL: s.cfg.UIBaseURL, CanonicalPath: r.URL.Path, CacheDelete: s.cfg.CacheDelete, + Projects: s.cfg.Projects.Enabled, } } diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go new file mode 100644 index 00000000..a0073193 --- /dev/null +++ b/internal/server/projects_api.go @@ -0,0 +1,335 @@ +package server + +import ( + "errors" + "io" + "log/slog" + "net/http" + "net/url" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/inventory" + "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/go-chi/chi/v5" +) + +// maxSBOMBytes caps an uploaded SBOM. syft's CycloneDX output for a large +// monorepo is a few MiB. +const maxSBOMBytes = 32 << 20 + +// maxRefLength caps the ref and branch query parameters of an upload. +const maxRefLength = 255 + +// sbomPathSuffix ends the upload path, /api/projects/{project}/sbom. +const sbomPathSuffix = "/sbom" + +// ProjectsAPIHandler serves /api/projects, where repositories upload the SBOM +// of their default branch so the proxy can show which of them use which +// package versions. +type ProjectsAPIHandler struct { + db *database.DB + logger *slog.Logger +} + +// NewProjectsAPIHandler creates the /api/projects handler. +func NewProjectsAPIHandler(db *database.DB, logger *slog.Logger) *ProjectsAPIHandler { + return &ProjectsAPIHandler{db: db, logger: logger} +} + +// ProjectUploadResponse summarizes an uploaded SBOM. +type ProjectUploadResponse struct { + Project string `json:"project"` + // Components is the number of package versions recorded. + Components int `json:"components"` + // Skipped is the number of SBOM entries without a package URL or of a + // package type the proxy doesn't serve. + Skipped int `json:"skipped"` +} + +// ProjectSummary describes a project in the inventory. +type ProjectSummary struct { + Name string `json:"name"` + Ref string `json:"ref,omitempty"` + Branch string `json:"branch,omitempty"` + UploadedAt time.Time `json:"uploaded_at"` + // ComponentCount is the number of package versions recorded. + ComponentCount int64 `json:"component_count"` + // Version is set when listing the projects using a package: the version + // of it this project uses. + Version string `json:"version,omitempty"` +} + +// ProjectListResponse lists projects. +type ProjectListResponse struct { + Projects []ProjectSummary `json:"projects"` +} + +// ProjectComponent is a package version a project uses. +type ProjectComponent struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + Version string `json:"version"` + PURL string `json:"purl"` +} + +// ProjectResponse is a project with its components. +type ProjectResponse struct { + ProjectSummary + Components []ProjectComponent `json:"components"` +} + +// mountProjectAPI registers /api/projects when projects.enabled is set. +func (s *Server) mountProjectAPI(r chi.Router) { + if s.cfg.Projects.Enabled { + registerProjectRoutes(r, NewProjectsAPIHandler(s.db, s.logger), s.cfg.Projects.TokenExpanded()) + } +} + +// registerProjectRoutes mounts /api/projects. Reading is open like the rest +// of the API; uploading and deleting require the bearer token. +// +// A project name such as "group/sub/repo" is accepted both as path segments, +// /api/projects/group/sub/repo, and URL-encoded as one segment, +// /api/projects/group%2Fsub%2Frepo, which is how GitLab's CI_PROJECT_PATH +// usually ends up when a client escapes it. +func registerProjectRoutes(r chi.Router, h *ProjectsAPIHandler, token string) { + r.Get("/api/projects", h.HandleList) + r.Get("/api/projects/*", h.HandleGet) + r.Group(func(w chi.Router) { + w.Use(requireBearerToken(token)) + w.Put("/api/projects/*", h.HandleUpload) + w.Delete("/api/projects/*", h.HandleDelete) + }) +} + +// HandleUpload handles PUT /api/projects/{project}/sbom. +// @Summary Upload a project's SBOM +// @Description Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment. +// @Tags projects +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param project path string true "Project path, e.g. group/sub/repo" +// @Param ref query string false "Commit the SBOM was generated from" +// @Param branch query string false "Branch the SBOM was generated from" +// @Success 200 {object} ProjectUploadResponse +// @Failure 400 {object} ErrorResponse +// @Failure 401 {object} ErrorResponse +// @Failure 413 {object} ErrorResponse +// @Router /api/projects/{project}/sbom [put] +func (h *ProjectsAPIHandler) HandleUpload(w http.ResponseWriter, r *http.Request) { + path, err := projectPath(r) + if err != nil { + badRequest(w, err.Error()) + return + } + name, ok := strings.CutSuffix(path, sbomPathSuffix) + if !ok { + badRequest(w, "path must be /api/projects/{project}/sbom") + return + } + if err := inventory.ValidateProjectName(name); err != nil { + badRequest(w, err.Error()) + return + } + ref, branch := r.URL.Query().Get("ref"), r.URL.Query().Get("branch") + if len(ref) > maxRefLength || len(branch) > maxRefLength { + badRequest(w, "ref and branch must not be longer than 255 characters") + return + } + + data, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxSBOMBytes)) + var tooLarge *http.MaxBytesError + switch { + case errors.As(err, &tooLarge): + writeError(w, http.StatusRequestEntityTooLarge, ErrCodeBadRequest, "SBOM is larger than 32 MiB") + return + case err != nil: + badRequest(w, "failed to read request body") + return + case len(data) == 0: + badRequest(w, "request body is empty") + return + } + + components, skipped, err := inventory.Parse(data) + if err != nil { + badRequest(w, "request body is not a CycloneDX or SPDX SBOM: "+err.Error()) + return + } + records := make([]database.ProjectComponent, len(components)) + for i, c := range components { + records[i] = database.ProjectComponent{ + PackagePURL: c.PackagePURL, Ecosystem: c.Ecosystem, Name: c.Name, Version: c.Version, + } + } + if err := h.db.SaveProject(name, ref, branch, time.Now(), records); err != nil { + h.logger.Error("failed to save project", "project", name, "error", err) + internalError(w, "failed to save project") + return + } + h.logger.Info("recorded project SBOM", "project", name, "ref", ref, + "components", len(records), "skipped", skipped) + writeJSON(w, ProjectUploadResponse{Project: name, Components: len(records), Skipped: skipped}) +} + +// HandleDelete handles DELETE /api/projects/{project}. +// @Summary Delete a project +// @Description Removes the project and the package versions recorded for it. +// @Tags projects +// @Security BearerAuth +// @Param project path string true "Project path, e.g. group/sub/repo" +// @Success 204 +// @Failure 400 {object} ErrorResponse +// @Failure 401 {object} ErrorResponse +// @Failure 404 {object} ErrorResponse +// @Router /api/projects/{project} [delete] +func (h *ProjectsAPIHandler) HandleDelete(w http.ResponseWriter, r *http.Request) { + name, ok := validProjectPath(w, r) + if !ok { + return + } + deleted, err := h.db.DeleteProject(name) + switch { + case err != nil: + h.logger.Error("failed to delete project", "project", name, "error", err) + internalError(w, "failed to delete project") + case !deleted: + notFound(w, "project not found") + default: + h.logger.Info("deleted project", "project", name) + w.WriteHeader(http.StatusNoContent) + } +} + +// HandleList handles GET /api/projects. +// @Summary List projects +// @Description Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses. +// @Tags projects +// @Produce json +// @Param purl query string false "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20" +// @Success 200 {object} ProjectListResponse +// @Failure 400 {object} ErrorResponse +// @Router /api/projects [get] +func (h *ProjectsAPIHandler) HandleList(w http.ResponseWriter, r *http.Request) { + if query := r.URL.Query().Get("purl"); query != "" { + h.listUsages(w, query) + return + } + projects, err := h.db.ListProjects() + if err != nil { + h.logger.Error("failed to list projects", "error", err) + internalError(w, "failed to list projects") + return + } + resp := ProjectListResponse{Projects: make([]ProjectSummary, len(projects))} + for i, p := range projects { + resp.Projects[i] = projectSummary(p) + } + writeJSON(w, resp) +} + +func (h *ProjectsAPIHandler) listUsages(w http.ResponseWriter, query string) { + c, err := inventory.FromPURL(query) + if err != nil { + badRequest(w, "invalid purl: "+err.Error()) + return + } + usages, err := h.db.GetProjectsUsingPackage(c.PackagePURL, c.Version) + if err != nil { + h.logger.Error("failed to look up package usage", "purl", query, "error", err) + internalError(w, "failed to look up package usage") + return + } + resp := ProjectListResponse{Projects: make([]ProjectSummary, len(usages))} + for i, u := range usages { + resp.Projects[i] = projectSummary(u.Project) + resp.Projects[i].Version = u.Version + } + writeJSON(w, resp) +} + +// HandleGet handles GET /api/projects/{project}. +// @Summary Get a project +// @Description Returns the project with the package versions recorded for it. +// @Tags projects +// @Produce json +// @Param project path string true "Project path, e.g. group/sub/repo" +// @Success 200 {object} ProjectResponse +// @Failure 400 {object} ErrorResponse +// @Failure 404 {object} ErrorResponse +// @Router /api/projects/{project} [get] +func (h *ProjectsAPIHandler) HandleGet(w http.ResponseWriter, r *http.Request) { + name, ok := validProjectPath(w, r) + if !ok { + return + } + project, err := h.db.GetProject(name) + if err != nil { + h.logger.Error("failed to get project", "project", name, "error", err) + internalError(w, "failed to get project") + return + } + if project == nil { + notFound(w, "project not found") + return + } + components, err := h.db.GetProjectComponents(project.ID) + if err != nil { + h.logger.Error("failed to get project components", "project", name, "error", err) + internalError(w, "failed to get project components") + return + } + resp := ProjectResponse{ProjectSummary: projectSummary(*project), Components: make([]ProjectComponent, len(components))} + for i, c := range components { + resp.Components[i] = ProjectComponent{ + Ecosystem: c.Ecosystem, + Name: c.Name, + Version: c.Version, + PURL: packageurl.WithVersionString(c.PackagePURL, c.Version), + } + } + writeJSON(w, resp) +} + +func projectSummary(p database.Project) ProjectSummary { + return ProjectSummary{ + Name: p.Name, + Ref: p.Ref.String, + Branch: p.Branch.String, + UploadedAt: p.UploadedAt, + ComponentCount: p.Components, + } +} + +// projectPath returns the decoded wildcard of a /api/projects or +// /ui/projects route. chi hands it over still encoded when the request path +// had escapes its canonical form doesn't (wildcardIsEncoded), e.g. a project +// name sent as one segment with %2F. +func projectPath(r *http.Request) (string, error) { + wildcard := chi.URLParam(r, "*") + if !wildcardIsEncoded(r) { + return wildcard, nil + } + decoded, err := url.PathUnescape(wildcard) + if err != nil { + return "", errors.New("invalid escape in project path") + } + return decoded, nil +} + +// validProjectPath returns the project named by the route, answering 400 if +// it is not a valid project name. +func validProjectPath(w http.ResponseWriter, r *http.Request) (string, bool) { + name, err := projectPath(r) + if err == nil { + err = inventory.ValidateProjectName(name) + } + if err != nil { + badRequest(w, err.Error()) + return "", false + } + return name, true +} diff --git a/internal/server/projects_api_test.go b/internal/server/projects_api_test.go new file mode 100644 index 00000000..880a4cf7 --- /dev/null +++ b/internal/server/projects_api_test.go @@ -0,0 +1,354 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/go-chi/chi/v5" +) + +const testProjectsToken = "sbom-token" + +// syftSBOM is a CycloneDX document syft produced for a directory with an +// npm, Composer and Go lock file and an Alpine package database. It lists +// ten packages; five entries (the OS and the lock files) have no PURL. +func syftSBOM(t *testing.T) []byte { + t.Helper() + data, err := os.ReadFile("../inventory/testdata/syft-cyclonedx.json") + if err != nil { + t.Fatal(err) + } + return data +} + +type projectsTestServer struct { + server *Server + router http.Handler + db *database.DB +} + +// newProjectsTestServer routes the project API and the UI pages that show +// the inventory, with projects.enabled set. +func newProjectsTestServer(t *testing.T) projectsTestServer { + t.Helper() + db, err := database.Create(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("database.Create: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + cfg := &config.Config{Projects: config.ProjectsConfig{Enabled: true, Token: testProjectsToken}} + s := &Server{cfg: cfg, db: db, logger: logger, templates: &Templates{}} + + r := chi.NewRouter() + registerProjectRoutes(r, NewProjectsAPIHandler(db, logger), testProjectsToken) + r.Route("/ui", func(ui chi.Router) { + ui.Get("/", s.handleRoot) + ui.Get("/package/{ecosystem}/*", s.handlePackagePath) + s.mountProjectUI(ui) + }) + return projectsTestServer{server: s, router: r, db: db} +} + +func (ts projectsTestServer) do(t *testing.T, method, target, token string, body []byte) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(method, target, bytes.NewReader(body)) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + ts.router.ServeHTTP(rec, req) + return rec +} + +func (ts projectsTestServer) upload(t *testing.T, target string, body []byte) ProjectUploadResponse { + t.Helper() + rec := ts.do(t, http.MethodPut, target, testProjectsToken, body) + if rec.Code != http.StatusOK { + t.Fatalf("PUT %s = %d: %s", target, rec.Code, rec.Body) + } + var resp ProjectUploadResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decoding upload response: %v", err) + } + return resp +} + +func decodeJSON(t *testing.T, rec *httptest.ResponseRecorder, v any) { + t.Helper() + if rec.Code != http.StatusOK { + t.Fatalf("status = %d: %s", rec.Code, rec.Body) + } + if err := json.Unmarshal(rec.Body.Bytes(), v); err != nil { + t.Fatalf("decoding response: %v", err) + } +} + +func TestProjectUploadAndGet(t *testing.T) { + ts := newProjectsTestServer(t) + + resp := ts.upload(t, "/api/projects/acme/shop/web/sbom?ref=0123abcd&branch=main", syftSBOM(t)) + if resp != (ProjectUploadResponse{Project: "acme/shop/web", Components: 10, Skipped: 5}) { + t.Errorf("upload response = %+v", resp) + } + + // The same project, URL-encoded as one segment. + var project ProjectResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme%2Fshop%2Fweb", "", nil), &project) + if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 10 { + t.Errorf("project = %+v", project.ProjectSummary) + } + var purls []string + for _, c := range project.Components { + purls = append(purls, c.PURL) + } + for _, want := range []string{"pkg:npm/%40babel/core@7.23.0", "pkg:composer/symfony/console@v6.4.1", "pkg:golang/github.com/spf13/cobra@v1.8.0", "pkg:apk/alpine/busybox@1.36.1-r15"} { + if !strings.Contains(strings.Join(purls, " "), want) { + t.Errorf("components %v do not include %s", purls, want) + } + } + + var list ProjectListResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects", "", nil), &list) + if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 10 { + t.Errorf("list = %+v", list) + } +} + +func TestProjectUploadReplacesComponents(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme%2Fweb/sbom?ref=one", syftSBOM(t)) + resp := ts.upload(t, "/api/projects/acme%2Fweb/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + if resp.Components != 1 || resp.Skipped != 0 { + t.Errorf("upload response = %+v", resp) + } + var project ProjectResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil), &project) + if len(project.Components) != 1 || project.Components[0].PURL != "pkg:npm/lodash@4.17.21" || project.Ref != "" { + t.Errorf("project after second upload = %+v", project) + } +} + +func TestProjectLookupByPURL(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + ts.upload(t, "/api/projects/acme/api/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + + tests := []struct { + purl string + want []string + }{ + {"pkg:npm/lodash", []string{"acme/api@4.17.21", "acme/web@4.17.20"}}, + {"pkg:npm/lodash@4.17.20", []string{"acme/web@4.17.20"}}, + {"pkg:npm/lodash@1.0.0", nil}, + {"pkg:npm/%40babel/core", []string{"acme/web@7.23.0"}}, + {"pkg:apk/alpine/musl@1.2.4-r2?arch=x86_64", []string{"acme/web@1.2.4-r2"}}, + } + for _, tt := range tests { + t.Run(tt.purl, func(t *testing.T) { + var list ProjectListResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects?purl="+url.QueryEscape(tt.purl), "", nil), &list) + var got []string + for _, p := range list.Projects { + got = append(got, p.Name+"@"+p.Version) + } + if strings.Join(got, ",") != strings.Join(tt.want, ",") { + t.Errorf("projects = %v, want %v", got, tt.want) + } + }) + } + + for _, bad := range []string{"lodash", "pkg:github/actions/checkout@v4"} { + if rec := ts.do(t, http.MethodGet, "/api/projects?purl="+url.QueryEscape(bad), "", nil); rec.Code != http.StatusBadRequest { + t.Errorf("purl %q status = %d, want 400", bad, rec.Code) + } + } +} + +func TestProjectWritesRequireToken(t *testing.T) { + ts := newProjectsTestServer(t) + for _, token := range []string{"", "wrong-token"} { + if rec := ts.do(t, http.MethodPut, "/api/projects/acme/web/sbom", token, syftSBOM(t)); rec.Code != http.StatusUnauthorized { + t.Errorf("upload with token %q status = %d, want 401", token, rec.Code) + } + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme/web", token, nil); rec.Code != http.StatusUnauthorized { + t.Errorf("delete with token %q status = %d, want 401", token, rec.Code) + } + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("project exists after unauthorized upload: status %d", rec.Code) + } +} + +func TestProjectDelete(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme%2Fweb", testProjectsToken, nil); rec.Code != http.StatusNoContent { + t.Fatalf("delete status = %d: %s", rec.Code, rec.Body) + } + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme/web", testProjectsToken, nil); rec.Code != http.StatusNotFound { + t.Errorf("second delete status = %d, want 404", rec.Code) + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("get after delete status = %d, want 404", rec.Code) + } +} + +func TestProjectPathsAreValidated(t *testing.T) { + ts := newProjectsTestServer(t) + uploads := []string{ + "/api/projects/sbom", + "/api/projects/acme/web", + "/api/projects/acme/../web/sbom", + "/api/projects/acme%2F..%2Fweb/sbom", + "/api/projects/..%2Fweb/sbom", + "/api/projects/acme//web/sbom", + "/api/projects/acme/./web/sbom", + "/api/projects/acme%20web/sbom", + "/api/projects/acme%2Fweb%3F/sbom", + "/api/projects/" + strings.Repeat("a", 256) + "/sbom", + } + for _, target := range uploads { + if rec := ts.do(t, http.MethodPut, target, testProjectsToken, syftSBOM(t)); rec.Code != http.StatusBadRequest { + t.Errorf("PUT %s status = %d, want 400", target, rec.Code) + } + } + for _, target := range []string{"/api/projects/", "/api/projects/acme/../web", "/api/projects/acme%2F..%2Fweb"} { + if rec := ts.do(t, http.MethodGet, target, "", nil); rec.Code != http.StatusBadRequest { + t.Errorf("GET %s status = %d, want 400", target, rec.Code) + } + if rec := ts.do(t, http.MethodDelete, target, testProjectsToken, nil); rec.Code != http.StatusBadRequest { + t.Errorf("DELETE %s status = %d, want 400", target, rec.Code) + } + } + if projects, err := ts.db.ListProjects(); err != nil || len(projects) != 0 { + t.Errorf("projects after invalid uploads = %v, %v", projects, err) + } +} + +func TestProjectUploadRejectsBadBodies(t *testing.T) { + ts := newProjectsTestServer(t) + tests := []struct { + name string + target string + body []byte + want int + }{ + {"empty", "/api/projects/acme/web/sbom", nil, http.StatusBadRequest}, + {"not an SBOM", "/api/projects/acme/web/sbom", []byte(`{"hello":"world"}`), http.StatusBadRequest}, + {"too large", "/api/projects/acme/web/sbom", make([]byte, maxSBOMBytes+1), http.StatusRequestEntityTooLarge}, + {"long ref", "/api/projects/acme/web/sbom?ref=" + strings.Repeat("a", maxRefLength+1), syftSBOM(t), http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if rec := ts.do(t, http.MethodPut, tt.target, testProjectsToken, tt.body); rec.Code != tt.want { + t.Errorf("status = %d, want %d: %s", rec.Code, tt.want, rec.Body) + } + }) + } +} + +// startProjectsServer runs the real server with the given projects config. +func startProjectsServer(t *testing.T, projects config.ProjectsConfig) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserving proxy address: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + + tempDir := t.TempDir() + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(tempDir, "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") + cfg.Projects = projects + if err := cfg.Validate(); err != nil { + t.Fatalf("validating config: %v", err) + } + + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{Version: "test", Commit: "test"}) + if err != nil { + t.Fatalf("creating server: %v", err) + } + startErr := make(chan error, 1) + go func() { startErr <- proxyServer.Start(listener) }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = proxyServer.Shutdown(ctx) + if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("server stopped with %v", err) + } + }) + return cfg.BaseURL +} + +func request(t *testing.T, method, url, token string, body []byte) (int, string) { + t.Helper() + req, err := http.NewRequest(method, url, bytes.NewReader(body)) + if err != nil { + t.Fatalf("building request: %v", err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, url, err) + } + defer func() { _ = resp.Body.Close() }() + data, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(data) +} + +func TestProjectRoutesFollowConfig(t *testing.T) { + disabled := startProjectsServer(t, config.ProjectsConfig{}) + for _, path := range []string{"/api/projects", "/api/projects/acme/web", "/ui/projects"} { + if status, _ := request(t, http.MethodGet, disabled+path, "", nil); status != http.StatusNotFound { + t.Errorf("GET %s with projects disabled = %d, want 404", path, status) + } + } + if status, _ := request(t, http.MethodPut, disabled+"/api/projects/acme/web/sbom", testProjectsToken, syftSBOM(t)); status != http.StatusNotFound { + t.Errorf("upload with projects disabled = %d, want 404", status) + } + if _, body := request(t, http.MethodGet, disabled+"/ui/", "", nil); strings.Contains(body, `href="/ui/projects"`) { + t.Error("navigation links to projects while they are disabled") + } + + enabled := startProjectsServer(t, config.ProjectsConfig{Enabled: true, Token: testProjectsToken, MaxAge: "90d"}) + if status, _ := request(t, http.MethodPut, enabled+"/api/projects/acme%2Fweb/sbom", "", syftSBOM(t)); status != http.StatusUnauthorized { + t.Errorf("upload without token = %d, want 401", status) + } + if status, body := request(t, http.MethodPut, enabled+"/api/projects/acme%2Fweb/sbom", testProjectsToken, syftSBOM(t)); status != http.StatusOK { + t.Fatalf("upload = %d: %s", status, body) + } + if status, body := request(t, http.MethodGet, enabled+"/api/projects/acme/web", "", nil); status != http.StatusOK || !strings.Contains(body, `"name":"acme/web"`) { + t.Errorf("GET project = %d: %s", status, body) + } + if status, body := request(t, http.MethodGet, enabled+"/ui/projects", "", nil); status != http.StatusOK || !strings.Contains(body, `href="/ui/projects/acme/web"`) { + t.Errorf("projects page = %d, missing the project link", status) + } + if _, body := request(t, http.MethodGet, enabled+"/ui/", "", nil); !strings.Contains(body, `href="/ui/projects"`) { + t.Error("navigation does not link to projects") + } +} diff --git a/internal/server/projects_prune.go b/internal/server/projects_prune.go new file mode 100644 index 00000000..b9ba9362 --- /dev/null +++ b/internal/server/projects_prune.go @@ -0,0 +1,43 @@ +package server + +import ( + "context" + "time" +) + +// projectPruneInterval is how often projects older than projects.max_age +// are removed. +const projectPruneInterval = time.Hour + +// startProjectPruning removes projects that have not uploaded an SBOM for +// projects.max_age, once at startup and then every projectPruneInterval. +func (s *Server) startProjectPruning(ctx context.Context) { + maxAge := s.cfg.Projects.MaxAgeDuration() + if !s.cfg.Projects.Enabled || maxAge <= 0 { + return + } + prune := func() { + s.runExclusive(ctx, "project-pruning", func() { + pruned, err := s.db.PruneProjects(time.Now().Add(-maxAge)) + if err != nil { + s.logger.Warn("pruning projects failed", "error", err) + } + if pruned > 0 { + s.logger.Info("pruned projects", "count", pruned, "max_age", maxAge) + } + }) + } + s.background.Go(func() { + prune() + ticker := time.NewTicker(projectPruneInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + prune() + } + } + }) +} diff --git a/internal/server/projects_prune_test.go b/internal/server/projects_prune_test.go new file mode 100644 index 00000000..0f74d559 --- /dev/null +++ b/internal/server/projects_prune_test.go @@ -0,0 +1,48 @@ +package server + +import ( + "context" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" +) + +func TestProjectPruningRemovesStaleProjects(t *testing.T) { + ts := newProjectsTestServer(t) + ts.server.cfg.Projects = config.ProjectsConfig{Enabled: true, Token: testProjectsToken, MaxAge: "30d"} + if err := ts.server.cfg.Projects.Validate(); err != nil { + t.Fatal(err) + } + if err := ts.db.SaveProject("acme/old", "", "", time.Now().Add(-31*24*time.Hour), nil); err != nil { + t.Fatal(err) + } + if err := ts.db.SaveProject("acme/new", "", "", time.Now(), nil); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + ts.server.startProjectPruning(ctx) + defer func() { + cancel() + ts.server.background.Wait() + }() + + deadline := time.Now().Add(5 * time.Second) + for { + old, err := ts.db.GetProject("acme/old") + if err != nil { + t.Fatal(err) + } + if old == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("acme/old was not pruned") + } + time.Sleep(10 * time.Millisecond) + } + if current, err := ts.db.GetProject("acme/new"); err != nil || current == nil { + t.Errorf("acme/new was pruned: %v, %v", current, err) + } +} diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go new file mode 100644 index 00000000..76ae8bd1 --- /dev/null +++ b/internal/server/projects_ui.go @@ -0,0 +1,287 @@ +package server + +import ( + "cmp" + "net/http" + "net/url" + "slices" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/inventory" + "github.com/go-chi/chi/v5" +) + +// ProjectsListData contains data for rendering the projects list page. +type ProjectsListData struct { + Layout + Projects []ProjectRow +} + +// ProjectRow is a project in the UI. +type ProjectRow struct { + Name string + Ref string + Branch string + UploadedAt string + Components int64 +} + +// ProjectShowData contains data for rendering a project's page. +type ProjectShowData struct { + Layout + Project ProjectRow + Components []ProjectComponentRow + Vulnerable int + Outdated int +} + +// ProjectComponentRow is a package version a project uses, with what the +// proxy knows about it. +type ProjectComponentRow struct { + Ecosystem string + Name string + Version string + // Cached is set when the proxy has a package record, so the package + // page exists; CachedEcosystem and CachedName address it. + Cached bool + CachedEcosystem string + CachedName string + LatestVersion string + Outdated bool + VulnCount int + Severity string +} + +// VersionUsage lists the projects using one version of a package. +type VersionUsage struct { + Version string + EscapedVersion string + // Cached is set when the proxy has a record of the version. + Cached bool + Projects []string +} + +// VulnerableVersion is a package version with known vulnerabilities that +// projects use. +type VulnerableVersion struct { + Ecosystem string + Name string + Version string + Severity string + VulnCount int + Projects []string +} + +// mountProjectUI registers the project inventory pages when +// projects.enabled is set. +func (s *Server) mountProjectUI(ui chi.Router) { + if !s.cfg.Projects.Enabled { + return + } + ui.Get("/projects", s.handleProjectsList) + ui.Get("/projects/*", s.handleProjectShow) +} + +func (s *Server) handleProjectsList(w http.ResponseWriter, r *http.Request) { + projects, err := s.db.ListProjects() + if err != nil { + s.logger.Error("failed to list projects", "error", err) + http.Error(w, "failed to list projects", http.StatusInternalServerError) + return + } + data := ProjectsListData{Layout: s.layoutFor(r), Projects: make([]ProjectRow, len(projects))} + for i, p := range projects { + data.Projects[i] = projectRow(p) + } + if err := s.templates.Render(w, "projects_list", data); err != nil { + s.logger.Error("failed to render projects list", "error", err) + } +} + +func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { + name, err := projectPath(r) + if err == nil { + err = inventory.ValidateProjectName(name) + } + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + project, err := s.db.GetProject(name) + if err != nil { + s.logger.Error("failed to get project", "project", name, "error", err) + http.Error(w, "failed to get project", http.StatusInternalServerError) + return + } + if project == nil { + http.Error(w, "project not found", http.StatusNotFound) + return + } + components, err := s.db.GetProjectComponents(project.ID) + if err != nil { + s.logger.Error("failed to get project components", "project", name, "error", err) + components = nil + } + vulns, err := s.db.GetComponentVulnerabilities(name) + if err != nil { + s.logger.Error("failed to get project vulnerabilities", "project", name, "error", err) + } + affecting := affectingVulnerabilities(vulns) + + data := ProjectShowData{Layout: s.layoutFor(r), Project: projectRow(*project)} + for _, c := range components { + row := ProjectComponentRow{ + Ecosystem: c.Ecosystem, + Name: c.Name, + Version: c.Version, + Cached: c.CachedName.Valid, + CachedEcosystem: c.CachedEcosystem.String, + CachedName: c.CachedName.String, + LatestVersion: c.LatestVersion.String, + Outdated: inventory.Outdated(c.Ecosystem, c.Version, c.LatestVersion.String), + } + if v, ok := affecting[componentKey{c.PackagePURL, c.Version}]; ok { + row.VulnCount, row.Severity = len(v.ids), v.severity + data.Vulnerable++ + } + if row.Outdated { + data.Outdated++ + } + data.Components = append(data.Components, row) + } + if err := s.templates.Render(w, "project_show", data); err != nil { + s.logger.Error("failed to render project page", "error", err) + } +} + +func projectRow(p database.Project) ProjectRow { + return ProjectRow{ + Name: p.Name, + Ref: p.Ref.String, + Branch: p.Branch.String, + UploadedAt: formatTimeAgo(p.UploadedAt), + Components: p.Components, + } +} + +// packageUsedBy lists the projects using each version of pkg, highest +// version first, or nothing when the project inventory is disabled. versions +// are the versions the proxy has records of; a version only projects know of +// is listed too. +func (s *Server) packageUsedBy(pkg *database.Package, versions []database.Version) []VersionUsage { + if !s.cfg.Projects.Enabled { + return nil + } + usages, err := s.db.GetProjectsUsingPackage(pkg.PURL, "") + if err != nil { + s.logger.Error("failed to get projects using package", "purl", pkg.PURL, "error", err) + return nil + } + known := make(map[string]bool, len(versions)) + for i := range versions { + known[versions[i].Version()] = true + } + byVersion := map[string]*VersionUsage{} + var result []*VersionUsage + for _, u := range usages { + vu, ok := byVersion[u.Version] + if !ok { + vu = &VersionUsage{Version: u.Version, EscapedVersion: url.PathEscape(u.Version), Cached: known[u.Version]} + byVersion[u.Version] = vu + result = append(result, vu) + } + vu.Projects = append(vu.Projects, u.Name) + } + slices.SortFunc(result, func(a, b *VersionUsage) int { + return inventory.CompareVersions(pkg.Ecosystem, b.Version, a.Version) + }) + out := make([]VersionUsage, len(result)) + for i, vu := range result { + out[i] = *vu + } + return out +} + +// vulnerableVersionsInUse returns the package versions projects use that +// have vulnerabilities, most severe and most used first, at most limit. It +// returns nothing when the project inventory is disabled. +func (s *Server) vulnerableVersionsInUse(limit int) []VulnerableVersion { + if !s.cfg.Projects.Enabled { + return nil + } + vulns, err := s.db.GetComponentVulnerabilities("") + if err != nil { + s.logger.Error("failed to get vulnerable versions in use", "error", err) + return nil + } + affecting := affectingVulnerabilities(vulns) + result := make([]VulnerableVersion, 0, len(affecting)) + for _, v := range affecting { + projects := make([]string, 0, len(v.projects)) + for p := range v.projects { + projects = append(projects, p) + } + slices.Sort(projects) + result = append(result, VulnerableVersion{ + Ecosystem: v.ecosystem, Name: v.name, Version: v.version, + Severity: v.severity, VulnCount: len(v.ids), Projects: projects, + }) + } + slices.SortFunc(result, func(a, b VulnerableVersion) int { + return cmp.Or( + cmp.Compare(severityRank(b.Severity), severityRank(a.Severity)), + cmp.Compare(len(b.Projects), len(a.Projects)), + cmp.Compare(a.Name, b.Name), + cmp.Compare(a.Version, b.Version), + ) + }) + if len(result) > limit { + result = result[:limit] + } + return result +} + +type componentKey struct{ packagePURL, version string } + +// affectingVersion collects the vulnerabilities affecting one package +// version, and the projects using it. +type affectingVersion struct { + ecosystem, name, version string + severity string + ids map[string]bool + projects map[string]bool +} + +// affectingVulnerabilities keeps the vulnerabilities that affect the version +// each component uses, grouped by package version. +func affectingVulnerabilities(vulns []database.ComponentVulnerability) map[componentKey]*affectingVersion { + out := map[componentKey]*affectingVersion{} + for _, v := range vulns { + if !inventory.Affected(v.Ecosystem, v.Version, v.FixedVersion.String) { + continue + } + key := componentKey{v.PackagePURL, v.Version} + a, ok := out[key] + if !ok { + a = &affectingVersion{ + ecosystem: v.CachedEcosystem, name: v.CachedName, version: v.Version, + ids: map[string]bool{}, projects: map[string]bool{}, + } + out[key] = a + } + a.ids[v.VulnID] = true + a.projects[v.Project] = true + if severityRank(v.Severity.String) > severityRank(a.severity) { + a.severity = v.Severity.String + } + } + return out +} + +// severityOrder lists severities from least to most severe. +var severityOrder = []string{"low", "medium", "high", "critical"} + +// severityRank orders severities; unknown ones rank below "low". +func severityRank(severity string) int { + return slices.Index(severityOrder, severity) + 1 +} diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go new file mode 100644 index 00000000..18fc5c18 --- /dev/null +++ b/internal/server/projects_ui_test.go @@ -0,0 +1,137 @@ +package server + +import ( + "database/sql" + "net/http" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/database" +) + +// seedLodash records lodash with 4.17.21 as its latest and only cached +// version, and a vulnerability fixed in 4.17.21. The syft fixture uses +// 4.17.20. +func seedLodash(t *testing.T, db *database.DB, withVuln bool) { + t.Helper() + if err := db.UpsertPackage(&database.Package{ + PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", + LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } + if err := db.UpsertVersion(&database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}); err != nil { + t.Fatal(err) + } + if !withVuln { + return + } + if err := db.UpsertVulnerability(&database.Vulnerability{ + VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", + Severity: sql.NullString{String: "high", Valid: true}, + FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } +} + +func (ts projectsTestServer) page(t *testing.T, target string) string { + t.Helper() + rec := ts.do(t, http.MethodGet, target, "", nil) + if rec.Code != http.StatusOK { + t.Fatalf("GET %s = %d: %s", target, rec.Code, rec.Body) + } + return rec.Body.String() +} + +func assertContains(t *testing.T, page string, want ...string) { + t.Helper() + for _, w := range want { + if !strings.Contains(page, w) { + t.Errorf("page does not contain %q", w) + } + } +} + +func TestProjectPageShowsComponents(t *testing.T) { + ts := newProjectsTestServer(t) + seedLodash(t, ts.db, true) + ts.upload(t, "/api/projects/acme/web/sbom?branch=main", syftSBOM(t)) + + page := ts.page(t, "/ui/projects/acme/web") + assertContains(t, page, + "acme/web", + `href="/ui/package/npm/lodash"`, // cached, so linked + "outdated, latest 4.17.21", + "1 vulns", + "1 vulnerable", + "symfony/console", // not cached: listed without a link + "not cached", + ) + if strings.Contains(page, `href="/ui/package/composer/symfony/console"`) { + t.Error("links to the package page of a package the proxy has no record of") + } + + // Encoded as one segment, too. + assertContains(t, ts.page(t, "/ui/projects/acme%2Fweb"), "symfony/console") + + if rec := ts.do(t, http.MethodGet, "/ui/projects/acme/other", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("unknown project status = %d, want 404", rec.Code) + } + if rec := ts.do(t, http.MethodGet, "/ui/projects/acme/../web", "", nil); rec.Code != http.StatusBadRequest { + t.Errorf("invalid project status = %d, want 400", rec.Code) + } + + list := ts.page(t, "/ui/projects") + assertContains(t, list, `href="/ui/projects/acme/web"`, "10 packages") +} + +func TestPackagePageShowsUsedBy(t *testing.T) { + ts := newProjectsTestServer(t) + seedLodash(t, ts.db, false) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + ts.upload(t, "/api/projects/acme/api/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + + page := ts.page(t, "/ui/package/npm/lodash") + assertContains(t, page, + "Used by", + `href="/ui/package/npm/lodash/4.17.21"`, + `href="/ui/projects/acme/api"`, + `href="/ui/projects/acme/web"`, + ) + // 4.17.20 is used but not cached. + start := strings.Index(page, "Used by") + if start < 0 { + t.Fatal("no Used by section") + } + used := page[start:] + if i := strings.Index(used, "4.17.20"); i < 0 || !strings.Contains(used[i:], "not cached") { + t.Error("the uncached version 4.17.20 is not listed as not cached") + } + if strings.Index(used, "4.17.21") > strings.Index(used, "4.17.20") { + t.Error("versions are not listed highest first") + } +} + +func TestDashboardShowsVulnerableVersionsInUse(t *testing.T) { + ts := newProjectsTestServer(t) + seedLodash(t, ts.db, false) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + + if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { + t.Error("dashboard shows the vulnerable versions card without vulnerabilities") + } + assertContains(t, ts.page(t, "/ui/"), `href="/ui/projects"`) + + seedLodash(t, ts.db, true) + page := ts.page(t, "/ui/") + assertContains(t, page, "Vulnerable versions in use", "@4.17.20", `href="/ui/projects/acme/web"`, ">high<") + + // A project on the fixed version doesn't count. + ts.upload(t, "/api/projects/acme/web/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { + t.Error("dashboard lists a version the vulnerability is fixed in") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 12619a6a..d96b9dcb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -39,6 +39,7 @@ // - /ui/packages - List all cached packages // - /ui/search - Search packages // - /ui/package/... - Package and version detail pages +// - /ui/projects/... - Project inventory (when projects.enabled) // - /ui/api/browse/... - Archive browsing (used by the UI) // - /ui/api/compare/... - Archive diffing (used by the UI) // @@ -50,6 +51,12 @@ // - POST /api/outdated - Check outdated packages // - POST /api/bulk - Bulk package lookup // - GET /api/packages - List cached packages (JSON) +// +// Project inventory endpoints (when projects.enabled): +// - PUT /api/projects/{project}/sbom - Upload a project's SBOM (token) +// - DELETE /api/projects/{project} - Delete a project (token) +// - GET /api/projects - List projects, or those using ?purl= +// - GET /api/projects/{project} - A project's package versions package server import ( @@ -357,6 +364,7 @@ func (s *Server) serve(listener net.Listener) error { if s.cfg.CacheDelete { ui.Delete("/api/cache/{ecosystem}/*", s.handleCacheDelete) } + s.mountProjectUI(ui) }) r.Get("/", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/ui/", http.StatusFound) @@ -404,6 +412,10 @@ func (s *Server) serve(listener net.Listener) error { registerHostedRoutes(r, hostedAPI, s.cfg.Hosted.TokenExpanded()) } + // Project inventory endpoints (opt-in via projects.enabled or PROXY_PROJECTS_ENABLED) + s.mountProjectAPI(r) + s.startProjectPruning(bgCtx) + s.http = &http.Server{ Addr: s.cfg.Listen, Handler: r, @@ -833,6 +845,7 @@ func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) { }, EnrichmentStats: enrichmentStatsView(enrichStats), } + data.VulnerableInUse = s.vulnerableVersionsInUse(dashboardTopN) for _, p := range popular { pkgInfo := PackageInfo{ @@ -1170,6 +1183,7 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, Vulnerabilities: vulns, LicenseCategory: categorizeLicense(pkg.License), } + data.UsedBy = s.packageUsedBy(pkg, versions) if err := s.templates.Render(w, "package_show", data); err != nil { s.logger.Error("failed to render package show", "error", err) diff --git a/internal/server/templates/components/severity_badge.html b/internal/server/templates/components/severity_badge.html new file mode 100644 index 00000000..a060ed2f --- /dev/null +++ b/internal/server/templates/components/severity_badge.html @@ -0,0 +1,3 @@ +{{define "severity_badge"}} +{{if .}}{{.}}{{else}}unknown{{end}} +{{end}} diff --git a/internal/server/templates/layout/header.html b/internal/server/templates/layout/header.html index 560fe2b3..8b4d94f0 100644 --- a/internal/server/templates/layout/header.html +++ b/internal/server/templates/layout/header.html @@ -54,6 +54,7 @@ {{end}} {{define "nav_links"}} +{{if .Projects}}Projects{{end}} Analytics Install Health diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index e48e205f..ff9bc1b3 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -25,6 +25,33 @@ {{template "security_overview" .EnrichmentStats}} {{end}} +{{if .VulnerableInUse}} +
+
+

Vulnerable versions in use

+
+
+ {{range .VulnerableInUse}} +
+
+ {{template "ecosystem_badge" .Ecosystem}} + {{.Name}} + @{{.Version}} + {{template "severity_badge" .Severity}} + {{.VulnCount}} vulns +
+
+ {{range .Projects}}{{.}}{{end}} +
+
+ {{end}} +
+
+ See all projects → +
+
+{{end}} +
diff --git a/internal/server/templates/pages/package_show.html b/internal/server/templates/pages/package_show.html index a2724a89..4c5a0fc1 100644 --- a/internal/server/templates/pages/package_show.html +++ b/internal/server/templates/pages/package_show.html @@ -49,6 +49,34 @@

Security V {{end}}

+{{if .UsedBy}} +
+
+

Used by

+

Projects whose latest SBOM lists this package

+
+
+ {{range .UsedBy}} +
+
+ {{if .Cached}} + {{if .Version}}{{.Version}}{{else}}unknown version{{end}} + {{else}} + {{if .Version}}{{.Version}}{{else}}unknown version{{end}} + not cached + {{end}} +
+
+ {{range .Projects}} + {{.}} + {{end}} +
+
+ {{end}} +
+
+{{end}} + {{if .Versions}}
diff --git a/internal/server/templates/pages/project_show.html b/internal/server/templates/pages/project_show.html new file mode 100644 index 00000000..0125f9c6 --- /dev/null +++ b/internal/server/templates/pages/project_show.html @@ -0,0 +1,53 @@ +{{define "title"}}{{.Project.Name}} - git-pkgs proxy{{end}} + +{{define "content"}} +
+ ← Projects +

{{.Project.Name}}

+

+ {{.Project.Components}} packages, uploaded {{.Project.UploadedAt}}{{if .Project.Branch}} from {{.Project.Branch}}{{end}}{{if .Project.Ref}} at {{.Project.Ref}}{{end}} +

+ {{if or .Vulnerable .Outdated}} +
+ {{if .Vulnerable}}{{.Vulnerable}} vulnerable{{end}} + {{if .Outdated}}{{.Outdated}} outdated{{end}} +
+ {{end}} +
+ +{{if .Components}} +
+
+ {{range .Components}} +
+
+ {{template "ecosystem_badge" .Ecosystem}} + {{if .Cached}} + {{.Name}} + {{else}} + {{.Name}} + {{end}} + {{.Version}} +
+
+ {{if .VulnCount}} + {{template "severity_badge" .Severity}} + {{.VulnCount}} vulns + {{end}} + {{if .Outdated}} + outdated, latest {{.LatestVersion}} + {{end}} + {{if not .Cached}} + not cached + {{end}} +
+
+ {{end}} +
+
+{{else}} +
+

The latest SBOM lists no packages the proxy serves

+
+{{end}} +{{end}} diff --git a/internal/server/templates/pages/projects_list.html b/internal/server/templates/pages/projects_list.html new file mode 100644 index 00000000..f5ce62d2 --- /dev/null +++ b/internal/server/templates/pages/projects_list.html @@ -0,0 +1,33 @@ +{{define "title"}}Projects - git-pkgs proxy{{end}} + +{{define "content"}} +
+

Projects

+

{{len .Projects}} projects, as of their latest SBOM upload

+
+ +{{if .Projects}} +
+
+ {{range .Projects}} +
+
+ {{.Name}} + {{if or .Branch .Ref}} +
{{.Branch}}{{if and .Branch .Ref}} @ {{end}}{{.Ref}}
+ {{end}} +
+
+ {{.Components}} packages + {{.UploadedAt}} +
+
+ {{end}} +
+
+{{else}} +
+

No project has uploaded an SBOM yet

+
+{{end}} +{{end}} From 895f22a601b47f31539fa2a9d57e05a2da57b523 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 19:43:43 +0100 Subject: [PATCH 02/19] Skip the project's own packages in uploaded SBOMs A syft SBOM of a repository lists the repository itself: the module in go.mod and the root of package-lock.json, both without a version or with the version from package.json, and the module its Go binaries were built from. The project page showed them as dependencies. Entries without a version are now skipped, as are lock-file roots (lock entries that depend on others while nothing depends on them) and modules syft marks as a binary's main module. --- docs/configuration.md | 4 +- internal/inventory/inventory.go | 67 ++++++++++++++++++++++++++-- internal/inventory/inventory_test.go | 37 +++++++++++++-- internal/server/projects_api_test.go | 6 +-- internal/server/projects_ui_test.go | 2 +- 5 files changed, 103 insertions(+), 13 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c5c6f7a0..3a5579dc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1545,10 +1545,10 @@ curl -X PUT --data-binary @sbom.json \ The response counts the recorded package versions and the skipped entries: ```json -{"project":"acme/shop/web","components":10,"skipped":5} +{"project":"acme/shop/web","components":9,"skipped":6} ``` -Entries are recorded by their package URL. Entries without one (files, the operating system) and package types the proxy doesn't serve (such as `pkg:github`) are skipped. Duplicates are recorded once. +Entries are recorded by their package URL. Entries without one (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`) and entries without a version are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once. A GitLab CI job on the default branch could look like this: diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 954d46e4..9df6ce45 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -91,18 +91,33 @@ func FromPURL(s string) (Component, error) { return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil } +// syftMainModule is the property syft sets on modules found in a Go binary, +// naming the module the binary was built from. +const syftMainModule = "syft:metadata:mainModule" + +// syftMetadataType is the property naming where syft read a package from. +const syftMetadataType = "syft:package:metadataType" + +// unknownVersion is what syft gives a package without a resolved version, +// such as the project's own module in go.mod or the root of a lock file. +const unknownVersion = "UNKNOWN" + // Parse reads a CycloneDX (JSON or XML) or SPDX SBOM and returns the package -// versions it lists, without duplicates. Packages without a package URL, or -// of a type the proxy doesn't serve, are counted in skipped. +// versions it lists, without duplicates. Packages without a package URL, of a +// type the proxy doesn't serve, or without a version are counted in skipped. +// So are the project's own packages: Go modules its binaries were built from, +// and the root of a lock file. func Parse(data []byte) (components []Component, skipped int, err error) { doc, err := sbom.Parse(data) if err != nil { return nil, 0, err } + ownModules := mainModules(doc.Packages) + lockRoots := lockFileRoots(doc) seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok { + if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { skipped++ continue } @@ -128,9 +143,55 @@ func componentOf(pkg sbom.Package) (Component, bool) { if c.Version == "" { c.Version = pkg.Version } + // A package without a version is the project itself (the module in + // go.mod, the root of package-lock.json), not something it depends on. + if c.Version == "" || c.Version == unknownVersion { + return Component{}, false + } return c, true } +// lockFileRoots returns the IDs of packages read from a lock file that depend +// on others while nothing depends on them: the project's own entry, such as +// the root of package-lock.json, which carries the version from package.json. +func lockFileRoots(doc *sbom.SBOM) map[string]bool { + dependents := map[string]bool{} + dependencies := map[string]bool{} + for _, r := range doc.Relationships { + if r.Type != sbom.RelDependsOn { + continue + } + dependencies[r.SourceID] = true + dependents[r.TargetID] = true + } + roots := map[string]bool{} + for _, pkg := range doc.Packages { + if pkg.ID == "" || !dependencies[pkg.ID] || dependents[pkg.ID] { + continue + } + for _, prop := range pkg.Properties { + if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { + roots[pkg.ID] = true + } + } + } + return roots +} + +// mainModules returns the Go modules the SBOM's binaries were built from, +// which are the project's own. +func mainModules(pkgs []sbom.Package) map[string]bool { + own := map[string]bool{} + for _, pkg := range pkgs { + for _, prop := range pkg.Properties { + if prop.Name == syftMainModule && prop.Value != "" { + own[prop.Value] = true + } + } + } + return own +} + // ValidateProjectName checks a project name such as "group/sub/repo": one or // more segments of letters, digits, ".", "_" and "-", none of them "." or // "..". diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 3d55d506..b4c6cc39 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -27,14 +27,14 @@ func TestParseSyftCycloneDX(t *testing.T) { {Ecosystem: "composer", Name: "monolog/monolog", Version: "3.5.0", PackagePURL: "pkg:composer/monolog/monolog"}, {Ecosystem: "alpine", Name: "musl", Version: "1.2.4-r2", PackagePURL: "pkg:apk/alpine/musl"}, {Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1", PackagePURL: "pkg:composer/symfony/console"}, - {Ecosystem: "npm", Name: "webapp", Version: "1.0.0", PackagePURL: "pkg:npm/webapp"}, } if !slices.Equal(components, want) { t.Errorf("components =\n%v\nwant\n%v", components, want) } - // The operating system and the four lock files syft lists carry no PURL. - if skipped != 5 { - t.Errorf("skipped = %d, want 5", skipped) + // The operating system and the four lock files syft lists carry no PURL, + // and webapp is the root of package-lock.json, the project itself. + if skipped != 6 { + t.Errorf("skipped = %d, want 6", skipped) } } @@ -131,3 +131,32 @@ func TestOutdatedAndAffected(t *testing.T) { t.Error("a vulnerability without a fix should affect every version") } } + +func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"mod","type":"library","name":"github.com/acme/app","version":"UNKNOWN","purl":"pkg:golang/github.com/acme/app"}, + {"bom-ref":"bin","type":"library","name":"github.com/acme/app","version":"v0.0.0-20261027162038-a3e9dd9fd3f4+dirty","purl":"pkg:golang/github.com/acme/app@v0.0.0-20261027162038-a3e9dd9fd3f4%2Bdirty", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/acme/app"}]}, + {"bom-ref":"cobra","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/acme/app"}]}, + {"bom-ref":"root","type":"library","name":"app-frontend","version":"2.1.0","purl":"pkg:npm/app-frontend@2.1.0", + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]}, + {"bom-ref":"nover","type":"library","name":"app-tools","purl":"pkg:npm/app-tools"}, + {"bom-ref":"jsdom","type":"library","name":"jsdom","version":"30.1.3","purl":"pkg:npm/jsdom@30.1.3", + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]} + ],"dependencies":[{"ref":"root","dependsOn":["jsdom"]}]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + want := []Component{ + {Ecosystem: "golang", Name: "github.com/spf13/cobra", Version: "v1.8.0", PackagePURL: "pkg:golang/github.com/spf13/cobra"}, + {Ecosystem: "npm", Name: "jsdom", Version: "30.1.3", PackagePURL: "pkg:npm/jsdom"}, + } + if !slices.Equal(components, want) { + t.Errorf("components =\n%v\nwant\n%v", components, want) + } + if skipped != 4 { + t.Errorf("skipped = %d, want 4", skipped) + } +} diff --git a/internal/server/projects_api_test.go b/internal/server/projects_api_test.go index 880a4cf7..9a7d9b98 100644 --- a/internal/server/projects_api_test.go +++ b/internal/server/projects_api_test.go @@ -104,14 +104,14 @@ func TestProjectUploadAndGet(t *testing.T) { ts := newProjectsTestServer(t) resp := ts.upload(t, "/api/projects/acme/shop/web/sbom?ref=0123abcd&branch=main", syftSBOM(t)) - if resp != (ProjectUploadResponse{Project: "acme/shop/web", Components: 10, Skipped: 5}) { + if resp != (ProjectUploadResponse{Project: "acme/shop/web", Components: 9, Skipped: 6}) { t.Errorf("upload response = %+v", resp) } // The same project, URL-encoded as one segment. var project ProjectResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme%2Fshop%2Fweb", "", nil), &project) - if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 10 { + if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 9 { t.Errorf("project = %+v", project.ProjectSummary) } var purls []string @@ -126,7 +126,7 @@ func TestProjectUploadAndGet(t *testing.T) { var list ProjectListResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects", "", nil), &list) - if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 10 { + if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 9 { t.Errorf("list = %+v", list) } } diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index 18fc5c18..46dbaa4f 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -83,7 +83,7 @@ func TestProjectPageShowsComponents(t *testing.T) { } list := ts.page(t, "/ui/projects") - assertContains(t, list, `href="/ui/projects/acme/web"`, "10 packages") + assertContains(t, list, `href="/ui/projects/acme/web"`, "9 packages") } func TestPackagePageShowsUsedBy(t *testing.T) { From bd05533bfff6e6876f08dac21786dd84f1900636 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 19:46:51 +0100 Subject: [PATCH 03/19] Delete projects from the project page The page asks for the upload token and sends it to DELETE /api/projects/{project}, so /ui gains no unauthenticated way to delete. The token is kept in session storage for the next delete. --- docs/configuration.md | 2 +- internal/server/projects_ui_test.go | 2 + internal/server/static/project-delete.js | 64 +++++++++++++++++++ .../server/templates/pages/project_show.html | 11 ++++ 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 internal/server/static/project-delete.js diff --git a/docs/configuration.md b/docs/configuration.md index 3a5579dc..e44b28a6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1578,7 +1578,7 @@ curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ https://proxy.example.com/api/projects/acme/shop/web ``` -Reading needs no token, like the rest of `/api`. +Reading needs no token, like the rest of `/api`. A project's page in the UI also has a delete button; it asks for the token and sends it to `DELETE /api/projects/{project}`, keeping it for the browser session. ### What it shows diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index 46dbaa4f..cb1eb5d3 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -67,6 +67,8 @@ func TestProjectPageShowsComponents(t *testing.T) { "1 vulnerable", "symfony/console", // not cached: listed without a link "not cached", + `data-project="acme/web"`, // delete control, asks for the token + `/ui/static/project-delete.js`, ) if strings.Contains(page, `href="/ui/package/composer/symfony/console"`) { t.Error("links to the package page of a package the proxy has no record of") diff --git a/internal/server/static/project-delete.js b/internal/server/static/project-delete.js new file mode 100644 index 00000000..b7422d76 --- /dev/null +++ b/internal/server/static/project-delete.js @@ -0,0 +1,64 @@ +// Deleting a project needs the upload token: /ui has no authentication of its +// own, so the page sends the token the user enters to the API, the same as an +// upload. The token is kept for the browser session only. +(() => { + const root = document.getElementById('project-delete'); + if (!root) { + return; + } + const start = document.getElementById('project-delete-start'); + const form = document.getElementById('project-delete-form'); + const input = document.getElementById('project-delete-token'); + const cancel = document.getElementById('project-delete-cancel'); + const errorBox = document.getElementById('project-delete-error'); + const submit = form.querySelector('button[type="submit"]'); + const storageKey = 'projects-token'; + + const show = (open) => { + start.classList.toggle('hidden', open); + form.classList.toggle('hidden', !open); + form.classList.toggle('flex', open); + errorBox.classList.add('hidden'); + }; + + start.addEventListener('click', () => { + try { + input.value = sessionStorage.getItem(storageKey) || ''; + } catch (e) { + input.value = ''; + } + show(true); + input.focus(); + }); + cancel.addEventListener('click', () => show(false)); + + form.addEventListener('submit', async (event) => { + event.preventDefault(); + const url = '/api/projects/' + encodeURIComponent(root.dataset.project); + submit.disabled = true; + errorBox.classList.add('hidden'); + try { + const resp = await fetch(url, { + method: 'DELETE', + headers: { Authorization: 'Bearer ' + input.value }, + }); + if (resp.status === 401) { + throw new Error('the token was rejected'); + } + if (!resp.ok) { + const body = await resp.json().catch(() => ({})); + throw new Error(body.message || body.error || `HTTP ${resp.status}`); + } + try { + sessionStorage.setItem(storageKey, input.value); + } catch (e) { + // Without storage the token is just asked for again. + } + window.location.href = '/ui/projects'; + } catch (err) { + errorBox.textContent = 'Delete failed: ' + err.message; + errorBox.classList.remove('hidden'); + submit.disabled = false; + } + }); +})(); diff --git a/internal/server/templates/pages/project_show.html b/internal/server/templates/pages/project_show.html index 0125f9c6..9e0cf9e1 100644 --- a/internal/server/templates/pages/project_show.html +++ b/internal/server/templates/pages/project_show.html @@ -13,7 +13,18 @@

{{.Project.Name}}

{{if .Outdated}}{{.Outdated}} outdated{{end}}
{{end}} +
+ + + +
+ {{if .Components}}
From b5eec9c696ded690a5fa9e95f85c0166aa56fb3f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 20:15:50 +0100 Subject: [PATCH 04/19] Drop vulnerability markers from the project inventory The vulnerabilities table is only written by tests: nothing in the proxy calls UpsertVulnerability, so the dashboard card, the per-project count and the per-component markers were always empty. Remove them along with GetComponentVulnerabilities and inventory.Affected, and keep the outdated marker, which uses the latest version the handlers record. --- README.md | 2 +- docs/architecture.md | 2 +- docs/configuration.md | 5 +- internal/database/projects.go | 39 ------- internal/database/projects_test.go | 21 ---- internal/inventory/inventory.go | 12 -- internal/inventory/inventory_test.go | 23 ++-- internal/server/dashboard.go | 3 - internal/server/projects_ui.go | 108 ------------------ internal/server/projects_ui_test.go | 41 +------ internal/server/server.go | 1 - .../templates/components/severity_badge.html | 3 - .../server/templates/pages/dashboard.html | 27 ----- .../server/templates/pages/project_show.html | 9 +- 14 files changed, 20 insertions(+), 276 deletions(-) delete mode 100644 internal/server/templates/components/severity_badge.html diff --git a/README.md b/README.md index 61edf9dd..7060960e 100644 --- a/README.md +++ b/README.md @@ -1514,7 +1514,7 @@ The proxy serves a web UI under `/ui`. No separate frontend build is needed -- t - **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts. - **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews. - **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files. -- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated or vulnerable. Package pages then show which projects use each version, and the dashboard lists vulnerable versions in use. +- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated. Package pages then show which projects use each version. ## Monitoring diff --git a/docs/architecture.md b/docs/architecture.md index 0105d3f5..f1b5e32f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -365,7 +365,7 @@ The package also provides a `MetadataCache` for storing raw upstream metadata bl ### `internal/inventory` -Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated and vulnerability markers on the project pages. +Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated marker on the project pages. ### `internal/config` diff --git a/docs/configuration.md b/docs/configuration.md index e44b28a6..9bc11df2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1508,9 +1508,8 @@ Pushed archives are stored like cached ones but marked as local. The cache size The project inventory answers "which repository uses which version of a package?". Each repository uploads an SBOM of its default branch, and the proxy records the package versions it lists. The UI then shows: -- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version or vulnerabilities for that version. +- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version. - On a package page: which projects use each version, including versions the proxy hasn't cached. -- On the dashboard: vulnerable versions in use and the projects using them. It is disabled by default. Uploads and deletes require a token: @@ -1582,7 +1581,7 @@ Reading needs no token, like the rest of `/api`. A project's page in the UI also ### What it shows -The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. Outdated and vulnerability markers use what the proxy knows about a package: its recorded latest version and the vulnerabilities recorded for it, matched against the version a project uses by the vulnerability's fixed version. A package the proxy has no record of has neither. +The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy. diff --git a/internal/database/projects.go b/internal/database/projects.go index b00596b9..729e09f2 100644 --- a/internal/database/projects.go +++ b/internal/database/projects.go @@ -42,20 +42,6 @@ type ProjectUsage struct { Version string `db:"version"` } -// ComponentVulnerability is a vulnerability recorded for the package of a -// component a project uses. It is not yet filtered by version: whether it -// affects Version depends on FixedVersion. -type ComponentVulnerability struct { - Project string `db:"project"` - ProjectComponent - CachedEcosystem string `db:"cached_ecosystem"` - CachedName string `db:"cached_name"` - VulnID string `db:"vuln_id"` - Severity sql.NullString `db:"severity"` - FixedVersion sql.NullString `db:"fixed_version"` - CVSSScore sql.NullFloat64 `db:"cvss_score"` -} - const projectColumns = ` SELECT pr.id, pr.name, pr.ref, pr.branch, pr.uploaded_at, (SELECT COUNT(*) FROM project_components pc WHERE pc.project_id = pr.id) AS components` @@ -227,31 +213,6 @@ func (db *DB) GetProjectsUsingPackage(packagePURL, version string) ([]ProjectUsa return usages, nil } -// GetComponentVulnerabilities returns the vulnerabilities the proxy holds for -// packages that projects use, for one project or, with an empty name, for all -// of them. Vulnerabilities are recorded per package, so only packages the -// proxy has a record of are matched. -func (db *DB) GetComponentVulnerabilities(project string) ([]ComponentVulnerability, error) { - query := ` - SELECT pr.name AS project, c.package_purl, c.ecosystem, c.name, c.version, - p.ecosystem AS cached_ecosystem, p.name AS cached_name, - v.vuln_id, v.severity, v.fixed_version, v.cvss_score - FROM project_components c - JOIN projects pr ON pr.id = c.project_id - JOIN packages p ON p.purl = c.package_purl - JOIN vulnerabilities v ON v.ecosystem = p.ecosystem AND v.package_name = p.name` - var args []any - if project != "" { - query += ` WHERE pr.name = ?` - args = append(args, project) - } - vulns := []ComponentVulnerability{} - if err := db.Select(&vulns, db.Rebind(query+` ORDER BY c.package_purl, c.version, pr.name, v.vuln_id`), args...); err != nil { - return nil, err - } - return vulns, nil -} - func nullString(s string) sql.NullString { return sql.NullString{String: s, Valid: s != ""} } diff --git a/internal/database/projects_test.go b/internal/database/projects_test.go index 4eaaf500..2a5ddedf 100644 --- a/internal/database/projects_test.go +++ b/internal/database/projects_test.go @@ -159,13 +159,6 @@ func TestProjectComponentsJoinPackages(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := db.UpsertVulnerability(&Vulnerability{ - VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", - Severity: sql.NullString{String: "high", Valid: true}, - FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, - }); err != nil { - t.Fatal(err) - } saveProject(t, db, "acme/web", time.Now(), lodash20, console) saveProject(t, db, "acme/api", time.Now(), lodash21) @@ -188,20 +181,6 @@ func TestProjectComponentsJoinPackages(t *testing.T) { t.Errorf("cached component = %+v", components[1]) } - vulns, err := db.GetComponentVulnerabilities("acme/web") - if err != nil { - t.Fatal(err) - } - if len(vulns) != 1 || vulns[0].Version != "4.17.20" || vulns[0].VulnID != "GHSA-35jh-r3h4-6jhm" { - t.Errorf("acme/web vulnerabilities = %+v", vulns) - } - all, err := db.GetComponentVulnerabilities("") - if err != nil { - t.Fatal(err) - } - if len(all) != 2 { - t.Errorf("all vulnerabilities = %+v, want one row per project", all) - } }) } diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 9df6ce45..cfacdb83 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -232,15 +232,3 @@ func Outdated(ecosystem, version, latest string) bool { } return CompareVersions(ecosystem, version, latest) < 0 } - -// Affected reports whether a vulnerability fixed in fixedVersion affects -// version. A vulnerability without a fixed version affects every version. -func Affected(ecosystem, version, fixedVersion string) bool { - if fixedVersion == "" { - return true - } - if version == "" { - return false - } - return CompareVersions(ecosystem, version, fixedVersion) < 0 -} diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index b4c6cc39..998c93b4 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -100,36 +100,29 @@ func TestValidateProjectName(t *testing.T) { } } -func TestOutdatedAndAffected(t *testing.T) { +func TestOutdated(t *testing.T) { tests := []struct { name string ecosystem string version string other string outdated bool - affected bool }{ - {"npm older", "npm", "4.17.20", "4.17.21", true, true}, - {"npm same", "npm", "4.17.21", "4.17.21", false, false}, - {"npm newer", "npm", "4.18.0", "4.17.21", false, false}, - {"composer v prefix", "composer", "v6.4.1", "v6.4.2", true, true}, - {"golang", "golang", "v1.8.0", "v1.10.0", true, true}, - {"alpine revisions", "alpine", "1.36.1-r15", "1.36.1-r2", false, false}, - {"empty version", "npm", "", "1.0.0", false, false}, + {"npm older", "npm", "4.17.20", "4.17.21", true}, + {"npm same", "npm", "4.17.21", "4.17.21", false}, + {"npm newer", "npm", "4.18.0", "4.17.21", false}, + {"composer v prefix", "composer", "v6.4.1", "v6.4.2", true}, + {"golang", "golang", "v1.8.0", "v1.10.0", true}, + {"alpine revisions", "alpine", "1.36.1-r15", "1.36.1-r2", false}, + {"empty version", "npm", "", "1.0.0", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := Outdated(tt.ecosystem, tt.version, tt.other); got != tt.outdated { t.Errorf("Outdated = %v, want %v", got, tt.outdated) } - if got := Affected(tt.ecosystem, tt.version, tt.other); got != tt.affected { - t.Errorf("Affected = %v, want %v", got, tt.affected) - } }) } - if !Affected("npm", "1.0.0", "") { - t.Error("a vulnerability without a fix should affect every version") - } } func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index ebcc3331..56bfddc1 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -15,9 +15,6 @@ type DashboardData struct { EnrichmentStats EnrichmentStatsView RecentPackages []PackageInfo PopularPackages []PackageInfo - // VulnerableInUse lists vulnerable package versions projects use, when - // the project inventory is enabled. - VulnerableInUse []VulnerableVersion } // DashboardStats contains cache statistics for the dashboard. diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go index 76ae8bd1..0de4601b 100644 --- a/internal/server/projects_ui.go +++ b/internal/server/projects_ui.go @@ -1,7 +1,6 @@ package server import ( - "cmp" "net/http" "net/url" "slices" @@ -31,7 +30,6 @@ type ProjectShowData struct { Layout Project ProjectRow Components []ProjectComponentRow - Vulnerable int Outdated int } @@ -48,8 +46,6 @@ type ProjectComponentRow struct { CachedName string LatestVersion string Outdated bool - VulnCount int - Severity string } // VersionUsage lists the projects using one version of a package. @@ -61,17 +57,6 @@ type VersionUsage struct { Projects []string } -// VulnerableVersion is a package version with known vulnerabilities that -// projects use. -type VulnerableVersion struct { - Ecosystem string - Name string - Version string - Severity string - VulnCount int - Projects []string -} - // mountProjectUI registers the project inventory pages when // projects.enabled is set. func (s *Server) mountProjectUI(ui chi.Router) { @@ -122,11 +107,6 @@ func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { s.logger.Error("failed to get project components", "project", name, "error", err) components = nil } - vulns, err := s.db.GetComponentVulnerabilities(name) - if err != nil { - s.logger.Error("failed to get project vulnerabilities", "project", name, "error", err) - } - affecting := affectingVulnerabilities(vulns) data := ProjectShowData{Layout: s.layoutFor(r), Project: projectRow(*project)} for _, c := range components { @@ -140,10 +120,6 @@ func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { LatestVersion: c.LatestVersion.String, Outdated: inventory.Outdated(c.Ecosystem, c.Version, c.LatestVersion.String), } - if v, ok := affecting[componentKey{c.PackagePURL, c.Version}]; ok { - row.VulnCount, row.Severity = len(v.ids), v.severity - data.Vulnerable++ - } if row.Outdated { data.Outdated++ } @@ -201,87 +177,3 @@ func (s *Server) packageUsedBy(pkg *database.Package, versions []database.Versio } return out } - -// vulnerableVersionsInUse returns the package versions projects use that -// have vulnerabilities, most severe and most used first, at most limit. It -// returns nothing when the project inventory is disabled. -func (s *Server) vulnerableVersionsInUse(limit int) []VulnerableVersion { - if !s.cfg.Projects.Enabled { - return nil - } - vulns, err := s.db.GetComponentVulnerabilities("") - if err != nil { - s.logger.Error("failed to get vulnerable versions in use", "error", err) - return nil - } - affecting := affectingVulnerabilities(vulns) - result := make([]VulnerableVersion, 0, len(affecting)) - for _, v := range affecting { - projects := make([]string, 0, len(v.projects)) - for p := range v.projects { - projects = append(projects, p) - } - slices.Sort(projects) - result = append(result, VulnerableVersion{ - Ecosystem: v.ecosystem, Name: v.name, Version: v.version, - Severity: v.severity, VulnCount: len(v.ids), Projects: projects, - }) - } - slices.SortFunc(result, func(a, b VulnerableVersion) int { - return cmp.Or( - cmp.Compare(severityRank(b.Severity), severityRank(a.Severity)), - cmp.Compare(len(b.Projects), len(a.Projects)), - cmp.Compare(a.Name, b.Name), - cmp.Compare(a.Version, b.Version), - ) - }) - if len(result) > limit { - result = result[:limit] - } - return result -} - -type componentKey struct{ packagePURL, version string } - -// affectingVersion collects the vulnerabilities affecting one package -// version, and the projects using it. -type affectingVersion struct { - ecosystem, name, version string - severity string - ids map[string]bool - projects map[string]bool -} - -// affectingVulnerabilities keeps the vulnerabilities that affect the version -// each component uses, grouped by package version. -func affectingVulnerabilities(vulns []database.ComponentVulnerability) map[componentKey]*affectingVersion { - out := map[componentKey]*affectingVersion{} - for _, v := range vulns { - if !inventory.Affected(v.Ecosystem, v.Version, v.FixedVersion.String) { - continue - } - key := componentKey{v.PackagePURL, v.Version} - a, ok := out[key] - if !ok { - a = &affectingVersion{ - ecosystem: v.CachedEcosystem, name: v.CachedName, version: v.Version, - ids: map[string]bool{}, projects: map[string]bool{}, - } - out[key] = a - } - a.ids[v.VulnID] = true - a.projects[v.Project] = true - if severityRank(v.Severity.String) > severityRank(a.severity) { - a.severity = v.Severity.String - } - } - return out -} - -// severityOrder lists severities from least to most severe. -var severityOrder = []string{"low", "medium", "high", "critical"} - -// severityRank orders severities; unknown ones rank below "low". -func severityRank(severity string) int { - return slices.Index(severityOrder, severity) + 1 -} diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index cb1eb5d3..e8031d45 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -10,9 +10,8 @@ import ( ) // seedLodash records lodash with 4.17.21 as its latest and only cached -// version, and a vulnerability fixed in 4.17.21. The syft fixture uses -// 4.17.20. -func seedLodash(t *testing.T, db *database.DB, withVuln bool) { +// version. The syft fixture uses 4.17.20. +func seedLodash(t *testing.T, db *database.DB) { t.Helper() if err := db.UpsertPackage(&database.Package{ PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", @@ -23,16 +22,6 @@ func seedLodash(t *testing.T, db *database.DB, withVuln bool) { if err := db.UpsertVersion(&database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}); err != nil { t.Fatal(err) } - if !withVuln { - return - } - if err := db.UpsertVulnerability(&database.Vulnerability{ - VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", - Severity: sql.NullString{String: "high", Valid: true}, - FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, - }); err != nil { - t.Fatal(err) - } } func (ts projectsTestServer) page(t *testing.T, target string) string { @@ -55,7 +44,7 @@ func assertContains(t *testing.T, page string, want ...string) { func TestProjectPageShowsComponents(t *testing.T) { ts := newProjectsTestServer(t) - seedLodash(t, ts.db, true) + seedLodash(t, ts.db) ts.upload(t, "/api/projects/acme/web/sbom?branch=main", syftSBOM(t)) page := ts.page(t, "/ui/projects/acme/web") @@ -63,8 +52,7 @@ func TestProjectPageShowsComponents(t *testing.T) { "acme/web", `href="/ui/package/npm/lodash"`, // cached, so linked "outdated, latest 4.17.21", - "1 vulns", - "1 vulnerable", + "1 outdated", "symfony/console", // not cached: listed without a link "not cached", `data-project="acme/web"`, // delete control, asks for the token @@ -90,7 +78,7 @@ func TestProjectPageShowsComponents(t *testing.T) { func TestPackagePageShowsUsedBy(t *testing.T) { ts := newProjectsTestServer(t) - seedLodash(t, ts.db, false) + seedLodash(t, ts.db) ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) ts.upload(t, "/api/projects/acme/api/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) @@ -116,24 +104,7 @@ func TestPackagePageShowsUsedBy(t *testing.T) { } } -func TestDashboardShowsVulnerableVersionsInUse(t *testing.T) { +func TestNavLinksToProjects(t *testing.T) { ts := newProjectsTestServer(t) - seedLodash(t, ts.db, false) - ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) - - if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { - t.Error("dashboard shows the vulnerable versions card without vulnerabilities") - } assertContains(t, ts.page(t, "/ui/"), `href="/ui/projects"`) - - seedLodash(t, ts.db, true) - page := ts.page(t, "/ui/") - assertContains(t, page, "Vulnerable versions in use", "@4.17.20", `href="/ui/projects/acme/web"`, ">high<") - - // A project on the fixed version doesn't count. - ts.upload(t, "/api/projects/acme/web/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ - {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) - if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { - t.Error("dashboard lists a version the vulnerability is fixed in") - } } diff --git a/internal/server/server.go b/internal/server/server.go index d96b9dcb..88897f70 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -845,7 +845,6 @@ func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) { }, EnrichmentStats: enrichmentStatsView(enrichStats), } - data.VulnerableInUse = s.vulnerableVersionsInUse(dashboardTopN) for _, p := range popular { pkgInfo := PackageInfo{ diff --git a/internal/server/templates/components/severity_badge.html b/internal/server/templates/components/severity_badge.html deleted file mode 100644 index a060ed2f..00000000 --- a/internal/server/templates/components/severity_badge.html +++ /dev/null @@ -1,3 +0,0 @@ -{{define "severity_badge"}} -{{if .}}{{.}}{{else}}unknown{{end}} -{{end}} diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index ff9bc1b3..e48e205f 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -25,33 +25,6 @@ {{template "security_overview" .EnrichmentStats}} {{end}} -{{if .VulnerableInUse}} -
-
-

Vulnerable versions in use

-
-
- {{range .VulnerableInUse}} -
-
- {{template "ecosystem_badge" .Ecosystem}} - {{.Name}} - @{{.Version}} - {{template "severity_badge" .Severity}} - {{.VulnCount}} vulns -
-
- {{range .Projects}}{{.}}{{end}} -
-
- {{end}} -
-
- See all projects → -
-
-{{end}} -
diff --git a/internal/server/templates/pages/project_show.html b/internal/server/templates/pages/project_show.html index 9e0cf9e1..2d1b56a8 100644 --- a/internal/server/templates/pages/project_show.html +++ b/internal/server/templates/pages/project_show.html @@ -7,10 +7,9 @@

{{.Project.Name}}

{{.Project.Components}} packages, uploaded {{.Project.UploadedAt}}{{if .Project.Branch}} from {{.Project.Branch}}{{end}}{{if .Project.Ref}} at {{.Project.Ref}}{{end}}

- {{if or .Vulnerable .Outdated}} + {{if .Outdated}}
- {{if .Vulnerable}}{{.Vulnerable}} vulnerable{{end}} - {{if .Outdated}}{{.Outdated}} outdated{{end}} + {{.Outdated}} outdated
{{end}}
@@ -41,10 +40,6 @@

{{.Project.Name}}

{{.Version}}
- {{if .VulnCount}} - {{template "severity_badge" .Severity}} - {{.VulnCount}} vulns - {{end}} {{if .Outdated}} outdated, latest {{.LatestVersion}} {{end}} From 8855c86fc54f84663fef740fb091cde88ea4e102 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 20:18:50 +0100 Subject: [PATCH 05/19] Match more SBOM names to cached packages FromPURL now names packages the way their handlers store them: - NuGet IDs are lowercased, as the NuGet handler stores them. - PyPI names are normalized as in PEP 503. The PyPI handler takes the name from the file name, which spells "." as "_", so pkg:pypi/zope.interface has to match the cached zope-interface. - Conda package URLs without a channel qualifier are skipped: the conda handler records packages as channel/name, so they can never match. - The Go main-module check ignores case. syft keeps the module path's case in its mainModule property, while golang package URLs are lowercased. A table test checks every mapped PURL type against the name its handler passes when caching. --- docs/configuration.md | 2 +- internal/inventory/inventory.go | 76 +++++++++++++++--- internal/inventory/inventory_test.go | 110 +++++++++++++++++++++++++++ internal/server/projects_api.go | 5 +- 4 files changed, 178 insertions(+), 15 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9bc11df2..b8e9ae02 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1547,7 +1547,7 @@ The response counts the recorded package versions and the skipped entries: {"project":"acme/shop/web","components":9,"skipped":6} ``` -Entries are recorded by their package URL. Entries without one (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`) and entries without a version are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once. +Entries are recorded by their package URL, with the package named the way the proxy caches it: NuGet IDs lowercased, PyPI names normalized as in PEP 503 (`zope.interface` becomes `zope-interface`), the distribution dropped from `deb`, `rpm` and `apk` package URLs. Entries without a package URL (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`), entries without a version, and conda entries without a `channel` qualifier (the proxy records conda packages as `channel/name`) are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once. A GitLab CI job on the default branch could look like this: diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index cfacdb83..ccd9ecdf 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "strings" + "unicode" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" @@ -66,8 +67,13 @@ var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": // doesn't serve. var ErrUnsupportedPURL = errors.New("package type is not served by the proxy") -// FromPURL maps a package URL to a component. Qualifiers and subpath are -// dropped; the version is kept as given. +// ErrNoCondaChannel is returned for a conda package URL without a channel +// qualifier: the proxy records conda packages per channel. +var ErrNoCondaChannel = errors.New("conda package URL has no channel qualifier") + +// FromPURL maps a package URL to a component, with the package named the way +// the handler for its type records it. Qualifiers and subpath are dropped; +// the version is kept as given. func FromPURL(s string) (Component, error) { p, err := purl.Parse(s) if err != nil { @@ -77,12 +83,9 @@ func FromPURL(s string) (Component, error) { if !ok { return Component{}, fmt.Errorf("%w: %s", ErrUnsupportedPURL, p.Type) } - name := p.FullName() - switch { - case distroTypes[p.Type]: - name = p.Name - case p.Type == "conda" && p.Namespace == "" && p.Qualifier("channel") != "": - name = p.Qualifier("channel") + "/" + p.Name + name, err := handlerName(p) + if err != nil { + return Component{}, err } packagePURL, _ := packageurl.MakeCacheStrings(ecosystem, name, "") if packagePURL == "" { @@ -91,6 +94,53 @@ func FromPURL(s string) (Component, error) { return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil } +// handlerName returns the package name the handler for p's type stores. +func handlerName(p *purl.PURL) (string, error) { + switch { + case distroTypes[p.Type]: + // The namespace is the distribution; the handlers store the bare + // package name. + return p.Name, nil + case p.Type == "conda" && p.Namespace == "": + // The conda handler stores "channel/name". + channel := p.Qualifier("channel") + if channel == "" { + return "", ErrNoCondaChannel + } + return channel + "/" + p.Name, nil + case p.Type == "nuget": + // The NuGet handler lowercases package IDs. + return strings.ToLower(p.FullName()), nil + case p.Type == "pypi": + return normalizePyPIName(p.FullName()), nil + } + return p.FullName(), nil +} + +// normalizePyPIName normalizes a PyPI project name as PEP 503 does: runs of +// "-", "_" and "." become one "-", and letters are lowercased. The PyPI +// handler stores the name from the file name, which wheels and current sdists +// spell with "_" for any of these, so "zope.interface" is cached as +// "zope-interface". (An old sdist named "zope.interface-5.0.tar.gz" is +// stored with the dot and doesn't match.) +func normalizePyPIName(name string) string { + var b strings.Builder + b.Grow(len(name)) + separator := false + for _, r := range name { + if r == '-' || r == '_' || r == '.' { + separator = true + continue + } + if separator && b.Len() > 0 { + b.WriteByte('-') + } + separator = false + b.WriteRune(unicode.ToLower(r)) + } + return b.String() +} + // syftMainModule is the property syft sets on modules found in a Go binary, // naming the module the binary was built from. const syftMainModule = "syft:metadata:mainModule" @@ -104,7 +154,8 @@ const unknownVersion = "UNKNOWN" // Parse reads a CycloneDX (JSON or XML) or SPDX SBOM and returns the package // versions it lists, without duplicates. Packages without a package URL, of a -// type the proxy doesn't serve, or without a version are counted in skipped. +// type the proxy doesn't serve, without a version, or conda packages without +// a channel are counted in skipped. // So are the project's own packages: Go modules its binaries were built from, // and the root of a lock file. func Parse(data []byte) (components []Component, skipped int, err error) { @@ -117,7 +168,7 @@ func Parse(data []byte) (components []Component, skipped int, err error) { seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { + if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[strings.ToLower(c.Name)]) { skipped++ continue } @@ -179,13 +230,14 @@ func lockFileRoots(doc *sbom.SBOM) map[string]bool { } // mainModules returns the Go modules the SBOM's binaries were built from, -// which are the project's own. +// which are the project's own, lowercased: syft keeps the module path's case +// while FromPURL lowercases golang names. func mainModules(pkgs []sbom.Package) map[string]bool { own := map[string]bool{} for _, pkg := range pkgs { for _, prop := range pkg.Properties { if prop.Name == syftMainModule && prop.Value != "" { - own[prop.Value] = true + own[strings.ToLower(prop.Value)] = true } } } diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 998c93b4..e5b62256 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -6,6 +6,8 @@ import ( "slices" "strings" "testing" + + "github.com/git-pkgs/proxy/internal/packageurl" ) func TestParseSyftCycloneDX(t *testing.T) { @@ -81,6 +83,96 @@ func TestFromPURLUnsupported(t *testing.T) { } } +// TestFromPURLMatchesHandlerNames checks, for every PURL type the inventory +// maps, that an SBOM's package URL ends up as the package URL the handler +// records when it caches that package. handlerName is the name the handler +// passes to GetOrFetchArtifactFromURL, taken from the handler's code. +func TestFromPURLMatchesHandlerNames(t *testing.T) { + tests := []struct { + purlType string + sbomPURL string + handlerName string + }{ + {"alpm", "pkg:alpm/arch/glibc@2.39-1?arch=x86_64", "glibc"}, + {"apk", "pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64", "busybox"}, + {"bsd", "pkg:bsd/freebsd/curl@8.7.1", "curl"}, + {"cargo", "pkg:cargo/serde@1.0.200", "serde"}, + {"chef", "pkg:chef/nginx@12.0.0", "nginx"}, + {"composer", "pkg:composer/symfony/console@v6.4.1", "symfony/console"}, + {"conan", "pkg:conan/zlib@1.3.1", "zlib"}, + {"conda", "pkg:conda/numpy@1.26.4?channel=conda-forge&subdir=linux-64", "conda-forge/numpy"}, + {"cran", "pkg:cran/ggplot2@3.5.1", "ggplot2"}, + {"deb", "pkg:deb/debian/libc6@2.36-9?arch=amd64&distro=debian-12", "libc6"}, + {"gem", "pkg:gem/rails@7.1.3", "rails"}, + // The go handler stores the module path with its case; PURLs of + // golang modules are lowercased on both sides. + {"golang", "pkg:golang/github.com/Azure/azure-sdk-for-go@v68.0.0", "github.com/Azure/azure-sdk-for-go"}, + {"hex", "pkg:hex/phoenix@1.7.12", "phoenix"}, + {"julia", "pkg:julia/Example@0.5.3?uuid=7876af07-990d-54b4-ab0e-23690620f79a", "Example"}, + {"maven", "pkg:maven/org.slf4j/slf4j-api@2.0.13?type=jar", "org.slf4j:slf4j-api"}, + {"npm", "pkg:npm/%40babel/core@7.23.0", "@babel/core"}, + // The NuGet handler lowercases the package ID. + {"nuget", "pkg:nuget/Newtonsoft.Json@13.0.3", "newtonsoft.json"}, + {"pub", "pkg:pub/http@1.2.1", "http"}, + // The PyPI handler takes the name from the wheel or sdist file name, + // which spells "." and "-" as "_". + {"pypi", "pkg:pypi/zope.interface@6.4", "zope_interface"}, + {"rpm", "pkg:rpm/fedora/curl@8.6.0-7.fc40?arch=x86_64", "curl"}, + {"swift", "pkg:swift/github.com/apple/swift-argument-parser@1.3.1", "github.com/apple/swift-argument-parser"}, + } + covered := map[string]bool{} + for _, tt := range tests { + t.Run(tt.purlType, func(t *testing.T) { + covered[tt.purlType] = true + c, err := FromPURL(tt.sbomPURL) + if err != nil { + t.Fatalf("FromPURL(%q): %v", tt.sbomPURL, err) + } + want, _ := packageurl.MakeCacheStrings(ecosystems[tt.purlType], tt.handlerName, "") + if want == "" { + t.Fatalf("handler name %q has no package URL", tt.handlerName) + } + if c.PackagePURL != want { + t.Errorf("PackagePURL = %q, want %q as the handler records it", c.PackagePURL, want) + } + }) + } + for purlType := range ecosystems { + if !covered[purlType] { + t.Errorf("no case for PURL type %q", purlType) + } + } +} + +func TestFromPURLNormalizesPyPINames(t *testing.T) { + for _, s := range []string{"pkg:pypi/zope.interface", "pkg:pypi/Zope_Interface", "pkg:pypi/zope--interface", "pkg:pypi/zope._-interface"} { + c, err := FromPURL(s) + if err != nil { + t.Fatal(err) + } + if c.Name != "zope-interface" || c.PackagePURL != "pkg:pypi/zope-interface" { + t.Errorf("FromPURL(%q) = %+v, want zope-interface", s, c) + } + } +} + +func TestFromPURLCondaNeedsChannel(t *testing.T) { + if _, err := FromPURL("pkg:conda/numpy@1.26.4"); !errors.Is(err, ErrNoCondaChannel) { + t.Errorf("err = %v, want ErrNoCondaChannel", err) + } + doc := `{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"numpy","version":"1.26.4","purl":"pkg:conda/numpy@1.26.4"}, + {"type":"library","name":"numpy","version":"1.26.4","purl":"pkg:conda/numpy@1.26.4?channel=conda-forge"} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "conda-forge/numpy" || skipped != 1 { + t.Errorf("components = %+v, skipped = %d; want conda-forge/numpy and 1 skipped", components, skipped) + } +} + func TestValidateProjectName(t *testing.T) { valid := []string{"repo", "group/repo", "group/sub/my_repo-2.0", strings.Repeat("a", MaxProjectNameLength)} for _, name := range valid { @@ -153,3 +245,21 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { t.Errorf("skipped = %d, want 4", skipped) } } + +func TestParseSkipsMainModuleRegardlessOfCase(t *testing.T) { + // syft keeps the module path's case in mainModule; package URLs of Go + // modules are lowercased. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"bin","type":"library","name":"github.com/Azure/foo","version":"v1.2.0","purl":"pkg:golang/github.com/Azure/foo@v1.2.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]}, + {"bom-ref":"cobra","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "github.com/spf13/cobra" || skipped != 1 { + t.Errorf("components = %+v, skipped = %d; want only cobra, 1 skipped", components, skipped) + } +} diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go index a0073193..7417830a 100644 --- a/internal/server/projects_api.go +++ b/internal/server/projects_api.go @@ -43,8 +43,9 @@ type ProjectUploadResponse struct { Project string `json:"project"` // Components is the number of package versions recorded. Components int `json:"components"` - // Skipped is the number of SBOM entries without a package URL or of a - // package type the proxy doesn't serve. + // Skipped is the number of SBOM entries not recorded: without a package + // URL or a version, of a package type the proxy doesn't serve, conda + // entries without a channel, and the project's own packages. Skipped int `json:"skipped"` } From 1c706fffd0dfe3f0ac589a72988a31c1c8d50da8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 20:23:14 +0100 Subject: [PATCH 06/19] Review fixes for the project inventory - GetProjectsUsingPackage no longer counts each project's components per row; the package page only needs names and versions. The ?purl= listing leaves component_count out accordingly. - The delete button sends to DELETE /ui/api/projects/{project}, the same handler as the API and still behind the token, so it sits under /ui like the cache deletes. - The project page answers 500 when its components can't be read, instead of showing an empty project. - DELETE of a path ending in /sbom that names no project answers 400 with a hint, since it's most likely the upload path. - The upload endpoint lists JSON, XML and plain text (SPDX tag-value) as accepted. - The docs say that versions are compared as the SBOM spells them. --- docs/configuration.md | 6 ++- docs/swagger/docs.go | 14 +++--- docs/swagger/swagger.json | 14 +++--- internal/database/projects.go | 10 ++-- internal/server/projects_api.go | 29 +++++++---- internal/server/projects_api_test.go | 62 +++++++++++++++++++++++- internal/server/projects_ui.go | 10 +++- internal/server/projects_ui_test.go | 12 ++++- internal/server/server.go | 1 + internal/server/static/project-delete.js | 6 +-- 10 files changed, 129 insertions(+), 35 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b8e9ae02..e6019fdb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1577,11 +1577,13 @@ curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ https://proxy.example.com/api/projects/acme/shop/web ``` -Reading needs no token, like the rest of `/api`. A project's page in the UI also has a delete button; it asks for the token and sends it to `DELETE /api/projects/{project}`, keeping it for the browser session. +Reading needs no token, like the rest of `/api`. With `?purl=`, each project carries the `version` it uses and no `component_count`. A delete names the project itself: `DELETE /api/projects/acme/shop/web/sbom` is answered with 400 unless a project `acme/shop/web/sbom` exists. + +A project's page in the UI also has a delete button. It asks for the token and sends it to `DELETE /ui/api/projects/{project}`, which works like the API endpoint and also requires the token; the token is kept for the browser session. Like `/ui/api/cache`, it sits under `/ui`, so rules for the UI at an ingress cover it. ### What it shows -The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. +The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. Versions are compared as the SBOM gives them: a version spelled differently from how the proxy caches it, such as Composer's `v6.4.1` for `6.4.1` or a Debian version with or without its epoch, shows as "not cached" on the package page. Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 7ddf6301..18882891 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -540,7 +540,7 @@ const docTemplate = `{ "BearerAuth": [] } ], - "description": "Removes the project and the package versions recorded for it.", + "description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.", "tags": [ "projects" ], @@ -586,9 +586,11 @@ const docTemplate = `{ "BearerAuth": [] } ], - "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.", "consumes": [ - "application/json" + "application/json", + "application/xml", + "text/plain" ], "produces": [ "application/json" @@ -1341,7 +1343,7 @@ const docTemplate = `{ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "components": { @@ -1372,7 +1374,7 @@ const docTemplate = `{ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "name": { @@ -1401,7 +1403,7 @@ const docTemplate = `{ "type": "string" }, "skipped": { - "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.", "type": "integer" } } diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index eb6f3b9d..a76f8687 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -533,7 +533,7 @@ "BearerAuth": [] } ], - "description": "Removes the project and the package versions recorded for it.", + "description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.", "tags": [ "projects" ], @@ -579,9 +579,11 @@ "BearerAuth": [] } ], - "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.", "consumes": [ - "application/json" + "application/json", + "application/xml", + "text/plain" ], "produces": [ "application/json" @@ -1334,7 +1336,7 @@ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "components": { @@ -1365,7 +1367,7 @@ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "name": { @@ -1394,7 +1396,7 @@ "type": "string" }, "skipped": { - "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.", "type": "integer" } } diff --git a/internal/database/projects.go b/internal/database/projects.go index 729e09f2..c14b0d62 100644 --- a/internal/database/projects.go +++ b/internal/database/projects.go @@ -38,8 +38,11 @@ type ProjectComponentInfo struct { // ProjectUsage is a project using a version of a package. type ProjectUsage struct { - Project - Version string `db:"version"` + Name string `db:"name"` + Ref sql.NullString `db:"ref"` + Branch sql.NullString `db:"branch"` + UploadedAt time.Time `db:"uploaded_at"` + Version string `db:"version"` } const projectColumns = ` @@ -197,7 +200,8 @@ func (db *DB) GetProjectComponents(projectID int64) ([]ProjectComponentInfo, err // GetProjectsUsingPackage returns the projects using a package, or only // those using version when it is not empty, ordered by project and version. func (db *DB) GetProjectsUsingPackage(packagePURL, version string) ([]ProjectUsage, error) { - query := projectColumns + `, c.version + query := ` + SELECT pr.name, pr.ref, pr.branch, pr.uploaded_at, c.version FROM project_components c JOIN projects pr ON pr.id = c.project_id WHERE c.package_purl = ?` diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go index 7417830a..f297a9ac 100644 --- a/internal/server/projects_api.go +++ b/internal/server/projects_api.go @@ -55,8 +55,9 @@ type ProjectSummary struct { Ref string `json:"ref,omitempty"` Branch string `json:"branch,omitempty"` UploadedAt time.Time `json:"uploaded_at"` - // ComponentCount is the number of package versions recorded. - ComponentCount int64 `json:"component_count"` + // ComponentCount is the number of package versions recorded. It is not + // set when listing the projects using a package. + ComponentCount *int64 `json:"component_count,omitempty"` // Version is set when listing the projects using a package: the version // of it this project uses. Version string `json:"version,omitempty"` @@ -107,9 +108,9 @@ func registerProjectRoutes(r chi.Router, h *ProjectsAPIHandler, token string) { // HandleUpload handles PUT /api/projects/{project}/sbom. // @Summary Upload a project's SBOM -// @Description Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment. +// @Description Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment. // @Tags projects -// @Accept json +// @Accept json,application/xml,plain // @Produce json // @Security BearerAuth // @Param project path string true "Project path, e.g. group/sub/repo" @@ -176,9 +177,10 @@ func (h *ProjectsAPIHandler) HandleUpload(w http.ResponseWriter, r *http.Request writeJSON(w, ProjectUploadResponse{Project: name, Components: len(records), Skipped: skipped}) } -// HandleDelete handles DELETE /api/projects/{project}. +// HandleDelete handles DELETE /api/projects/{project}, and the same under +// /ui/api/projects for the delete button on a project's page. // @Summary Delete a project -// @Description Removes the project and the package versions recorded for it. +// @Description Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400. // @Tags projects // @Security BearerAuth // @Param project path string true "Project path, e.g. group/sub/repo" @@ -197,6 +199,10 @@ func (h *ProjectsAPIHandler) HandleDelete(w http.ResponseWriter, r *http.Request case err != nil: h.logger.Error("failed to delete project", "project", name, "error", err) internalError(w, "failed to delete project") + case !deleted && (name == "sbom" || strings.HasSuffix(name, sbomPathSuffix)): + // Most likely the upload path; a project whose last segment is + // "sbom" is deleted above when it exists. + badRequest(w, "project not found; to delete a project, send DELETE /api/projects/{project} without /sbom") case !deleted: notFound(w, "project not found") default: @@ -246,8 +252,13 @@ func (h *ProjectsAPIHandler) listUsages(w http.ResponseWriter, query string) { } resp := ProjectListResponse{Projects: make([]ProjectSummary, len(usages))} for i, u := range usages { - resp.Projects[i] = projectSummary(u.Project) - resp.Projects[i].Version = u.Version + resp.Projects[i] = ProjectSummary{ + Name: u.Name, + Ref: u.Ref.String, + Branch: u.Branch.String, + UploadedAt: u.UploadedAt, + Version: u.Version, + } } writeJSON(w, resp) } @@ -301,7 +312,7 @@ func projectSummary(p database.Project) ProjectSummary { Ref: p.Ref.String, Branch: p.Branch.String, UploadedAt: p.UploadedAt, - ComponentCount: p.Components, + ComponentCount: &p.Components, } } diff --git a/internal/server/projects_api_test.go b/internal/server/projects_api_test.go index 9a7d9b98..ba68dae4 100644 --- a/internal/server/projects_api_test.go +++ b/internal/server/projects_api_test.go @@ -100,6 +100,14 @@ func decodeJSON(t *testing.T, rec *httptest.ResponseRecorder, v any) { } } +// componentCount returns a project's component_count, or -1 if not set. +func componentCount(p ProjectSummary) int64 { + if p.ComponentCount == nil { + return -1 + } + return *p.ComponentCount +} + func TestProjectUploadAndGet(t *testing.T) { ts := newProjectsTestServer(t) @@ -111,7 +119,7 @@ func TestProjectUploadAndGet(t *testing.T) { // The same project, URL-encoded as one segment. var project ProjectResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme%2Fshop%2Fweb", "", nil), &project) - if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 9 { + if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || componentCount(project.ProjectSummary) != 9 { t.Errorf("project = %+v", project.ProjectSummary) } var purls []string @@ -126,7 +134,7 @@ func TestProjectUploadAndGet(t *testing.T) { var list ProjectListResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects", "", nil), &list) - if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 9 { + if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || componentCount(list.Projects[0]) != 9 { t.Errorf("list = %+v", list) } } @@ -169,6 +177,9 @@ func TestProjectLookupByPURL(t *testing.T) { var got []string for _, p := range list.Projects { got = append(got, p.Name+"@"+p.Version) + if p.ComponentCount != nil { + t.Errorf("%s has component_count set when listing by purl", p.Name) + } } if strings.Join(got, ",") != strings.Join(tt.want, ",") { t.Errorf("projects = %v, want %v", got, tt.want) @@ -212,6 +223,40 @@ func TestProjectDelete(t *testing.T) { } } +func TestProjectDeleteOfUploadPath(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + rec := ts.do(t, http.MethodDelete, "/api/projects/acme/web/sbom", testProjectsToken, nil) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "without /sbom") { + t.Errorf("DELETE of the upload path = %d: %s, want 400 with a hint", rec.Code, rec.Body) + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusOK { + t.Errorf("project is gone after DELETE of the upload path: %d", rec.Code) + } + + // A project whose last segment is "sbom" is deleted as usual. + ts.upload(t, "/api/projects/acme/sbom/sbom", syftSBOM(t)) + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme/sbom", testProjectsToken, nil); rec.Code != http.StatusNoContent { + t.Errorf("DELETE of project acme/sbom = %d: %s", rec.Code, rec.Body) + } +} + +func TestProjectDeleteFromUI(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + for _, token := range []string{"", "wrong-token"} { + if rec := ts.do(t, http.MethodDelete, "/ui/api/projects/acme%2Fweb", token, nil); rec.Code != http.StatusUnauthorized { + t.Errorf("UI delete with token %q status = %d, want 401", token, rec.Code) + } + } + if rec := ts.do(t, http.MethodDelete, "/ui/api/projects/acme%2Fweb", testProjectsToken, nil); rec.Code != http.StatusNoContent { + t.Fatalf("UI delete status = %d: %s", rec.Code, rec.Body) + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("get after UI delete status = %d, want 404", rec.Code) + } +} + func TestProjectPathsAreValidated(t *testing.T) { ts := newProjectsTestServer(t) uploads := []string{ @@ -331,6 +376,9 @@ func TestProjectRoutesFollowConfig(t *testing.T) { if status, _ := request(t, http.MethodPut, disabled+"/api/projects/acme/web/sbom", testProjectsToken, syftSBOM(t)); status != http.StatusNotFound { t.Errorf("upload with projects disabled = %d, want 404", status) } + if status, _ := request(t, http.MethodDelete, disabled+"/ui/api/projects/acme%2Fweb", testProjectsToken, nil); status != http.StatusNotFound { + t.Errorf("UI delete with projects disabled = %d, want 404", status) + } if _, body := request(t, http.MethodGet, disabled+"/ui/", "", nil); strings.Contains(body, `href="/ui/projects"`) { t.Error("navigation links to projects while they are disabled") } @@ -351,4 +399,14 @@ func TestProjectRoutesFollowConfig(t *testing.T) { if _, body := request(t, http.MethodGet, enabled+"/ui/", "", nil); !strings.Contains(body, `href="/ui/projects"`) { t.Error("navigation does not link to projects") } + if status, _ := request(t, http.MethodDelete, enabled+"/ui/api/projects/acme%2Fweb", "", nil); status != http.StatusUnauthorized { + t.Errorf("UI delete without token = %d, want 401", status) + } + if status, body := request(t, http.MethodDelete, enabled+"/ui/api/projects/acme%2Fweb", testProjectsToken, nil); status != http.StatusNoContent { + t.Errorf("UI delete = %d: %s", status, body) + } + // The delete button sends its request there, too. + if _, body := request(t, http.MethodGet, enabled+"/ui/static/project-delete.js", "", nil); !strings.Contains(body, "'/ui/api/projects/'") { + t.Error("project-delete.js does not send to /ui/api/projects") + } } diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go index 0de4601b..5bafb392 100644 --- a/internal/server/projects_ui.go +++ b/internal/server/projects_ui.go @@ -58,13 +58,18 @@ type VersionUsage struct { } // mountProjectUI registers the project inventory pages when -// projects.enabled is set. +// projects.enabled is set, and DELETE /ui/api/projects/{project} for the +// delete button on a project's page. Like DELETE /api/projects/{project} it +// requires the bearer token; it sits under /ui so that an ingress can treat +// the UI's requests together. func (s *Server) mountProjectUI(ui chi.Router) { if !s.cfg.Projects.Enabled { return } ui.Get("/projects", s.handleProjectsList) ui.Get("/projects/*", s.handleProjectShow) + api := NewProjectsAPIHandler(s.db, s.logger) + ui.With(requireBearerToken(s.cfg.Projects.TokenExpanded())).Delete("/api/projects/*", api.HandleDelete) } func (s *Server) handleProjectsList(w http.ResponseWriter, r *http.Request) { @@ -105,7 +110,8 @@ func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { components, err := s.db.GetProjectComponents(project.ID) if err != nil { s.logger.Error("failed to get project components", "project", name, "error", err) - components = nil + http.Error(w, "failed to get project components", http.StatusInternalServerError) + return } data := ProjectShowData{Layout: s.layoutFor(r), Project: projectRow(*project)} diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index e8031d45..5be22247 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -104,7 +104,15 @@ func TestPackagePageShowsUsedBy(t *testing.T) { } } -func TestNavLinksToProjects(t *testing.T) { +func TestProjectPageFailsWhenComponentsCannotBeRead(t *testing.T) { ts := newProjectsTestServer(t) - assertContains(t, ts.page(t, "/ui/"), `href="/ui/projects"`) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + // The components query joins packages; without the table it fails while + // the project itself can still be read. + if _, err := ts.db.Exec("ALTER TABLE packages RENAME TO packages_gone"); err != nil { + t.Fatal(err) + } + if rec := ts.do(t, http.MethodGet, "/ui/projects/acme/web", "", nil); rec.Code != http.StatusInternalServerError { + t.Errorf("status = %d, want 500 rather than an empty project", rec.Code) + } } diff --git a/internal/server/server.go b/internal/server/server.go index 88897f70..6946e5c6 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -42,6 +42,7 @@ // - /ui/projects/... - Project inventory (when projects.enabled) // - /ui/api/browse/... - Archive browsing (used by the UI) // - /ui/api/compare/... - Archive diffing (used by the UI) +// - /ui/api/projects/... - Project deletion (used by the UI, needs the token) // // API endpoints for enrichment data: // - GET /api/package/{ecosystem}/{name} - Package metadata diff --git a/internal/server/static/project-delete.js b/internal/server/static/project-delete.js index b7422d76..637cdc13 100644 --- a/internal/server/static/project-delete.js +++ b/internal/server/static/project-delete.js @@ -1,6 +1,6 @@ // Deleting a project needs the upload token: /ui has no authentication of its -// own, so the page sends the token the user enters to the API, the same as an -// upload. The token is kept for the browser session only. +// own, so the page sends the token the user enters with the request, the same +// as an upload. The token is kept for the browser session only. (() => { const root = document.getElementById('project-delete'); if (!root) { @@ -34,7 +34,7 @@ form.addEventListener('submit', async (event) => { event.preventDefault(); - const url = '/api/projects/' + encodeURIComponent(root.dataset.project); + const url = '/ui/api/projects/' + encodeURIComponent(root.dataset.project); submit.disabled = true; errorBox.classList.add('hidden'); try { From 9458dc84e1ccdeace927c990a22510e819bf644a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:29:11 +0100 Subject: [PATCH 07/19] Share one PURL-to-handler mapping between the mirror and the inventory packageurl.HandlerPackage returns the ecosystem and name the handlers store a package URL under. The mirror and the inventory both use it, which settles where they disagreed: distribution namespaces of apk, alpm, bsd, deb and rpm are dropped, PyPI and NuGet names use purl's comparison form, and Maven names are group:artifact as the handler stores them, where the mirror used group/artifact. --- internal/inventory/inventory.go | 93 ++++------------------------ internal/inventory/inventory_test.go | 4 +- internal/mirror/files.go | 14 ----- internal/mirror/source.go | 21 +------ internal/mirror/source_test.go | 27 +++++--- internal/packageurl/handler.go | 59 ++++++++++++++++++ internal/packageurl/handler_test.go | 55 ++++++++++++++++ 7 files changed, 150 insertions(+), 123 deletions(-) create mode 100644 internal/packageurl/handler.go create mode 100644 internal/packageurl/handler_test.go diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index ccd9ecdf..9a13a8bd 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "strings" - "unicode" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" @@ -33,36 +32,16 @@ type Component struct { PackagePURL string } -// ecosystems maps the PURL types the proxy serves to the ecosystem name its -// handlers store packages under. -var ecosystems = map[string]string{ - "alpm": "arch", - "apk": "alpine", - "bsd": "freebsd", - "cargo": "cargo", - "chef": "chef", - "composer": "composer", - "conan": "conan", - "conda": "conda", - "cran": "cran", - "deb": "deb", - "gem": "gem", - "golang": "golang", - "hex": "hex", - "julia": "julia", - "maven": "maven", - "npm": "npm", - "nuget": "nuget", - "pub": "pub", - "pypi": "pypi", - "rpm": "rpm", - "swift": "swift", +// servedTypes are the PURL types the proxy serves packages of and the +// inventory records. +var servedTypes = map[string]bool{ + "alpm": true, "apk": true, "bsd": true, "cargo": true, "chef": true, + "composer": true, "conan": true, "conda": true, "cran": true, "deb": true, + "gem": true, "golang": true, "hex": true, "julia": true, "maven": true, + "npm": true, "nuget": true, "pub": true, "pypi": true, "rpm": true, + "swift": true, } -// distroTypes are PURL types whose namespace names the distribution -// ("debian", "alpine"), not part of the package name the proxy stores. -var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} - // ErrUnsupportedPURL is returned for a package URL whose type the proxy // doesn't serve. var ErrUnsupportedPURL = errors.New("package type is not served by the proxy") @@ -79,13 +58,12 @@ func FromPURL(s string) (Component, error) { if err != nil { return Component{}, err } - ecosystem, ok := ecosystems[p.Type] - if !ok { + if !servedTypes[p.Type] { return Component{}, fmt.Errorf("%w: %s", ErrUnsupportedPURL, p.Type) } - name, err := handlerName(p) - if err != nil { - return Component{}, err + ecosystem, name := packageurl.HandlerPackage(p) + if ecosystem == "conda" && !strings.Contains(name, "/") { + return Component{}, ErrNoCondaChannel } packagePURL, _ := packageurl.MakeCacheStrings(ecosystem, name, "") if packagePURL == "" { @@ -94,53 +72,6 @@ func FromPURL(s string) (Component, error) { return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil } -// handlerName returns the package name the handler for p's type stores. -func handlerName(p *purl.PURL) (string, error) { - switch { - case distroTypes[p.Type]: - // The namespace is the distribution; the handlers store the bare - // package name. - return p.Name, nil - case p.Type == "conda" && p.Namespace == "": - // The conda handler stores "channel/name". - channel := p.Qualifier("channel") - if channel == "" { - return "", ErrNoCondaChannel - } - return channel + "/" + p.Name, nil - case p.Type == "nuget": - // The NuGet handler lowercases package IDs. - return strings.ToLower(p.FullName()), nil - case p.Type == "pypi": - return normalizePyPIName(p.FullName()), nil - } - return p.FullName(), nil -} - -// normalizePyPIName normalizes a PyPI project name as PEP 503 does: runs of -// "-", "_" and "." become one "-", and letters are lowercased. The PyPI -// handler stores the name from the file name, which wheels and current sdists -// spell with "_" for any of these, so "zope.interface" is cached as -// "zope-interface". (An old sdist named "zope.interface-5.0.tar.gz" is -// stored with the dot and doesn't match.) -func normalizePyPIName(name string) string { - var b strings.Builder - b.Grow(len(name)) - separator := false - for _, r := range name { - if r == '-' || r == '_' || r == '.' { - separator = true - continue - } - if separator && b.Len() > 0 { - b.WriteByte('-') - } - separator = false - b.WriteRune(unicode.ToLower(r)) - } - return b.String() -} - // syftMainModule is the property syft sets on modules found in a Go binary, // naming the module the binary was built from. const syftMainModule = "syft:metadata:mainModule" diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index e5b62256..818129ad 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -128,7 +128,7 @@ func TestFromPURLMatchesHandlerNames(t *testing.T) { if err != nil { t.Fatalf("FromPURL(%q): %v", tt.sbomPURL, err) } - want, _ := packageurl.MakeCacheStrings(ecosystems[tt.purlType], tt.handlerName, "") + want, _ := packageurl.MakeCacheStrings(packageurl.HandlerEcosystem(tt.purlType), tt.handlerName, "") if want == "" { t.Fatalf("handler name %q has no package URL", tt.handlerName) } @@ -137,7 +137,7 @@ func TestFromPURLMatchesHandlerNames(t *testing.T) { } }) } - for purlType := range ecosystems { + for purlType := range servedTypes { if !covered[purlType] { t.Errorf("no case for PURL type %q", purlType) } diff --git a/internal/mirror/files.go b/internal/mirror/files.go index 8cf61fe7..7486e8c8 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -116,17 +116,3 @@ func fileQualifiers(ecosystem string, qualifiers map[string]string) map[string]s } return kept } - -// condaName returns a conda package name with its channel, taken from the -// namespace or the channel qualifier. Without either it stays bare, and -// mirroring refuses it. -func condaName(namespace, name, channel string) string { - switch { - case namespace != "": - return namespace + "/" + name - case channel != "": - return channel + "/" + name - default: - return name - } -} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index ecc3a0c5..22812b65 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" "github.com/git-pkgs/sbom" @@ -28,18 +29,6 @@ func (pv PackageVersion) String() string { return fmt.Sprintf("pkg:%s/%s@%s", pv.Ecosystem, pv.Name, pv.Version) } -// handlerEcosystem returns the ecosystem name the proxy's handlers store -// packages of a PURL type under. purl names gem and composer after their -// registries, rubygems and packagist; the handlers use the type. -func handlerEcosystem(purlType string) string { - switch purlType { - case "gem", "composer": - return purlType - default: - return purl.PURLTypeToEcosystem(purlType) - } -} - // Source produces PackageVersion items for mirroring. type Source interface { Enumerate(ctx context.Context, fn func(PackageVersion) error) error @@ -140,13 +129,7 @@ func (s *PURLSource) listVersions(ctx context.Context, versions VersionLister, e // mirrorTarget returns the ecosystem, name and file qualifiers a PURL is // mirrored under. func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]string) { - ecosystem = handlerEcosystem(p.Type) - name = p.Name - if ecosystem == "conda" { - name = condaName(p.Namespace, p.Name, p.Qualifier("channel")) - } else if p.Namespace != "" { - name = p.Namespace + "/" + p.Name - } + ecosystem, name = packageurl.HandlerPackage(p) qualifiers = fileQualifiers(ecosystem, p.Qualifiers.Map()) if ecosystem == "julia" { qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} diff --git a/internal/mirror/source_test.go b/internal/mirror/source_test.go index 9c69afee..faddf35f 100644 --- a/internal/mirror/source_test.go +++ b/internal/mirror/source_test.go @@ -3,6 +3,7 @@ package mirror import ( "context" "encoding/json" + "slices" "strings" "testing" ) @@ -229,26 +230,38 @@ func marshalJSON(t *testing.T, v any) []byte { return data } -// The mirror files packages under the ecosystem names the handlers use, so -// a mirrored package and the same package cached by a client share one name. +// The mirror files packages under the ecosystem and package names the +// handlers use, so a mirrored package and the same package cached by a client +// share one name. func TestPURLSourceUsesHandlerEcosystemNames(t *testing.T) { source := &PURLSource{PURLs: []string{ "pkg:gem/rake@13.2.1", "pkg:composer/psr/log@3.0.2", - "pkg:apk/curl@8.14.1-r2", + "pkg:apk/alpine/curl@8.14.1-r2", "pkg:npm/left-pad@1.3.0", + "pkg:maven/org.slf4j/slf4j-api@2.0.13", + "pkg:pypi/Zope.Interface@6.4", + "pkg:nuget/Newtonsoft.Json@13.0.3", }} var got []string if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { - got = append(got, pv.Ecosystem) + got = append(got, pv.Ecosystem+" "+pv.Name) return nil }); err != nil { t.Fatalf("Enumerate() error = %v", err) } - want := []string{"gem", "composer", "alpine", "npm"} - if strings.Join(got, ",") != strings.Join(want, ",") { - t.Errorf("ecosystems = %v, want %v", got, want) + want := []string{ + "gem rake", + "composer psr/log", + "alpine curl", + "npm left-pad", + "maven org.slf4j:slf4j-api", + "pypi zope-interface", + "nuget newtonsoft.json", + } + if !slices.Equal(got, want) { + t.Errorf("packages = %q, want %q", got, want) } } diff --git a/internal/packageurl/handler.go b/internal/packageurl/handler.go new file mode 100644 index 00000000..6056fcb7 --- /dev/null +++ b/internal/packageurl/handler.go @@ -0,0 +1,59 @@ +package packageurl + +import ( + "github.com/git-pkgs/purl" +) + +// distroTypes are PURL types whose namespace names the distribution +// ("alpine", "debian"), not part of the package name the handlers store. +var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} + +// HandlerEcosystem returns the ecosystem name the proxy's handlers store +// packages of a PURL type under. purl names gem and composer after their +// registries, rubygems and packagist; the handlers use the type. +func HandlerEcosystem(purlType string) string { + switch purlType { + case "gem", "composer": + return purlType + default: + return purl.PURLTypeToEcosystem(purlType) + } +} + +// HandlerPackage returns the ecosystem and package name the proxy's handlers +// store p under, so a package named in a PURL, such as one from an SBOM, +// matches the package a client fetched through the proxy: +// +// - apk, alpm, bsd, deb and rpm: the namespace names the distribution and +// is dropped +// - conda: the channel, from the namespace or the channel qualifier, is +// part of the name ("conda-forge/numpy"); without either the name stays +// bare +// - pypi and nuget: the name in purl's comparison form, PEP 503 for PyPI +// and lowercased for NuGet, as the handlers store them +// +// Other types use purl's full name, which keeps the namespace: "@babel/core", +// or "org.slf4j:slf4j-api" for Maven, as its handler stores it. +func HandlerPackage(p *purl.PURL) (ecosystem, name string) { + ecosystem = HandlerEcosystem(p.Type) + switch { + case distroTypes[p.Type]: + return ecosystem, p.Name + case p.Type == "conda": + return ecosystem, condaName(p.Namespace, p.Name, p.Qualifier("channel")) + case p.Type == "pypi" || p.Type == "nuget": + return ecosystem, p.ComparisonForm().FullName() + } + return ecosystem, p.FullName() +} + +func condaName(namespace, name, channel string) string { + switch { + case namespace != "": + return namespace + "/" + name + case channel != "": + return channel + "/" + name + default: + return name + } +} diff --git a/internal/packageurl/handler_test.go b/internal/packageurl/handler_test.go new file mode 100644 index 00000000..5ae9443a --- /dev/null +++ b/internal/packageurl/handler_test.go @@ -0,0 +1,55 @@ +package packageurl + +import ( + "testing" + + "github.com/git-pkgs/purl" +) + +// HandlerPackage is used by both the mirror and the project inventory, so a +// mirrored package, a package listed in an SBOM and the same package fetched +// by a client end up under one name. +func TestHandlerPackage(t *testing.T) { + tests := []struct { + purl string + ecosystem string + name string + }{ + // purl names these after their registries; the handlers use the type. + {"pkg:gem/rake@13.2.1", "gem", "rake"}, + {"pkg:composer/psr/log@3.0.2", "composer", "psr/log"}, + // Distribution namespaces are dropped. + {"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64", "alpine", "busybox"}, + {"pkg:apk/curl@8.14.1-r2", "alpine", "curl"}, + {"pkg:alpm/arch/glibc@2.39-1", "arch", "glibc"}, + {"pkg:bsd/freebsd/curl@8.7.1", "freebsd", "curl"}, + {"pkg:deb/debian/libc6@2.36-9?arch=amd64", "deb", "libc6"}, + {"pkg:rpm/fedora/curl@8.6.0-7.fc40", "rpm", "curl"}, + // The conda channel is part of the name. + {"pkg:conda/numpy@1.26.4?channel=conda-forge", "conda", "conda-forge/numpy"}, + {"pkg:conda/conda-forge/numpy@1.26.4", "conda", "conda-forge/numpy"}, + {"pkg:conda/numpy@1.26.4", "conda", "numpy"}, + // PyPI names are normalized as PEP 503 does, NuGet IDs lowercased. + {"pkg:pypi/Zope.Interface@6.4", "pypi", "zope-interface"}, + {"pkg:pypi/zope_interface", "pypi", "zope-interface"}, + {"pkg:nuget/Newtonsoft.Json@13.0.3", "nuget", "newtonsoft.json"}, + // Other namespaces stay part of the name. + {"pkg:npm/%40babel/core@7.23.0", "npm", "@babel/core"}, + // The Maven handler stores "group:artifact". + {"pkg:maven/org.slf4j/slf4j-api@2.0.13", "maven", "org.slf4j:slf4j-api"}, + {"pkg:docker/library/alpine@3.20", "docker", "library/alpine"}, + {"pkg:julia/Example@0.5.3?uuid=7876af07-990d-54b4-ab0e-23690620f79a", "julia", "Example"}, + } + for _, tt := range tests { + t.Run(tt.purl, func(t *testing.T) { + p, err := purl.Parse(tt.purl) + if err != nil { + t.Fatal(err) + } + ecosystem, name := HandlerPackage(p) + if ecosystem != tt.ecosystem || name != tt.name { + t.Errorf("HandlerPackage(%q) = %q, %q, want %q, %q", tt.purl, ecosystem, name, tt.ecosystem, tt.name) + } + }) + } +} From 479f26e09e6128e71f7c424e335406326d30562f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:29:11 +0100 Subject: [PATCH 08/19] Start project pruning with the project API and move enrichment routes out of serve --- internal/server/projects_api.go | 7 +++++-- internal/server/server.go | 25 +++++++++++++------------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go index f297a9ac..750be170 100644 --- a/internal/server/projects_api.go +++ b/internal/server/projects_api.go @@ -1,6 +1,7 @@ package server import ( + "context" "errors" "io" "log/slog" @@ -82,11 +83,13 @@ type ProjectResponse struct { Components []ProjectComponent `json:"components"` } -// mountProjectAPI registers /api/projects when projects.enabled is set. -func (s *Server) mountProjectAPI(r chi.Router) { +// mountProjectAPI registers /api/projects when projects.enabled is set, and +// starts pruning projects that sent no SBOM within projects.max_age. +func (s *Server) mountProjectAPI(ctx context.Context, r chi.Router) { if s.cfg.Projects.Enabled { registerProjectRoutes(r, NewProjectsAPIHandler(s.db, s.logger), s.cfg.Projects.TokenExpanded()) } + s.startProjectPruning(ctx) } // registerProjectRoutes mounts /api/projects. Reading is open like the rest diff --git a/internal/server/server.go b/internal/server/server.go index 0ec64701..18129571 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -242,6 +242,17 @@ func (s *Server) Start(listeners ...net.Listener) error { return s.serve(listener) } +// mountEnrichmentAPI registers the API endpoints for enrichment data. +func (s *Server) mountEnrichmentAPI(r chi.Router) { + apiHandler := NewAPIHandler(enrichment.New(s.logger), s.db) + r.Get("/api/package/{ecosystem}/*", apiHandler.HandlePackagePath) + r.Get("/api/vulns/{ecosystem}/*", apiHandler.HandleVulnsPath) + r.Post("/api/outdated", apiHandler.HandleOutdated) + r.Post("/api/bulk", apiHandler.HandleBulkLookup) + r.Get("/api/search", apiHandler.HandleSearch) + r.Get("/api/packages", apiHandler.HandlePackagesList) +} + func (s *Server) serve(listener net.Listener) error { metadataClient, artifactClient := s.newUpstreamClients(s.cfg.ParseArtifactTimeout()) @@ -379,16 +390,7 @@ func (s *Server) serve(listener net.Listener) error { r.Delete("/api/cache/{ecosystem}/*", s.handleCacheDelete) } - // API endpoints for enrichment data - enrichSvc := enrichment.New(s.logger) - apiHandler := NewAPIHandler(enrichSvc, s.db) - - r.Get("/api/package/{ecosystem}/*", apiHandler.HandlePackagePath) - r.Get("/api/vulns/{ecosystem}/*", apiHandler.HandleVulnsPath) - r.Post("/api/outdated", apiHandler.HandleOutdated) - r.Post("/api/bulk", apiHandler.HandleBulkLookup) - r.Get("/api/search", apiHandler.HandleSearch) - r.Get("/api/packages", apiHandler.HandlePackagesList) + s.mountEnrichmentAPI(r) // Start background context (used by mirror jobs and cleanup) bgCtx, bgCancel := context.WithCancel(context.Background()) @@ -419,8 +421,7 @@ func (s *Server) serve(listener net.Listener) error { } // Project inventory endpoints (opt-in via projects.enabled or PROXY_PROJECTS_ENABLED) - s.mountProjectAPI(r) - s.startProjectPruning(bgCtx) + s.mountProjectAPI(bgCtx, r) s.http = &http.Server{ Addr: s.cfg.Listen, From 7a77baf01a58f0bfc280fa63499aac18052c53d2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 11:33:31 +0100 Subject: [PATCH 09/19] Keep namespace/name for Swift package URLs in the handler mapping With purl v0.2.1, FullName returns a Swift package as its registry identity, scope.name. The mirror splits namespace/name again to find the scope or the source repository, so HandlerPackage keeps that form, as mirrorTarget did before the two mappings were shared. --- internal/packageurl/handler.go | 5 +++++ internal/packageurl/handler_test.go | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/internal/packageurl/handler.go b/internal/packageurl/handler.go index 6056fcb7..ee3005ef 100644 --- a/internal/packageurl/handler.go +++ b/internal/packageurl/handler.go @@ -31,6 +31,9 @@ func HandlerEcosystem(purlType string) string { // bare // - pypi and nuget: the name in purl's comparison form, PEP 503 for PyPI // and lowercased for NuGet, as the handlers store them +// - swift: namespace/name, the registry scope or the source repository +// with the package, which the Swift mirror resolves; purl's full name +// is the registry identity scope.name // // Other types use purl's full name, which keeps the namespace: "@babel/core", // or "org.slf4j:slf4j-api" for Maven, as its handler stores it. @@ -43,6 +46,8 @@ func HandlerPackage(p *purl.PURL) (ecosystem, name string) { return ecosystem, condaName(p.Namespace, p.Name, p.Qualifier("channel")) case p.Type == "pypi" || p.Type == "nuget": return ecosystem, p.ComparisonForm().FullName() + case p.Type == "swift" && p.Namespace != "": + return ecosystem, p.Namespace + "/" + p.Name } return ecosystem, p.FullName() } diff --git a/internal/packageurl/handler_test.go b/internal/packageurl/handler_test.go index 5ae9443a..7283b9fd 100644 --- a/internal/packageurl/handler_test.go +++ b/internal/packageurl/handler_test.go @@ -39,6 +39,13 @@ func TestHandlerPackage(t *testing.T) { {"pkg:maven/org.slf4j/slf4j-api@2.0.13", "maven", "org.slf4j:slf4j-api"}, {"pkg:docker/library/alpine@3.20", "docker", "library/alpine"}, {"pkg:julia/Example@0.5.3?uuid=7876af07-990d-54b4-ab0e-23690620f79a", "julia", "Example"}, + // Swift keeps namespace/name, the scope or the source repository, + // which the Swift mirror splits again; purl's full name would be the + // registry identity "scope.name". + {"pkg:swift/Acme/Widget@1.0.0", "swift", "Acme/Widget"}, + {"pkg:swift/github.com/apple/swift-nio@2.62.0", "swift", "github.com/apple/swift-nio"}, + // Go module paths keep their case. + {"pkg:golang/github.com/Azure/foo@v1.2.0", "golang", "github.com/Azure/foo"}, } for _, tt := range tests { t.Run(tt.purl, func(t *testing.T) { From 1cadc4a7ba68980f8d1c6bd5a78490aa8ad68712 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 11:33:31 +0100 Subject: [PATCH 10/19] Compare Go main modules with their case purl v0.2.1 keeps the case of Go module paths, so the inventory no longer lowercases syft's mainModule. Module paths are case-sensitive: github.com/azure/foo is a dependency of github.com/Azure/foo, not part of it. --- internal/inventory/inventory.go | 8 ++++---- internal/inventory/inventory_test.go | 25 ++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 9a13a8bd..d115fb79 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -99,7 +99,7 @@ func Parse(data []byte) (components []Component, skipped int, err error) { seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[strings.ToLower(c.Name)]) { + if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { skipped++ continue } @@ -161,14 +161,14 @@ func lockFileRoots(doc *sbom.SBOM) map[string]bool { } // mainModules returns the Go modules the SBOM's binaries were built from, -// which are the project's own, lowercased: syft keeps the module path's case -// while FromPURL lowercases golang names. +// which are the project's own. Module paths are case-sensitive and package +// URLs keep their case, so they are compared as they are. func mainModules(pkgs []sbom.Package) map[string]bool { own := map[string]bool{} for _, pkg := range pkgs { for _, prop := range pkg.Properties { if prop.Name == syftMainModule && prop.Value != "" { - own[strings.ToLower(prop.Value)] = true + own[prop.Value] = true } } } diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 818129ad..3fbe113d 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -246,9 +246,9 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { } } -func TestParseSkipsMainModuleRegardlessOfCase(t *testing.T) { - // syft keeps the module path's case in mainModule; package URLs of Go - // modules are lowercased. +func TestParseSkipsMixedCaseMainModule(t *testing.T) { + // syft keeps the module path's case in mainModule, and so does the + // package URL. doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ {"bom-ref":"bin","type":"library","name":"github.com/Azure/foo","version":"v1.2.0","purl":"pkg:golang/github.com/Azure/foo@v1.2.0", "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]}, @@ -263,3 +263,22 @@ func TestParseSkipsMainModuleRegardlessOfCase(t *testing.T) { t.Errorf("components = %+v, skipped = %d; want only cobra, 1 skipped", components, skipped) } } + +func TestParseMainModuleKeepsCase(t *testing.T) { + // Go module paths are case-sensitive, and package URLs keep their case, + // so github.com/azure/foo is another module than the main module + // github.com/Azure/foo, and a dependency. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"bin","type":"library","name":"github.com/Azure/foo","version":"v1.2.0","purl":"pkg:golang/github.com/Azure/foo@v1.2.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]}, + {"bom-ref":"other","type":"library","name":"github.com/azure/foo","version":"v0.1.0","purl":"pkg:golang/github.com/azure/foo@v0.1.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "github.com/azure/foo" || skipped != 1 { + t.Errorf("components = %+v, skipped = %d; want only github.com/azure/foo, 1 skipped", components, skipped) + } +} From 3a0e79d6b4e5e94bc52ccf24bda18eaa8e347159 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 18:26:17 +0100 Subject: [PATCH 11/19] Fuzz SBOM parsing and test concurrent project uploads FuzzParse checks that an uploaded body never makes the inventory panic and that every component it records has a package URL, ecosystem and version. TestSaveProjectConcurrentUploads saves one project from eight uploads at once and checks that each succeeds and the components of exactly one remain. --- internal/database/projects_test.go | 52 ++++++++++++++++++++++++++++++ internal/inventory/fuzz_test.go | 28 ++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 internal/inventory/fuzz_test.go diff --git a/internal/database/projects_test.go b/internal/database/projects_test.go index 2a5ddedf..bf19d44a 100644 --- a/internal/database/projects_test.go +++ b/internal/database/projects_test.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "slices" + "sync" "testing" "time" ) @@ -205,3 +206,54 @@ func TestMigrateAddsProjects(t *testing.T) { } }) } + +// TestSaveProjectConcurrentUploads checks that concurrent uploads of one +// project all succeed and leave the components of exactly one of them. +func TestSaveProjectConcurrentUploads(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const uploads = 8 + sets := make([][]ProjectComponent, uploads) + for i := range sets { + for j := 0; j <= i; j++ { + sets[i] = append(sets[i], ProjectComponent{ + PackagePURL: "pkg:npm/dep" + string(rune('a'+j)), Ecosystem: "npm", + Name: "dep" + string(rune('a'+j)), Version: "1.0." + string(rune('0'+i)), + }) + } + } + now := time.Now().UTC() + errs := make(chan error, uploads) + var wg sync.WaitGroup + for i := range uploads { + wg.Go(func() { errs <- db.SaveProject("acme/web", "", "", now, sets[i]) }) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Errorf("SaveProject: %v", err) + } + } + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) == 0 || len(components) > uploads { + t.Fatalf("%d components, want those of one upload", len(components)) + } + version := components[0].Version + for _, c := range components { + if c.Version != version { + t.Fatalf("components mix uploads: %+v", components) + } + } + if want := int(version[len(version)-1]-'0') + 1; len(components) != want { + t.Errorf("%d components of the upload with version %s, want %d", len(components), version, want) + } + }) +} diff --git a/internal/inventory/fuzz_test.go b/internal/inventory/fuzz_test.go new file mode 100644 index 00000000..f7fcadda --- /dev/null +++ b/internal/inventory/fuzz_test.go @@ -0,0 +1,28 @@ +package inventory + +import ( + "os" + "testing" +) + +// FuzzParse checks that no uploaded body makes the SBOM parsing panic. +func FuzzParse(f *testing.F) { + if data, err := os.ReadFile("testdata/syft-cyclonedx.json"); err == nil { + f.Add(data) + } + f.Add([]byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[{"type":"library","name":"x","version":"1","purl":"pkg:npm/x@1"}]}`)) + f.Add([]byte(`{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT","packages":[{"SPDXID":"SPDXRef-a","name":"a","versionInfo":"1","externalRefs":[{"referenceCategory":"PACKAGE-MANAGER","referenceType":"purl","referenceLocator":"pkg:conda/a@1?channel=c"}]}]}`)) + f.Add([]byte("SPDXVersion: SPDX-2.3\nPackageName: a\nSPDXID: SPDXRef-a\nExternalRef: PACKAGE-MANAGER purl pkg:deb/debian/a@1\n")) + f.Add([]byte(`xpkg:pypi/X_Y@1`)) + f.Fuzz(func(t *testing.T, data []byte) { + components, _, err := Parse(data) + if err != nil { + return + } + for _, c := range components { + if c.PackagePURL == "" || c.Version == "" || c.Ecosystem == "" { + t.Errorf("component with an empty field: %+v", c) + } + } + }) +} From 88c0ba8ab3be75703296c33d3b1917478f25c8ee Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 13:20:29 +0100 Subject: [PATCH 12/19] Use sbom's root detection for the project's own packages Bump sbom to v0.1.8 and replace lockFileRoots with sbom.Roots. Inferred roots count as the project only when read from a lock file, so an image's packages that nothing depends on are kept. --- go.mod | 2 +- go.sum | 4 +-- internal/inventory/inventory.go | 49 +++++++++++++------------ internal/inventory/inventory_test.go | 53 ++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index e5cb0c82..2fe061fa 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/registries v0.13.0 - github.com/git-pkgs/sbom v0.1.7 + github.com/git-pkgs/sbom v0.1.8 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 52f4c624..3e33929d 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= -github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= -github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= +github.com/git-pkgs/sbom v0.1.8 h1:B+ohNEQfk1M+ch1/hMUKAitZp792g6fUrMaO4GTSqhI= +github.com/git-pkgs/sbom v0.1.8/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index d115fb79..a0b6b2bf 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -88,18 +88,18 @@ const unknownVersion = "UNKNOWN" // type the proxy doesn't serve, without a version, or conda packages without // a channel are counted in skipped. // So are the project's own packages: Go modules its binaries were built from, -// and the root of a lock file. +// and the roots sbom finds for the dependency graph. func Parse(data []byte) (components []Component, skipped int, err error) { doc, err := sbom.Parse(data) if err != nil { return nil, 0, err } ownModules := mainModules(doc.Packages) - lockRoots := lockFileRoots(doc) + roots := projectRoots(doc) seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { + if !ok || roots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { skipped++ continue } @@ -133,33 +133,36 @@ func componentOf(pkg sbom.Package) (Component, bool) { return c, true } -// lockFileRoots returns the IDs of packages read from a lock file that depend -// on others while nothing depends on them: the project's own entry, such as -// the root of package-lock.json, which carries the version from package.json. -func lockFileRoots(doc *sbom.SBOM) map[string]bool { - dependents := map[string]bool{} - dependencies := map[string]bool{} - for _, r := range doc.Relationships { - if r.Type != sbom.RelDependsOn { - continue - } - dependencies[r.SourceID] = true - dependents[r.TargetID] = true +// projectRoots returns the IDs of the roots sbom finds that are the +// project: all of them when the document declares them, otherwise only those +// read from a lock file. An image's packages that nothing depends on, such as +// busybox, are roots of the graph but not the project. +func projectRoots(doc *sbom.SBOM) map[string]bool { + ids, declared := doc.Roots() + roots := make(map[string]bool, len(ids)) + for _, id := range ids { + roots[id] = declared + } + if declared { + return roots } - roots := map[string]bool{} for _, pkg := range doc.Packages { - if pkg.ID == "" || !dependencies[pkg.ID] || dependents[pkg.ID] { - continue - } - for _, prop := range pkg.Properties { - if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { - roots[pkg.ID] = true - } + if _, ok := roots[pkg.ID]; ok && readFromLockFile(pkg) { + roots[pkg.ID] = true } } return roots } +func readFromLockFile(pkg sbom.Package) bool { + for _, prop := range pkg.Properties { + if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { + return true + } + } + return false +} + // mainModules returns the Go modules the SBOM's binaries were built from, // which are the project's own. Module paths are case-sensitive and package // URLs keep their case, so they are compared as they are. diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 3fbe113d..36e67efa 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -246,6 +246,59 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { } } +func TestParseKeepsTopLevelDependenciesOfLockFiles(t *testing.T) { + // A poetry.lock doesn't list the project, so requests is a direct + // dependency, not the project's own entry. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"requests","type":"library","name":"requests","version":"2.32.3","purl":"pkg:pypi/requests@2.32.3", + "properties":[{"name":"syft:package:metadataType","value":"python-poetry-lock-entry"},{"name":"syft:location:0:path","value":"/poetry.lock"}]}, + {"bom-ref":"urllib3","type":"library","name":"urllib3","version":"2.2.3","purl":"pkg:pypi/urllib3@2.2.3", + "properties":[{"name":"syft:package:metadataType","value":"python-poetry-lock-entry"},{"name":"syft:location:0:path","value":"/poetry.lock"}]} + ],"dependencies":[{"ref":"requests","dependsOn":["urllib3"]}]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 2 || skipped != 0 { + t.Errorf("components = %+v, skipped = %d; want requests and urllib3, 0 skipped", components, skipped) + } +} + +func TestParseKeepsOSPackagesNothingDependsOn(t *testing.T) { + // syft's CycloneDX of an image: metadata.component is the image, with + // no edges, and busybox is something nothing else depends on. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6", + "metadata":{"component":{"bom-ref":"image","type":"container","name":"alpine:3.20"}}, + "components":[ + {"bom-ref":"busybox","type":"library","name":"busybox","version":"1.36.1-r29","purl":"pkg:apk/alpine/busybox@1.36.1-r29?distro=alpine-3.20.3", + "properties":[{"name":"syft:package:metadataType","value":"apk-db-entry"}]}, + {"bom-ref":"musl","type":"library","name":"musl","version":"1.2.5-r0","purl":"pkg:apk/alpine/musl@1.2.5-r0?distro=alpine-3.20.3", + "properties":[{"name":"syft:package:metadataType","value":"apk-db-entry"}]} + ],"dependencies":[{"ref":"busybox","dependsOn":["musl"]}]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 2 || skipped != 0 { + t.Errorf("components = %+v, skipped = %d; want busybox and musl, 0 skipped", components, skipped) + } +} + +func TestParseSkipsTheDeclaredRoot(t *testing.T) { + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6", + "metadata":{"component":{"bom-ref":"app","type":"application","name":"app","version":"1.0.0","purl":"pkg:npm/app@1.0.0"}}, + "components":[ + {"bom-ref":"lodash","type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"} + ],"dependencies":[{"ref":"app","dependsOn":["lodash"]}]}` + components, _, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "lodash" { + t.Errorf("components = %+v, want only lodash", components) + } +} + func TestParseSkipsMixedCaseMainModule(t *testing.T) { // syft keeps the module path's case in mainModule, and so does the // package URL. From 6d3410df41f583816513b6bbdcc668b343810786 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:05:16 +0100 Subject: [PATCH 13/19] Use sbom's ProjectRoots for the project's own packages Drop projectRoots and readFromLockFile; sbom now decides which inferred roots are the project. sbom is on the merge commit of git-pkgs/sbom#27 until it is tagged. --- go.mod | 2 +- go.sum | 4 +-- internal/inventory/inventory.go | 38 +++------------------------- internal/inventory/inventory_test.go | 4 +-- 4 files changed, 9 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 2fe061fa..b6182d22 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/registries v0.13.0 - github.com/git-pkgs/sbom v0.1.8 + github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 3e33929d..6c0e3d1d 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= -github.com/git-pkgs/sbom v0.1.8 h1:B+ohNEQfk1M+ch1/hMUKAitZp792g6fUrMaO4GTSqhI= -github.com/git-pkgs/sbom v0.1.8/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= +github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 h1:mghNOVmaxRvz8I0TsY+augx8GcVuvqbtD5fjMrBXGa8= +github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index a0b6b2bf..c48f387d 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -76,9 +76,6 @@ func FromPURL(s string) (Component, error) { // naming the module the binary was built from. const syftMainModule = "syft:metadata:mainModule" -// syftMetadataType is the property naming where syft read a package from. -const syftMetadataType = "syft:package:metadataType" - // unknownVersion is what syft gives a package without a resolved version, // such as the project's own module in go.mod or the root of a lock file. const unknownVersion = "UNKNOWN" @@ -95,7 +92,10 @@ func Parse(data []byte) (components []Component, skipped int, err error) { return nil, 0, err } ownModules := mainModules(doc.Packages) - roots := projectRoots(doc) + roots := map[string]bool{} + for _, id := range doc.ProjectRoots() { + roots[id] = true + } seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) @@ -133,36 +133,6 @@ func componentOf(pkg sbom.Package) (Component, bool) { return c, true } -// projectRoots returns the IDs of the roots sbom finds that are the -// project: all of them when the document declares them, otherwise only those -// read from a lock file. An image's packages that nothing depends on, such as -// busybox, are roots of the graph but not the project. -func projectRoots(doc *sbom.SBOM) map[string]bool { - ids, declared := doc.Roots() - roots := make(map[string]bool, len(ids)) - for _, id := range ids { - roots[id] = declared - } - if declared { - return roots - } - for _, pkg := range doc.Packages { - if _, ok := roots[pkg.ID]; ok && readFromLockFile(pkg) { - roots[pkg.ID] = true - } - } - return roots -} - -func readFromLockFile(pkg sbom.Package) bool { - for _, prop := range pkg.Properties { - if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { - return true - } - } - return false -} - // mainModules returns the Go modules the SBOM's binaries were built from, // which are the project's own. Module paths are case-sensitive and package // URLs keep their case, so they are compared as they are. diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 36e67efa..89b070bb 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -225,10 +225,10 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { {"bom-ref":"cobra","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0", "properties":[{"name":"syft:metadata:mainModule","value":"github.com/acme/app"}]}, {"bom-ref":"root","type":"library","name":"app-frontend","version":"2.1.0","purl":"pkg:npm/app-frontend@2.1.0", - "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]}, + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]}, {"bom-ref":"nover","type":"library","name":"app-tools","purl":"pkg:npm/app-tools"}, {"bom-ref":"jsdom","type":"library","name":"jsdom","version":"30.1.3","purl":"pkg:npm/jsdom@30.1.3", - "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]} + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]} ],"dependencies":[{"ref":"root","dependsOn":["jsdom"]}]}` components, skipped, err := Parse([]byte(doc)) if err != nil { From 7095ab67ef105c5466f6431a162ba2839eec8a70 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:16:35 +0100 Subject: [PATCH 14/19] Keep the Projects link on an empty projects page ProjectsListData.Projects hid Layout.Projects, so the header left the link out when no project had been uploaded. --- internal/server/projects_ui.go | 8 +++++--- internal/server/projects_ui_test.go | 7 +++++++ internal/server/templates/pages/projects_list.html | 6 +++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go index 5bafb392..c7c2c054 100644 --- a/internal/server/projects_ui.go +++ b/internal/server/projects_ui.go @@ -13,7 +13,9 @@ import ( // ProjectsListData contains data for rendering the projects list page. type ProjectsListData struct { Layout - Projects []ProjectRow + // Rows isn't called Projects, which would hide Layout.Projects from + // the header. + Rows []ProjectRow } // ProjectRow is a project in the UI. @@ -79,9 +81,9 @@ func (s *Server) handleProjectsList(w http.ResponseWriter, r *http.Request) { http.Error(w, "failed to list projects", http.StatusInternalServerError) return } - data := ProjectsListData{Layout: s.layoutFor(r), Projects: make([]ProjectRow, len(projects))} + data := ProjectsListData{Layout: s.layoutFor(r), Rows: make([]ProjectRow, len(projects))} for i, p := range projects { - data.Projects[i] = projectRow(p) + data.Rows[i] = projectRow(p) } if err := s.templates.Render(w, "projects_list", data); err != nil { s.logger.Error("failed to render projects list", "error", err) diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index 5be22247..bd868c94 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -116,3 +116,10 @@ func TestProjectPageFailsWhenComponentsCannotBeRead(t *testing.T) { t.Errorf("status = %d, want 500 rather than an empty project", rec.Code) } } + +func TestProjectsPageLinksProjectsWhenEmpty(t *testing.T) { + ts := newProjectsTestServer(t) + + page := ts.page(t, "/ui/projects") + assertContains(t, page, `href="/ui/projects"`, "0 projects") +} diff --git a/internal/server/templates/pages/projects_list.html b/internal/server/templates/pages/projects_list.html index f5ce62d2..d7256da5 100644 --- a/internal/server/templates/pages/projects_list.html +++ b/internal/server/templates/pages/projects_list.html @@ -3,13 +3,13 @@ {{define "content"}}

Projects

-

{{len .Projects}} projects, as of their latest SBOM upload

+

{{len .Rows}} projects, as of their latest SBOM upload

-{{if .Projects}} +{{if .Rows}}
- {{range .Projects}} + {{range .Rows}}
{{.Name}} From 60872fe990f32788cb7c79c7eac5fc27b0c8d2d2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 09:34:02 +0100 Subject: [PATCH 15/19] Use sbom v0.1.9 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b6182d22..76a1fadc 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/registries v0.13.0 - github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 + github.com/git-pkgs/sbom v0.1.9 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 6c0e3d1d..c6b5f02a 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= -github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 h1:mghNOVmaxRvz8I0TsY+augx8GcVuvqbtD5fjMrBXGa8= -github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= +github.com/git-pkgs/sbom v0.1.9 h1:sfF5cJAwmgrm2N4HTXQWjiBe8tdu8xwhjwfAOyDNMfc= +github.com/git-pkgs/sbom v0.1.9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= From c20f64ebaa1151e4b65f31452c6d660f054dad4a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 10:48:44 +0100 Subject: [PATCH 16/19] Keep the apk namespace for the mirror's repository lookup The inventory drops the namespace from apk names so they match the handler's, but the apk mirror picks the upstream.apk repository from it. --- internal/mirror/apk.go | 6 +++--- internal/mirror/source.go | 11 +++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index 3e08572a..9746f200 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -47,9 +47,9 @@ type apkIndex struct { // and community indexes of that release. func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *progressTracker) { fail := func(err error) { m.recordFailure(tracker, pv, err) } - repository, name, ok := strings.Cut(pv.Name, "/") - if !ok { - repository, name = "alpine", pv.Name + repository, name := pv.Namespace, pv.Name + if repository == "" { + repository = "alpine" } repositoryURL, ok := handler.APKRepositories(m.APKRepositories)[repository] if !ok { diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 1e326ee9..2c417053 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -16,6 +16,9 @@ type PackageVersion struct { Ecosystem string Name string Version string + // Namespace is the PURL namespace of apk packages, which Name leaves + // out. It picks the upstream.apk repository. + Namespace string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and // arch; PyPI uses file_name, and conda subdir, build and type, to select // which files of the version are mirrored. Julia uses uuid to identify @@ -61,11 +64,15 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro } ecosystem, name, qualifiers := mirrorTarget(p) + var namespace string + if ecosystem == "alpine" { + namespace = p.Namespace + } // Images are never expanded to all tags, and conda packages without // a channel aren't listed; both fail when they are mirrored. if p.Version != "" || isImageEcosystem(ecosystem) || (ecosystem == "conda" && !strings.Contains(name, "/")) { - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Qualifiers: qualifiers} + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Namespace: namespace, Qualifiers: qualifiers} if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() } @@ -84,7 +91,7 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } for _, v := range listed { - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Qualifiers: qualifiers} + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Namespace: namespace, Qualifiers: qualifiers} if takesPlatforms(ecosystem) { pv.Platforms = s.Platforms } From 4d707933512e4e4c70adde33e5927e29381efea7 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 11:56:42 +0100 Subject: [PATCH 17/19] Name the apk repository in mirror errors --- internal/mirror/apk.go | 4 ++-- internal/server/mirror_apk_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index 9746f200..c4f68686 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -72,7 +72,7 @@ func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *prog dir := release + "/" + component + "/" + arch artifact, found, err := m.apkArtifact(ctx, repositoryURL+"/"+dir, name, pv.Version) if err != nil { - fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", dir, err)) + fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", repositoryURL+"/"+dir, err)) return } if !found { @@ -87,7 +87,7 @@ func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *prog m.recordResult(tracker, pv, result) return } - fail(fmt.Errorf("%s %s isn't in %s of %s for %s", name, pv.Version, strings.Join(apkComponents, " or "), release, arch)) + fail(fmt.Errorf("%s %s isn't in %s of %s %s for %s", name, pv.Version, strings.Join(apkComponents, " or "), repository, release, arch)) } // apkArtifact returns the file of version of name that the index in dirURL diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go index d5a8c0d6..0c46ba18 100644 --- a/internal/server/mirror_apk_test.go +++ b/internal/server/mirror_apk_test.go @@ -144,7 +144,7 @@ func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.21.0_alpha20240807": "distro", "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=debian-12": "distro", "pkg:apk/other/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3": "upstream.apk", - "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "9.9.9-r0", + "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "isn't in main or community of alpine v3.20", "pkg:apk/alpine/busybox@1.36.1-r29?arch=../../etc&distro=alpine-3.20.3": "arch", } for p, want := range tests { From eb34a1ac6f0049d6b0f126234afdbe900c48a8c3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 12:33:46 +0100 Subject: [PATCH 18/19] Name the apk repository, not its URL, in index errors --- internal/mirror/apk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index c4f68686..30a9771d 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -72,7 +72,7 @@ func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *prog dir := release + "/" + component + "/" + arch artifact, found, err := m.apkArtifact(ctx, repositoryURL+"/"+dir, name, pv.Version) if err != nil { - fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", repositoryURL+"/"+dir, err)) + fail(fmt.Errorf("reading %s %s/APKINDEX.tar.gz: %w", repository, dir, err)) return } if !found { From 903fd94df90930fc69c0b768e3143d28e4f02310 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 12:34:32 +0100 Subject: [PATCH 19/19] Test that apk index errors leave out the URL --- internal/server/mirror_apk_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go index 0c46ba18..ff74df92 100644 --- a/internal/server/mirror_apk_test.go +++ b/internal/server/mirror_apk_test.go @@ -156,3 +156,19 @@ func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { }) } } + +func TestMirrorAPKIndexErrorsLeaveOutTheURL(t *testing.T) { + upstream := newAPKUpstream(t) + upstream.files[apkMainIndex] = "not an index" + + progress, _ := mirrorWithConfig(t, apkMirrorConfig(upstream), + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3") + + if progress.Failed != 1 || len(progress.Errors) != 1 { + t.Fatalf("failed = %d, errors = %v; want one error", progress.Failed, progress.Errors) + } + msg := progress.Errors[0].Error + if !strings.Contains(msg, "alpine v3.20/main/x86_64") || strings.Contains(msg, upstream.URL) { + t.Errorf("error = %q; want the repository and directory without %s", msg, upstream.URL) + } +}