diff --git a/.github/scripts/configure-build.sh b/.github/scripts/configure-build.sh index 1b518b0..cabfa2b 100755 --- a/.github/scripts/configure-build.sh +++ b/.github/scripts/configure-build.sh @@ -21,6 +21,12 @@ if [ -n "${RELEASE_ID:-}" ]; then ) fi +if command -v go >/dev/null 2>&1; then + cmake_args+=( + "-DREQPACK_GO_EXECUTABLE=$(command -v go)" + ) +fi + if [ "$target_family" = "linux" ]; then cmake_args+=( -DBUILD_SHARED_LIBS=OFF @@ -36,6 +42,43 @@ if [ "$target_family" = "macos" ]; then ) fi +if [ "$target_family" = "windows" ]; then + : "${MSYSTEM_PREFIX:?MSYSTEM_PREFIX must be set for Windows builds}" + export PATH="${MSYSTEM_PREFIX}/bin:${PATH}" + + case "${MSYSTEM:-}" in + UCRT64) + c_compiler=gcc + cxx_compiler=g++ + ;; + CLANGARM64) + c_compiler=clang + cxx_compiler=clang++ + ;; + *) + echo "Unsupported MSYSTEM for Windows builds: ${MSYSTEM:-}" >&2 + exit 1 + ;; + esac + + for tool in "${c_compiler}" "${cxx_compiler}" ninja cmake; do + if ! command -v "${tool}" >/dev/null 2>&1; then + echo "Missing Windows build tool: ${tool}" >&2 + exit 1 + fi + done + + cmake_args+=( + -G Ninja + -DCMAKE_MAKE_PROGRAM=ninja + -DCMAKE_C_COMPILER="${c_compiler}" + -DCMAKE_CXX_COMPILER="${cxx_compiler}" + -DCMAKE_PREFIX_PATH="${MSYSTEM_PREFIX}" + -DLUA_INCLUDE_DIR="${MSYSTEM_PREFIX}/include/lua5.4" + -DLUA_LIBRARIES="${MSYSTEM_PREFIX}/lib/liblua5.4.dll.a" + ) +fi + cmake -Wno-dev -S . -B "$build_dir" "${cmake_args[@]}" # Force single-job builds in CI. Bare --parallel with Unix Makefiles becomes # gmake -j (unbounded), which can OOM GitHub-hosted runners on this project. diff --git a/.github/scripts/run-system-tests-windows.sh b/.github/scripts/run-system-tests-windows.sh new file mode 100755 index 0000000..21ad317 --- /dev/null +++ b/.github/scripts/run-system-tests-windows.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 3 ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +binary_path="$1" +repo_root="$2" +msys_prefix="$3" + +[ -x "$binary_path" ] || { + echo "rqp binary missing or not executable: $binary_path" >&2 + exit 1 +} + +[ -d "$repo_root/tests/system/fixtures/demo-plugin" ] || { + echo "system-test fixture missing under tests/system/fixtures/demo-plugin" >&2 + exit 1 +} + +export PATH="${msys_prefix}/bin:${PATH}" + +work_dir="$(mktemp -d)" +trap 'rm -rf "$work_dir"' EXIT INT TERM HUP + +workspace_dir="$work_dir/workspace" +plugin_dir="$workspace_dir/plugins/demo-plugin" +report_path="$workspace_dir/plugin-test-report.json" + +mkdir -p "$plugin_dir" +cp -R "$repo_root/tests/system/fixtures/demo-plugin/." "$plugin_dir/" + +config_path="$("${repo_root}/.github/scripts/create-system-test-config.sh" "$workspace_dir" "$workspace_dir/plugins")" + +"$binary_path" --help >/dev/null +"$binary_path" version >/dev/null +"$binary_path" --config "$config_path" test-plugin --plugin "$plugin_dir" --preset core --report "$report_path" + +grep -F '"failed": 0' "$report_path" >/dev/null diff --git a/.github/scripts/setup-sol2.sh b/.github/scripts/setup-sol2.sh index d8b5fb8..92b2e89 100755 --- a/.github/scripts/setup-sol2.sh +++ b/.github/scripts/setup-sol2.sh @@ -15,11 +15,24 @@ cache_sol_dir="${cache_root%/}/sol" install -d "$sol_dir" "$cache_sol_dir" +patch_sol_optional_construct() { + local header="$1" + if grep -q 'this->construct(std::forward(args)...);' "$header"; then + local tmp + tmp="$(mktemp)" + sed 's/this->construct(std::forward(args)...);/new (static_cast(this)) optional(std::in_place, std::forward(args)...); return **this;/g' "$header" > "$tmp" + mv "$tmp" "$header" + fi +} + for header in sol.hpp config.hpp forward.hpp; do if [ ! -f "$cache_sol_dir/$header" ]; then tmp_file="$cache_sol_dir/$header.tmp" curl --retry 5 --retry-all-errors --retry-delay 2 -fsSL "${base_url}/$header" -o "$tmp_file" mv "$tmp_file" "$cache_sol_dir/$header" fi + if [ "$header" = "sol.hpp" ]; then + patch_sol_optional_construct "$cache_sol_dir/$header" + fi install -m 0644 "$cache_sol_dir/$header" "$sol_dir/$header" done diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 46a0a2c..7b8a835 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -21,6 +21,7 @@ jobs: contents: write env: SOL2_VERSION: v3.3.0 + GO_VERSION: "1.23" COVERAGE_BUILD_DIR: build/coverage steps: - name: Checkout repository @@ -28,6 +29,12 @@ jobs: with: fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: tools/trivydb-exporter/go.sum + - name: Install Linux dependencies run: | sudo apt-get update @@ -47,6 +54,7 @@ jobs: cmake -S . -B "${COVERAGE_BUILD_DIR}" \ -DCMAKE_BUILD_TYPE=Debug \ -DREQPACK_ENABLE_COVERAGE=ON \ + -DREQPACK_GO_EXECUTABLE="$(command -v go)" \ -DSOL2_INCLUDE_DIR="${SOL2_INCLUDE_DIR}" - name: Build coverage targets diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2739187..c3e8406 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,6 +16,7 @@ defaults: env: RELEASE_ID: ${{ startsWith(github.ref, 'refs/tags/v') && github.ref_name || github.sha }} + GO_VERSION: "1.23" jobs: build-artifacts: @@ -45,6 +46,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: tools/trivydb-exporter/go.sum + - name: Compute cache scope id: cache-scope run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" @@ -69,12 +76,13 @@ jobs: - name: Install macOS dependencies if: matrix.family == 'macos' run: | - brew install ccache cmake cli11 fmt spdlog boost zstd openssl@3 lua@5.4 + brew install ccache cmake cli11 fmt spdlog boost zstd openssl@3 lua@5.4 go { echo "BREW_PREFIX=$(brew --prefix)" echo "OPENSSL_PREFIX=$(brew --prefix openssl@3)" echo "LUA_PREFIX=$(brew --prefix lua@5.4)" echo "ZSTD_PREFIX=$(brew --prefix zstd)" + echo "PATH=$(brew --prefix go)/bin:${PATH}" } >> "$GITHUB_ENV" - name: Install sol2 headers @@ -104,6 +112,9 @@ jobs: - name: Build macOS targets if: matrix.family == 'macos' run: | + export PATH="$(brew --prefix go)/bin:${PATH}" + command -v go + go version export PKG_CONFIG_PATH="${OPENSSL_PREFIX}/lib/pkgconfig:${LUA_PREFIX}/lib/pkgconfig:${ZSTD_PREFIX}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" rm -rf build release-artifacts build-artifacts bash .github/scripts/configure-build.sh build macos \ @@ -134,6 +145,7 @@ jobs: unit-integration-tests: name: ctest-${{ matrix.target }} needs: build-artifacts + if: ${{ !cancelled() }} runs-on: ${{ matrix.runner }} strategy: fail-fast: false @@ -149,6 +161,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: tools/trivydb-exporter/go.sum + - name: Install Linux runtime dependencies if: matrix.target != 'aarch64-darwin' run: | @@ -161,7 +179,7 @@ jobs: - name: Install macOS runtime dependencies if: matrix.target == 'aarch64-darwin' run: | - brew install cmake cli11 fmt spdlog boost zstd openssl@3 lua@5.4 + brew install cmake cli11 fmt spdlog boost zstd openssl@3 lua@5.4 go - name: Download build tree artifact uses: actions/download-artifact@v4 @@ -183,6 +201,7 @@ jobs: needs: - build-artifacts - unit-integration-tests + if: ${{ !cancelled() }} runs-on: ${{ matrix.runner }} strategy: fail-fast: false @@ -215,11 +234,17 @@ jobs: "${{ matrix.target }}" system-tests-macos: - name: system-macos-aarch64-darwin + name: system-macos-${{ matrix.target }} needs: - build-artifacts - unit-integration-tests + if: ${{ !cancelled() }} runs-on: macos-14 + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-darwin steps: - name: Checkout repository uses: actions/checkout@v4 @@ -227,13 +252,13 @@ jobs: - name: Download release bundle uses: actions/download-artifact@v4 with: - name: release-bundle-aarch64-darwin + name: release-bundle-${{ matrix.target }} path: release-artifacts - name: Run native system tests run: | bash .github/scripts/run-system-tests-macos.sh \ - "release-artifacts/rqp-${{ env.RELEASE_ID }}-aarch64-darwin.tar.gz" \ + "release-artifacts/rqp-${{ env.RELEASE_ID }}-${{ matrix.target }}.tar.gz" \ "${{ github.workspace }}" publish-container-arch: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 45c0d29..be30eae 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,6 +13,7 @@ defaults: env: RELEASE_ID: ${{ github.sha }} + GO_VERSION: "1.23" jobs: build-artifacts: @@ -40,15 +41,53 @@ jobs: - target: aarch64-darwin runner: macos-14 family: macos + - target: x86_64-windows + runner: windows-latest + family: windows + msystem: UCRT64 + msys_package_prefix: mingw-w64-ucrt-x86_64 + - target: aarch64-windows + runner: windows-11-arm + family: windows + msystem: CLANGARM64 + msys_package_prefix: mingw-w64-clang-aarch64 steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: tools/trivydb-exporter/go.sum + + - name: Set up MSYS2 + if: matrix.family == 'windows' + uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.msystem }} + update: true + install: >- + ${{ matrix.msys_package_prefix }}-toolchain + ${{ matrix.msys_package_prefix }}-cmake + ${{ matrix.msys_package_prefix }}-ninja + ${{ matrix.msys_package_prefix }}-boost + ${{ matrix.msys_package_prefix }}-curl + ${{ matrix.msys_package_prefix }}-openssl + ${{ matrix.msys_package_prefix }}-fmt + ${{ matrix.msys_package_prefix }}-spdlog + ${{ matrix.msys_package_prefix }}-lua54 + ${{ matrix.msys_package_prefix }}-zstd + ${{ matrix.msys_package_prefix }}-cli11 + ${{ matrix.msys_package_prefix }}-go + - name: Compute cache scope + if: matrix.family != 'windows' id: cache-scope run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" - name: Restore compiler cache + if: matrix.family != 'windows' uses: actions/cache@v4 with: path: ${{ env.CCACHE_DIR }} @@ -68,12 +107,13 @@ jobs: - name: Install macOS dependencies if: matrix.family == 'macos' run: | - brew install ccache cmake cli11 fmt spdlog boost zstd openssl@3 lua@5.4 + brew install ccache cmake cli11 fmt spdlog boost zstd openssl@3 lua@5.4 go { echo "BREW_PREFIX=$(brew --prefix)" echo "OPENSSL_PREFIX=$(brew --prefix openssl@3)" echo "LUA_PREFIX=$(brew --prefix lua@5.4)" echo "ZSTD_PREFIX=$(brew --prefix zstd)" + echo "PATH=$(brew --prefix go)/bin:${PATH}" } >> "$GITHUB_ENV" - name: Install sol2 headers @@ -83,6 +123,7 @@ jobs: echo "SOL2_INCLUDE_DIR=${{ github.workspace }}/third_party/include" >> "$GITHUB_ENV" - name: Prepare compiler cache + if: matrix.family != 'windows' run: | mkdir -p "$CCACHE_DIR" ccache --set-config=max_size="$CCACHE_MAXSIZE" @@ -95,7 +136,7 @@ jobs: bash .github/scripts/configure-build.sh build linux \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DSOL2_INCLUDE_DIR="${{ env.SOL2_INCLUDE_DIR }}" + -DSOL2_INCLUDE_DIR="${{ github.workspace }}/third_party/include" mkdir -p release-artifacts build-artifacts bash scripts/package_release_bundle.sh build/rqp "release-artifacts/rqp-${RELEASE_ID}-${{ matrix.target }}.tar.gz" linux bash .github/scripts/archive-build-tree.sh build "build-artifacts/build-tree-${{ matrix.target }}.tar.gz" @@ -103,6 +144,9 @@ jobs: - name: Build macOS targets if: matrix.family == 'macos' run: | + export PATH="$(brew --prefix go)/bin:${PATH}" + command -v go + go version export PKG_CONFIG_PATH="${OPENSSL_PREFIX}/lib/pkgconfig:${LUA_PREFIX}/lib/pkgconfig:${ZSTD_PREFIX}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" rm -rf build release-artifacts build-artifacts bash .github/scripts/configure-build.sh build macos \ @@ -111,13 +155,29 @@ jobs: -DCMAKE_PREFIX_PATH="${BREW_PREFIX};${OPENSSL_PREFIX};${LUA_PREFIX};${ZSTD_PREFIX}" \ -DOPENSSL_ROOT_DIR="${OPENSSL_PREFIX}" \ -DREQPACK_ZSTD_LIBRARY="${ZSTD_PREFIX}/lib/libzstd.dylib" \ - -DSOL2_INCLUDE_DIR="${{ env.SOL2_INCLUDE_DIR }}" \ + -DSOL2_INCLUDE_DIR="${{ github.workspace }}/third_party/include" \ -DLUA_INCLUDE_DIR="${LUA_PREFIX}/include/lua" \ -DLUA_LIBRARIES="${LUA_PREFIX}/lib/liblua.5.4.dylib" mkdir -p release-artifacts build-artifacts bash scripts/package_release_bundle.sh build/rqp "release-artifacts/rqp-${RELEASE_ID}-${{ matrix.target }}.tar.gz" macos bash .github/scripts/archive-build-tree.sh build "build-artifacts/build-tree-${{ matrix.target }}.tar.gz" + - name: Build Windows targets + if: matrix.family == 'windows' + shell: msys2 {0} + run: | + export PATH="${MSYSTEM_PREFIX}/bin:${PATH}" + case "${MSYSTEM}" in + UCRT64) command -v cmake gcc g++ ninja go ;; + CLANGARM64) command -v cmake clang clang++ ninja go ;; + *) echo "Unsupported MSYSTEM: ${MSYSTEM}" >&2; exit 1 ;; + esac + rm -rf build build-artifacts + mkdir -p build-artifacts + bash .github/scripts/configure-build.sh build windows \ + -DSOL2_INCLUDE_DIR="$(pwd)/third_party/include" + bash .github/scripts/archive-build-tree.sh build "build-artifacts/build-tree-${{ matrix.target }}.tar.gz" + - name: Upload build tree uses: actions/upload-artifact@v4 with: @@ -125,13 +185,14 @@ jobs: path: build-artifacts/build-tree-${{ matrix.target }}.tar.gz - name: Upload release bundle + if: matrix.family != 'windows' uses: actions/upload-artifact@v4 with: name: release-bundle-${{ matrix.target }} path: release-artifacts/rqp-${{ env.RELEASE_ID }}-${{ matrix.target }}.tar.gz - name: Show compiler cache stats - if: always() + if: always() && matrix.family != 'windows' run: ccache --show-stats || true - name: Upload failure logs @@ -148,6 +209,7 @@ jobs: unit-integration-tests: name: ctest-${{ matrix.target }} needs: build-artifacts + if: ${{ !cancelled() }} runs-on: ${{ matrix.runner }} strategy: fail-fast: false @@ -155,16 +217,54 @@ jobs: include: - target: x86_64-linux runner: ubuntu-24.04 + family: linux - target: aarch64-linux runner: ubuntu-24.04-arm + family: linux - target: aarch64-darwin runner: macos-14 + family: macos + - target: x86_64-windows + runner: windows-latest + family: windows + msystem: UCRT64 + msys_package_prefix: mingw-w64-ucrt-x86_64 + - target: aarch64-windows + runner: windows-11-arm + family: windows + msystem: CLANGARM64 + msys_package_prefix: mingw-w64-clang-aarch64 steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: tools/trivydb-exporter/go.sum + + - name: Set up MSYS2 + if: matrix.family == 'windows' + uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.msystem }} + update: true + install: >- + ${{ matrix.msys_package_prefix }}-toolchain + ${{ matrix.msys_package_prefix }}-cmake + ${{ matrix.msys_package_prefix }}-boost + ${{ matrix.msys_package_prefix }}-curl + ${{ matrix.msys_package_prefix }}-openssl + ${{ matrix.msys_package_prefix }}-fmt + ${{ matrix.msys_package_prefix }}-spdlog + ${{ matrix.msys_package_prefix }}-lua54 + ${{ matrix.msys_package_prefix }}-zstd + ${{ matrix.msys_package_prefix }}-cli11 + ${{ matrix.msys_package_prefix }}-go + - name: Install Linux runtime dependencies - if: matrix.target != 'aarch64-darwin' + if: matrix.family == 'linux' run: | sudo apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -173,9 +273,9 @@ jobs: liblua5.4-dev libspdlog-dev libssl-dev libzstd-dev - name: Install macOS runtime dependencies - if: matrix.target == 'aarch64-darwin' + if: matrix.family == 'macos' run: | - brew install cmake cli11 fmt spdlog boost zstd openssl@3 lua@5.4 + brew install cmake cli11 fmt spdlog boost zstd openssl@3 lua@5.4 go - name: Download build tree artifact uses: actions/download-artifact@v4 @@ -190,13 +290,22 @@ jobs: tar -xzf "build-artifacts/build-tree-${{ matrix.target }}.tar.gz" -C build - name: Run unit and integration tests + if: matrix.family != 'windows' run: bash .github/scripts/run-ctest.sh build + - name: Run unit and integration tests (Windows) + if: matrix.family == 'windows' + shell: msys2 {0} + run: | + export PATH="${MSYSTEM_PREFIX}/bin:${PATH}" + bash .github/scripts/run-ctest.sh build + system-tests-linux: name: system-linux-${{ matrix.target }} needs: - build-artifacts - unit-integration-tests + if: ${{ !cancelled() }} runs-on: ${{ matrix.runner }} strategy: fail-fast: false @@ -229,11 +338,18 @@ jobs: "${{ matrix.target }}" system-tests-macos: - name: system-macos-aarch64-darwin + name: system-macos-${{ matrix.target }} needs: - build-artifacts - unit-integration-tests - runs-on: macos-14 + if: ${{ !cancelled() }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-darwin + runner: macos-14 steps: - name: Checkout repository uses: actions/checkout@v4 @@ -241,11 +357,69 @@ jobs: - name: Download release bundle uses: actions/download-artifact@v4 with: - name: release-bundle-aarch64-darwin + name: release-bundle-${{ matrix.target }} path: release-artifacts - name: Run native system tests run: | bash .github/scripts/run-system-tests-macos.sh \ - "release-artifacts/rqp-${{ github.sha }}-aarch64-darwin.tar.gz" \ + "release-artifacts/rqp-${{ github.sha }}-${{ matrix.target }}.tar.gz" \ "${{ github.workspace }}" + + system-tests-windows: + name: system-windows-${{ matrix.target }} + needs: + - build-artifacts + - unit-integration-tests + if: ${{ !cancelled() }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-windows + runner: windows-latest + msystem: UCRT64 + msys_package_prefix: mingw-w64-ucrt-x86_64 + - target: aarch64-windows + runner: windows-11-arm + msystem: CLANGARM64 + msys_package_prefix: mingw-w64-clang-aarch64 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up MSYS2 + uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.msystem }} + update: true + install: >- + ${{ matrix.msys_package_prefix }}-toolchain + ${{ matrix.msys_package_prefix }}-lua54 + ${{ matrix.msys_package_prefix }}-curl + ${{ matrix.msys_package_prefix }}-openssl + ${{ matrix.msys_package_prefix }}-fmt + ${{ matrix.msys_package_prefix }}-spdlog + ${{ matrix.msys_package_prefix }}-boost + ${{ matrix.msys_package_prefix }}-zstd + + - name: Download build tree artifact + uses: actions/download-artifact@v4 + with: + name: build-tree-${{ matrix.target }} + path: build-artifacts + + - name: Restore build tree + run: | + rm -rf build + mkdir -p build + tar -xzf "build-artifacts/build-tree-${{ matrix.target }}.tar.gz" -C build + + - name: Run native system tests + shell: msys2 {0} + run: | + bash .github/scripts/run-system-tests-windows.sh \ + "${{ github.workspace }}/build/rqp.exe" \ + "${{ github.workspace }}" \ + "${MSYSTEM_PREFIX}" diff --git a/.gitignore b/.gitignore index 5e51a43..9a9cee5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ build/ gmon.out docs/ rqp-plugins/ + +build-check diff --git a/CMakeLists.txt b/CMakeLists.txt index 856490c..26fd14c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,8 @@ project(ReqPack LANGUAGES CXX) enable_language(C) enable_testing() +set(CMAKE_NO_SYSTEM_FROM_IMPORTED ON) + set(REQPACK_RELEASE_ID "v1.2.14" CACHE STRING "Build/release identity shown by rqp version output") option(REQPACK_ENABLE_COVERAGE "Build ReqPack targets with coverage instrumentation" OFF) @@ -55,6 +57,40 @@ find_library(REQPACK_ZSTD_LIBRARY NAMES zstd REQUIRED) find_package(fmt CONFIG REQUIRED) find_package(spdlog CONFIG REQUIRED) +if(NOT REQPACK_GO_EXECUTABLE) + unset(REQPACK_GO_EXECUTABLE CACHE) + find_program(REQPACK_GO_EXECUTABLE NAMES go go.exe + HINTS + "/opt/homebrew/bin" + "/usr/local/bin" + "/usr/bin" + "$ENV{HOME}/go/bin" + "/usr/local/go/bin" + ) +endif() +set(REQPACK_GO_EXECUTABLE "${REQPACK_GO_EXECUTABLE}" CACHE FILEPATH "Path to the Go compiler") +if(NOT REQPACK_GO_EXECUTABLE) + message(FATAL_ERROR "Go compiler not found. Install Go or pass -DREQPACK_GO_EXECUTABLE=/path/to/go") +endif() + +set(REQPACK_TRIVY_HELPER_DIR "${CMAKE_BINARY_DIR}/tools/trivydb-exporter") +set(REQPACK_TRIVY_HELPER_PATH "${REQPACK_TRIVY_HELPER_DIR}/trivydb-exporter") + +add_custom_command( + OUTPUT ${REQPACK_TRIVY_HELPER_PATH} + COMMAND ${CMAKE_COMMAND} -E make_directory ${REQPACK_TRIVY_HELPER_DIR} + COMMAND ${REQPACK_GO_EXECUTABLE} build -o ${REQPACK_TRIVY_HELPER_PATH} . + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tools/trivydb-exporter + DEPENDS + ${CMAKE_SOURCE_DIR}/tools/trivydb-exporter/go.mod + ${CMAKE_SOURCE_DIR}/tools/trivydb-exporter/go.sum + ${CMAKE_SOURCE_DIR}/tools/trivydb-exporter/main.go + COMMENT "Building Trivy DB helper" + VERBATIM +) + +add_custom_target(reqpack_trivy_helper ALL DEPENDS ${REQPACK_TRIVY_HELPER_PATH}) + set(REQPACK_LUA_LINK_LIBRARIES ${LUA_LIBRARIES}) if(REQPACK_LINK_STATIC_LUA) set(_reqpack_saved_library_suffixes ${CMAKE_FIND_LIBRARY_SUFFIXES}) @@ -86,12 +122,14 @@ if(REQPACK_LINK_STATIC_LUA) endif() set(REQPACK_FMT_TARGET fmt::fmt) -if(TARGET fmt::fmt-header-only) +# Prefer the compiled library on Windows/MinGW: header-only pulls format-inl.h and +# hits GCC 16 -Wtemplate-body failures around stdout/stderr in system packages. +if(NOT WIN32 AND TARGET fmt::fmt-header-only) set(REQPACK_FMT_TARGET fmt::fmt-header-only) endif() set(REQPACK_SPDLOG_TARGET spdlog::spdlog) -if(TARGET spdlog::spdlog_header_only) +if(NOT WIN32 AND TARGET spdlog::spdlog_header_only) set(REQPACK_SPDLOG_TARGET spdlog::spdlog_header_only) endif() @@ -99,7 +137,17 @@ if(APPLE) add_compile_options(-Wno-unknown-warning-option) endif() +# MinGW fmt/spdlog headers fail with undeclared stdout/stderr inside template bodies. +function(reqpack_apply_windows_compile_options target_name) + if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + target_compile_options(${target_name} PRIVATE -Wno-template-body) + endif() +endfunction() + set(REQPACK_PLATFORM_LIBS) +if(WIN32) + list(APPEND REQPACK_PLATFORM_LIBS ws2_32) +endif() if(UNIX AND NOT APPLE) list(APPEND REQPACK_PLATFORM_LIBS util) endif() @@ -125,12 +173,27 @@ find_path(SOL2_INCLUDE_DIR PATHS /usr/include /usr/local/include ) +if(SOL2_INCLUDE_DIR STREQUAL "/usr/include" OR SOL2_INCLUDE_DIR STREQUAL "/usr/local/include") + set(SOL2_INCLUDE_DIR "") +endif() + if(NOT SOL2_INCLUDE_DIR) - message(FATAL_ERROR "Sol2 Header nicht gefunden!") + if(NOT EXISTS "/usr/include/sol/sol.hpp" AND NOT EXISTS "/usr/local/include/sol/sol.hpp") + message(FATAL_ERROR "Sol2 Header nicht gefunden!") + endif() endif() include_directories(src/main/include) +set(REQPACK_PRIVATE_INCLUDE_DIRS + ${SOL2_INCLUDE_DIR} + src/main/include +) + +if(LUA_INCLUDE_DIR AND NOT LUA_INCLUDE_DIR STREQUAL "/usr/include") + list(APPEND REQPACK_PRIVATE_INCLUDE_DIRS ${LUA_INCLUDE_DIR}) +endif() + set(SOURCES src/main/cpp/main.cpp src/main/cpp/main_arg_parsing.cpp @@ -153,6 +216,7 @@ set(SOURCES src/main/cpp/output/command_output.cpp src/main/cpp/output/command_output_projection.cpp src/main/cpp/output/command_output_render.cpp + src/main/cpp/output/run_result_json.cpp src/main/cpp/output/ansi_color.cpp src/main/cpp/output/plain_display.cpp src/main/cpp/output/plain_display_session.cpp @@ -181,6 +245,8 @@ set(SOURCES src/main/cpp/core/archive/archive_resolver_process.cpp src/main/cpp/core/archive/archive_resolver_extract.cpp src/main/cpp/core/common/network_environment.cpp + src/main/cpp/core/common/process_runner.cpp + src/main/cpp/core/common/pipe_helpers.cpp src/main/cpp/core/common/version_compare.cpp src/main/cpp/core/config/configuration.cpp src/main/cpp/core/config/configuration_cli_overrides.cpp @@ -272,7 +338,10 @@ set(SOURCES src/main/cpp/core/remote/serve_remote_session.cpp src/main/cpp/core/remote/serve_remote_socket.cpp src/main/cpp/core/remote/serve_remote_upload.cpp + src/main/cpp/core/security/gh_advisory_vulnerability_sync_service.cpp src/main/cpp/core/security/osv_core.cpp + src/main/cpp/core/security/snyk_vulnerability_sync_service.cpp + src/main/cpp/core/security/trivy_vulnerability_sync_service.cpp src/main/cpp/core/security/security_gateway_service.cpp src/main/cpp/core/security/validator.cpp src/main/cpp/core/security/validator_prepare.cpp @@ -307,10 +376,17 @@ add_executable(exec_rules_integration_tests src/main/cpp/plugins/exec_rules_core.cpp ) +set(REQPACK_ORCHESTRATOR_INTEGRATION_TEST_SOURCES) +if(NOT WIN32) + list(APPEND REQPACK_ORCHESTRATOR_INTEGRATION_TEST_SOURCES + tests/integration/core/execution/orchestrator_service_tests.cpp + ) +endif() + add_executable(core_integration_tests tests/cpp/test_main.cpp tests/integration/core/execution/executor_service_tests.cpp - tests/integration/core/execution/orchestrator_service_tests.cpp + ${REQPACK_ORCHESTRATOR_INTEGRATION_TEST_SOURCES} tests/integration/core/plugins/plugin_test_runner_service_tests.cpp tests/integration/core/planning/planner_service_tests.cpp tests/integration/core/registry/registry_service_tests.cpp @@ -330,7 +406,10 @@ add_executable(core_integration_tests src/main/cpp/core/host/host_info.cpp src/main/cpp/core/host/host_info_codec.cpp src/main/cpp/core/host/host_info_probe.cpp + src/main/cpp/core/security/gh_advisory_vulnerability_sync_service.cpp src/main/cpp/core/security/osv_core.cpp + src/main/cpp/core/security/snyk_vulnerability_sync_service.cpp + src/main/cpp/core/security/trivy_vulnerability_sync_service.cpp src/main/cpp/core/planning/planner.cpp src/main/cpp/core/planning/planner_request_setup.cpp src/main/cpp/core/planning/planner_dependency_policy.cpp @@ -358,6 +437,7 @@ add_executable(core_integration_tests src/main/cpp/output/command_output.cpp src/main/cpp/output/command_output_projection.cpp src/main/cpp/output/command_output_render.cpp + src/main/cpp/output/run_result_json.cpp src/main/cpp/output/ansi_color.cpp src/main/cpp/output/plain_display.cpp src/main/cpp/output/plain_display_session.cpp @@ -386,6 +466,7 @@ add_executable(core_integration_tests src/main/cpp/core/archive/archive_resolver_process.cpp src/main/cpp/core/archive/archive_resolver_extract.cpp src/main/cpp/core/common/version_compare.cpp + src/main/cpp/core/common/pipe_helpers.cpp src/main/cpp/core/config/configuration.cpp src/main/cpp/core/config/configuration_cli_overrides.cpp src/main/cpp/core/config/configuration_loader.cpp @@ -403,6 +484,7 @@ add_executable(core_integration_tests src/main/cpp/core/history/history_manager_log.cpp src/main/cpp/core/history/history_manager_store.cpp src/main/cpp/core/common/network_environment.cpp + src/main/cpp/core/common/process_runner.cpp src/main/cpp/core/manifest/manifest_loader.cpp src/main/cpp/core/packages/rq_package.cpp src/main/cpp/core/packages/rq_package_metadata.cpp @@ -465,6 +547,8 @@ add_executable(core_integration_tests ) add_dependencies(core_integration_tests ReqPack) +add_dependencies(ReqPack reqpack_trivy_helper) +add_dependencies(core_integration_tests reqpack_trivy_helper) add_executable(core_unit_tests tests/cpp/test_main.cpp @@ -483,7 +567,10 @@ add_executable(core_unit_tests tests/unit/core/planning/planner_core_tests.cpp tests/unit/core/registry/registry_database_tests.cpp tests/unit/core/registry/registry_json_parser_tests.cpp + tests/unit/core/security/gh_advisory_vulnerability_sync_service_tests.cpp tests/unit/core/security/osv_core_tests.cpp + tests/unit/core/security/snyk_vulnerability_sync_service_tests.cpp + tests/unit/core/security/trivy_vulnerability_sync_service_tests.cpp tests/unit/core/security/security_gateway_service_tests.cpp tests/unit/core/security/validator_policy_tests.cpp tests/unit/core/security/validator_validate_tests.cpp @@ -491,7 +578,9 @@ add_executable(core_unit_tests tests/unit/core/state/rqp_state_store_tests.cpp tests/unit/core/state/transaction_database_serialization_tests.cpp tests/unit/output/logger/format_tests.cpp + tests/unit/output/run_result_json_tests.cpp src/main/cpp/core/common/version_compare.cpp + src/main/cpp/core/common/pipe_helpers.cpp src/main/cpp/core/config/configuration.cpp src/main/cpp/core/config/configuration_cli_overrides.cpp src/main/cpp/core/config/configuration_loader.cpp @@ -532,6 +621,7 @@ add_executable(core_unit_tests src/main/cpp/core/history/history_manager_log.cpp src/main/cpp/core/history/history_manager_store.cpp src/main/cpp/core/common/network_environment.cpp + src/main/cpp/core/common/process_runner.cpp src/main/cpp/core/packages/rq_package.cpp src/main/cpp/core/packages/rq_package_metadata.cpp src/main/cpp/core/packages/rq_package_payload.cpp @@ -551,7 +641,10 @@ add_executable(core_unit_tests src/main/cpp/core/registry/registry_database_core.cpp src/main/cpp/core/registry/registry_json_parser.cpp src/main/cpp/core/remote/remote_profiles.cpp + src/main/cpp/core/security/gh_advisory_vulnerability_sync_service.cpp src/main/cpp/core/security/osv_core.cpp + src/main/cpp/core/security/snyk_vulnerability_sync_service.cpp + src/main/cpp/core/security/trivy_vulnerability_sync_service.cpp src/main/cpp/core/security/security_gateway_service.cpp src/main/cpp/core/security/validator.cpp src/main/cpp/core/security/validator_prepare.cpp @@ -576,6 +669,7 @@ add_executable(core_unit_tests src/main/cpp/output/command_output.cpp src/main/cpp/output/command_output_projection.cpp src/main/cpp/output/command_output_render.cpp + src/main/cpp/output/run_result_json.cpp src/main/cpp/output/plain_display.cpp src/main/cpp/output/plain_display_session.cpp src/main/cpp/output/plain_display_table.cpp @@ -583,6 +677,8 @@ add_executable(core_unit_tests src/main/cpp/output/command_output.cpp ) +add_dependencies(core_unit_tests reqpack_trivy_helper) + add_custom_target(reqpack_test_targets DEPENDS core_unit_tests @@ -602,36 +698,26 @@ foreach(reqpack_target IN ITEMS endforeach() target_include_directories(ReqPack PRIVATE - ${SOL2_INCLUDE_DIR} - ${LUA_INCLUDE_DIR} - src/main/include + ${REQPACK_PRIVATE_INCLUDE_DIRS} ) target_include_directories(exec_rules_unit_tests PRIVATE - ${SOL2_INCLUDE_DIR} - ${LUA_INCLUDE_DIR} - src/main/include + ${REQPACK_PRIVATE_INCLUDE_DIRS} tests/cpp ) target_include_directories(exec_rules_integration_tests PRIVATE - ${SOL2_INCLUDE_DIR} - ${LUA_INCLUDE_DIR} - src/main/include + ${REQPACK_PRIVATE_INCLUDE_DIRS} tests/cpp ) target_include_directories(core_unit_tests PRIVATE - ${SOL2_INCLUDE_DIR} - ${LUA_INCLUDE_DIR} - src/main/include + ${REQPACK_PRIVATE_INCLUDE_DIRS} tests/cpp ) target_include_directories(core_integration_tests PRIVATE - ${SOL2_INCLUDE_DIR} - ${LUA_INCLUDE_DIR} - src/main/include + ${REQPACK_PRIVATE_INCLUDE_DIRS} tests/cpp ) @@ -691,6 +777,7 @@ target_link_libraries(core_integration_tests PRIVATE target_compile_definitions(ReqPack PRIVATE SOL_ALL_SAFETIES_ON=1) target_compile_definitions(ReqPack PRIVATE REQPACK_RELEASE_ID="${REQPACK_RELEASE_ID}") +target_compile_definitions(ReqPack PRIVATE REQPACK_TRIVY_HELPER_DEFAULT_PATH="${REQPACK_TRIVY_HELPER_PATH}") target_compile_definitions(exec_rules_unit_tests PRIVATE SOL_ALL_SAFETIES_ON=1 REQPACK_TEST_REPO_ROOT="${CMAKE_SOURCE_DIR}" @@ -706,14 +793,20 @@ target_compile_definitions(core_unit_tests PRIVATE REQPACK_TEST_REPO_ROOT="${CMAKE_SOURCE_DIR}" REQPACK_TEST_BUILD_DIR="${CMAKE_BINARY_DIR}" REQPACK_RELEASE_ID="${REQPACK_RELEASE_ID}" + REQPACK_TRIVY_HELPER_DEFAULT_PATH="${REQPACK_TRIVY_HELPER_PATH}" ) target_compile_definitions(core_integration_tests PRIVATE SOL_ALL_SAFETIES_ON=1 REQPACK_TEST_REPO_ROOT="${CMAKE_SOURCE_DIR}" REQPACK_TEST_BUILD_DIR="${CMAKE_BINARY_DIR}" REQPACK_RELEASE_ID="${REQPACK_RELEASE_ID}" + REQPACK_TRIVY_HELPER_DEFAULT_PATH="${REQPACK_TRIVY_HELPER_PATH}" ) +foreach(_reqpack_target IN ITEMS ReqPack exec_rules_unit_tests exec_rules_integration_tests core_unit_tests core_integration_tests) + reqpack_apply_windows_compile_options(${_reqpack_target}) +endforeach() + include(CTest) include(${catch2_SOURCE_DIR}/contrib/Catch.cmake) catch_discover_tests(exec_rules_unit_tests TEST_PREFIX "unit::") diff --git a/src/main/cpp/cli/cli_help_text.cpp b/src/main/cpp/cli/cli_help_text.cpp index 4e33384..6112c5a 100644 --- a/src/main/cpp/cli/cli_help_text.cpp +++ b/src/main/cpp/cli/cli_help_text.cpp @@ -52,6 +52,7 @@ std::string general_help_text() { "\nOptions:\n" " -h,--help " + HELP_DESCRIPTION + "\n" " -v,--verbose " + VERBOSE_DESCRIPTION + "\n" + " --json Emit machine-readable JSON result on stdout\n" "\nCommands:\n" " install (i) Installs requested packages\n" " remove (rm) Removes requested packages\n" @@ -134,6 +135,7 @@ std::string command_help_text(ActionType action) { " -h,--help Displays this help\n" " --stdin Read install payload lines or commands from stdin until EOF\n" " --dry-run Show planned actions without executing them\n" + " --json Emit machine-readable JSON result on stdout\n" " --prompt-on-unsafe Prompt before installing vulnerable packages\n" " --abort-on-unsafe Abort if vulnerable packages are found\n" " --severity-threshold Minimum severity to flag (low/medium/high/critical)\n" @@ -181,6 +183,7 @@ std::string command_help_text(ActionType action) { " -h,--help Displays this help\n" " --stdin Read remove commands from stdin until EOF\n" " --dry-run Show planned actions without executing them\n" + " --json Emit machine-readable JSON result on stdout\n" " --jobs Use exactly n workers for independent groups\n" " --jobs-max Use all logical CPU threads as workers\n" " --stop-on-first-failure Stop after the first failing system\n" @@ -224,6 +227,7 @@ std::string command_help_text(ActionType action) { " -h,--help Displays this help\n" " --stdin Read update payload lines or commands from stdin until EOF\n" " --dry-run Show planned actions without executing them\n" + " --json Emit machine-readable JSON result on stdout\n" " --all Update all packages for system, or all plugin wrappers without a system\n" " --prompt-on-unsafe Prompt before applying vulnerable updates\n" " --abort-on-unsafe Abort if vulnerable packages are found\n" diff --git a/src/main/cpp/core/archive/archive_resolver_process.cpp b/src/main/cpp/core/archive/archive_resolver_process.cpp index 8fae807..2050708 100644 --- a/src/main/cpp/core/archive/archive_resolver_process.cpp +++ b/src/main/cpp/core/archive/archive_resolver_process.cpp @@ -1,5 +1,9 @@ #include "archive_resolver_internal.h" +#include "core/common/pipe_helpers.h" +#include "core/common/process_runner.h" +#include "core/common/temp_directory.h" + #include #include #include @@ -11,15 +15,27 @@ #include #include #include -#include -#include #include +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#else #include +#include +#include #include +#endif namespace { +#if !defined(_WIN32) int normalize_exit_code(const int status) { if (status == -1) { return -1; @@ -32,10 +48,31 @@ int normalize_exit_code(const int status) { } return status; } +#endif class TerminalEchoGuard { public: explicit TerminalEchoGuard(const int fd) : fd_(fd) { +#if defined(_WIN32) + if (fd_ < 0 || !_isatty(fd_)) { + return; + } + + const HANDLE handle = reinterpret_cast(_get_osfhandle(fd_)); + if (handle == INVALID_HANDLE_VALUE) { + return; + } + + DWORD mode = 0; + if (!GetConsoleMode(handle, &mode)) { + return; + } + + originalMode_ = mode; + if (SetConsoleMode(handle, mode & ~ENABLE_ECHO_INPUT)) { + active_ = true; + } +#else if (fd_ < 0 || ::tcgetattr(fd_, &original_) != 0) { return; } @@ -45,12 +82,21 @@ class TerminalEchoGuard { if (::tcsetattr(fd_, TCSAFLUSH, &updated) == 0) { active_ = true; } +#endif } ~TerminalEchoGuard() { - if (active_) { - (void)::tcsetattr(fd_, TCSAFLUSH, &original_); + if (!active_) { + return; + } +#if defined(_WIN32) + const HANDLE handle = reinterpret_cast(_get_osfhandle(fd_)); + if (handle != INVALID_HANDLE_VALUE) { + (void)SetConsoleMode(handle, originalMode_); } +#else + (void)::tcsetattr(fd_, TCSAFLUSH, &original_); +#endif } bool active() const { @@ -59,7 +105,11 @@ class TerminalEchoGuard { private: int fd_{-1}; +#if defined(_WIN32) + DWORD originalMode_{0}; +#else termios original_{}; +#endif bool active_{false}; }; @@ -130,21 +180,7 @@ std::filesystem::path single_file_archive_output_path( } std::filesystem::path make_unique_directory(const std::filesystem::path& root, const std::string& prefix) { - std::error_code error; - std::filesystem::create_directories(root, error); - if (error) { - throw std::runtime_error("failed to create archive temp root: " + root.string()); - } - - const std::filesystem::path pattern = root / (prefix + "-XXXXXX"); - std::string templateString = pattern.string(); - std::vector buffer(templateString.begin(), templateString.end()); - buffer.push_back('\0'); - char* created = ::mkdtemp(buffer.data()); - if (created == nullptr) { - throw std::runtime_error("failed to create archive temp directory"); - } - return created; + return reqpack_make_unique_directory(root, prefix); } std::filesystem::path make_unique_file_path(const std::filesystem::path& root, const std::string& stem, const std::string& suffix) { @@ -177,6 +213,13 @@ std::string trim_line(std::string value) { } std::string run_command_capture(const std::string& command) { +#if defined(_WIN32) + const ReqpackProcessResult captured = reqpack_run_process_capture({"bash", "-c", command}); + if (!captured.success()) { + throw std::runtime_error("failed to inspect archive"); + } + return captured.stdoutText; +#else FILE* pipe = ::popen(command.c_str(), "r"); if (pipe == nullptr) { throw std::runtime_error("failed to inspect archive"); @@ -192,11 +235,25 @@ std::string run_command_capture(const std::string& command) { throw std::runtime_error("failed to inspect archive"); } return output; +#endif } CommandResult run_command_capture_status(const std::string& command) { +#if defined(_WIN32) + const ReqpackProcessResult captured = reqpack_run_process_capture({"bash", "-c", command}); + CommandResult result; + result.exitCode = captured.exitCode; + result.output = captured.stdoutText; + if (!captured.stderrText.empty()) { + if (!result.output.empty()) { + result.output.push_back('\n'); + } + result.output += captured.stderrText; + } + return result; +#else int outputPipe[2]; - if (::pipe(outputPipe) != 0) { + if (!create_pipe_cloexec(outputPipe)) { throw std::runtime_error("failed to run archive command"); } @@ -256,6 +313,7 @@ CommandResult run_command_capture_status(const std::string& command) { } return CommandResult{.exitCode = normalize_exit_code(status), .output = std::move(output)}; +#endif } bool path_has_invalid_segments(const std::filesystem::path& path) { diff --git a/src/main/cpp/core/common/pipe_helpers.cpp b/src/main/cpp/core/common/pipe_helpers.cpp new file mode 100644 index 0000000..3724c1c --- /dev/null +++ b/src/main/cpp/core/common/pipe_helpers.cpp @@ -0,0 +1,45 @@ +#include "core/common/pipe_helpers.h" + +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#include +#endif + +bool create_pipe_cloexec(int pipefd[2]) { +#if defined(_WIN32) + (void)pipefd; + return false; +#elif defined(__linux__) + if (::pipe2(pipefd, O_CLOEXEC) == 0) { + return true; + } + if (::pipe(pipefd) != 0) { + return false; + } + fcntl(pipefd[0], F_SETFD, FD_CLOEXEC); + fcntl(pipefd[1], F_SETFD, FD_CLOEXEC); + return true; +#elif defined(__APPLE__) + if (::pipe(pipefd) != 0) { + return false; + } + fcntl(pipefd[0], F_SETFD, FD_CLOEXEC); + fcntl(pipefd[1], F_SETFD, FD_CLOEXEC); + return true; +#else + if (::pipe(pipefd) != 0) { + return false; + } + fcntl(pipefd[0], F_SETFD, FD_CLOEXEC); + fcntl(pipefd[1], F_SETFD, FD_CLOEXEC); + return true; +#endif +} diff --git a/src/main/cpp/core/common/process_runner.cpp b/src/main/cpp/core/common/process_runner.cpp new file mode 100644 index 0000000..a942335 --- /dev/null +++ b/src/main/cpp/core/common/process_runner.cpp @@ -0,0 +1,510 @@ +#include "core/common/process_runner.h" + +#include "core/common/network_environment.h" + +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#include +#include +#include + +#include "core/common/pipe_helpers.h" +#endif + +namespace { + +#if defined(_WIN32) + +std::string quote_windows_argument(const std::string& argument) { + if (argument.empty()) { + return "\"\""; + } + + bool needsQuotes = false; + for (char ch : argument) { + if (ch == ' ' || ch == '\t' || ch == '"') { + needsQuotes = true; + break; + } + } + if (!needsQuotes) { + return argument; + } + + std::string quoted = "\""; + std::size_t backslashes = 0; + for (char ch : argument) { + if (ch == '\\') { + ++backslashes; + continue; + } + if (ch == '"') { + quoted.append(backslashes * 2 + 1, '\\'); + backslashes = 0; + quoted.push_back('"'); + continue; + } + if (backslashes != 0) { + quoted.append(backslashes, '\\'); + backslashes = 0; + } + quoted.push_back(ch); + } + quoted.append(backslashes * 2, '\\'); + quoted.push_back('"'); + return quoted; +} + +std::string build_windows_command_line(const std::vector& arguments) { + std::string commandLine; + for (const std::string& argument : arguments) { + if (!commandLine.empty()) { + commandLine.push_back(' '); + } + commandLine += quote_windows_argument(argument); + } + return commandLine; +} + +std::vector build_windows_environment_block(const std::vector& environment) { + std::size_t totalSize = 1; + for (const std::string& entry : environment) { + totalSize += entry.size() + 1; + } + + std::vector block(totalSize, '\0'); + std::size_t offset = 0; + for (const std::string& entry : environment) { + std::memcpy(block.data() + offset, entry.data(), entry.size()); + offset += entry.size() + 1; + } + return block; +} + +HANDLE open_windows_nul_handle() { + SECURITY_ATTRIBUTES securityAttributes{}; + securityAttributes.nLength = sizeof(securityAttributes); + securityAttributes.bInheritHandle = TRUE; + + return CreateFileA( + "NUL", + GENERIC_READ, + FILE_SHARE_READ, + &securityAttributes, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + nullptr + ); +} + +bool read_windows_pipe(HANDLE pipe, std::string& output, std::string& errorText) { + std::array buffer{}; + for (;;) { + DWORD bytesRead = 0; + if (!ReadFile(pipe, buffer.data(), static_cast(buffer.size()), &bytesRead, nullptr)) { + const DWORD error = GetLastError(); + if (error == ERROR_BROKEN_PIPE) { + return true; + } + errorText = "read failed: Windows error " + std::to_string(error); + return false; + } + if (bytesRead == 0) { + return true; + } + output.append(buffer.data(), bytesRead); + } +} + +struct WindowsProcessHandles { + HANDLE process{nullptr}; + HANDLE thread{nullptr}; + HANDLE stdoutRead{nullptr}; + HANDLE stderrRead{nullptr}; + HANDLE stdinHandle{nullptr}; + HANDLE stdoutWrite{nullptr}; + HANDLE stderrWrite{nullptr}; +}; + +void close_windows_handle(HANDLE& handle) { + if (handle != nullptr && handle != INVALID_HANDLE_VALUE) { + CloseHandle(handle); + } + handle = nullptr; +} + +void close_windows_process_handles(WindowsProcessHandles& handles) { + close_windows_handle(handles.stdoutWrite); + close_windows_handle(handles.stderrWrite); + close_windows_handle(handles.stdinHandle); + close_windows_handle(handles.stdoutRead); + close_windows_handle(handles.stderrRead); + close_windows_handle(handles.thread); + close_windows_handle(handles.process); +} + +std::optional start_windows_process( + const std::vector& arguments, + const std::filesystem::path& workingDirectory, + bool captureOutput, + std::string& errorText +) { + if (arguments.empty()) { + errorText = "empty command"; + return std::nullopt; + } + + SECURITY_ATTRIBUTES securityAttributes{}; + securityAttributes.nLength = sizeof(securityAttributes); + securityAttributes.bInheritHandle = TRUE; + + WindowsProcessHandles handles{}; + + if (captureOutput) { + HANDLE stdoutRead = nullptr; + HANDLE stdoutWrite = nullptr; + HANDLE stderrRead = nullptr; + HANDLE stderrWrite = nullptr; + if (!CreatePipe(&stdoutRead, &stdoutWrite, &securityAttributes, 0) || + !CreatePipe(&stderrRead, &stderrWrite, &securityAttributes, 0)) { + errorText = "failed to create pipe: Windows error " + std::to_string(GetLastError()); + close_windows_handle(stdoutRead); + close_windows_handle(stdoutWrite); + close_windows_handle(stderrRead); + close_windows_handle(stderrWrite); + return std::nullopt; + } + if (!SetHandleInformation(stdoutRead, HANDLE_FLAG_INHERIT, 0) || + !SetHandleInformation(stderrRead, HANDLE_FLAG_INHERIT, 0)) { + errorText = "failed to configure pipe inheritance: Windows error " + std::to_string(GetLastError()); + close_windows_handle(stdoutRead); + close_windows_handle(stdoutWrite); + close_windows_handle(stderrRead); + close_windows_handle(stderrWrite); + return std::nullopt; + } + handles.stdoutRead = stdoutRead; + handles.stderrRead = stderrRead; + handles.stdoutWrite = stdoutWrite; + handles.stderrWrite = stderrWrite; + } + + handles.stdinHandle = open_windows_nul_handle(); + if (handles.stdinHandle == INVALID_HANDLE_VALUE) { + errorText = "failed to open NUL: Windows error " + std::to_string(GetLastError()); + close_windows_process_handles(handles); + return std::nullopt; + } + + STARTUPINFOA startupInfo{}; + startupInfo.cb = sizeof(startupInfo); + startupInfo.dwFlags = STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW; + startupInfo.wShowWindow = SW_HIDE; + startupInfo.hStdInput = handles.stdinHandle; + startupInfo.hStdOutput = captureOutput ? handles.stdoutWrite : GetStdHandle(STD_OUTPUT_HANDLE); + startupInfo.hStdError = captureOutput ? handles.stderrWrite : GetStdHandle(STD_ERROR_HANDLE); + + PROCESS_INFORMATION processInfo{}; + std::string commandLine = build_windows_command_line(arguments); + std::vector commandLineBuffer(commandLine.begin(), commandLine.end()); + commandLineBuffer.push_back('\0'); + + std::vector environmentStorage = reqpack_sanitized_process_environment(); + std::vector environmentBlock = build_windows_environment_block(environmentStorage); + + const std::string workingDirectoryString = workingDirectory.empty() ? std::string{} : workingDirectory.string(); + const char* workingDirectoryPointer = workingDirectoryString.empty() ? nullptr : workingDirectoryString.c_str(); + + const BOOL created = CreateProcessA( + arguments.front().c_str(), + commandLineBuffer.data(), + nullptr, + nullptr, + TRUE, + CREATE_NO_WINDOW, + environmentBlock.data(), + workingDirectoryPointer, + &startupInfo, + &processInfo + ); + + close_windows_handle(handles.stdoutWrite); + close_windows_handle(handles.stderrWrite); + close_windows_handle(handles.stdinHandle); + + if (!created) { + errorText = "spawn failed: Windows error " + std::to_string(GetLastError()); + close_windows_handle(handles.stdoutRead); + close_windows_handle(handles.stderrRead); + return std::nullopt; + } + + handles.process = processInfo.hProcess; + handles.thread = processInfo.hThread; + return handles; +} + +bool wait_for_windows_process(HANDLE process, int& exitCode, std::string& errorText) { + if (WaitForSingleObject(process, INFINITE) != WAIT_OBJECT_0) { + errorText = "wait failed: Windows error " + std::to_string(GetLastError()); + return false; + } + + DWORD windowsExitCode = 1; + if (!GetExitCodeProcess(process, &windowsExitCode)) { + errorText = "failed to read exit code: Windows error " + std::to_string(GetLastError()); + return false; + } + + exitCode = static_cast(windowsExitCode); + return true; +} + +#else + +bool wait_for_posix_process(pid_t pid, int& exitCode, std::string& errorText) { + int status = 0; + while (waitpid(pid, &status, 0) == -1) { + if (errno != EINTR) { + errorText = std::string{"waitpid failed: "} + std::strerror(errno); + return false; + } + } + + if (WIFEXITED(status)) { + exitCode = WEXITSTATUS(status); + return true; + } + + if (WIFSIGNALED(status)) { + exitCode = 128 + WTERMSIG(status); + errorText = "process terminated by signal"; + return false; + } + + exitCode = 1; + errorText = "process exited abnormally"; + return false; +} + +bool read_posix_fd(int fd, std::string& output, std::string& errorText) { + char buffer[4096]; + while (true) { + const ssize_t bytesRead = ::read(fd, buffer, sizeof(buffer)); + if (bytesRead > 0) { + output.append(buffer, static_cast(bytesRead)); + continue; + } + if (bytesRead == 0) { + return true; + } + if (errno == EINTR) { + continue; + } + errorText += std::string{"read failed: "} + std::strerror(errno); + return false; + } +} + +std::optional start_posix_process( + const std::vector& arguments, + const std::filesystem::path& workingDirectory, + int stdoutReadFd, + int stdoutWriteFd, + int stderrReadFd, + int stderrWriteFd, + std::string& errorText +) { + posix_spawn_file_actions_t fileActions; + if (posix_spawn_file_actions_init(&fileActions) != 0) { + errorText = "posix_spawn_file_actions_init failed"; + return std::nullopt; + } + + bool ready = true; +#if defined(__linux__) || defined(__APPLE__) + if (!workingDirectory.empty()) { + ready = posix_spawn_file_actions_addchdir_np(&fileActions, workingDirectory.c_str()) == 0; + } +#endif + + if (ready && stdoutWriteFd >= 0 && stderrWriteFd >= 0) { + ready = + posix_spawn_file_actions_addopen(&fileActions, STDIN_FILENO, "/dev/null", O_RDONLY, 0) == 0 && + posix_spawn_file_actions_adddup2(&fileActions, stdoutWriteFd, STDOUT_FILENO) == 0 && + posix_spawn_file_actions_adddup2(&fileActions, stderrWriteFd, STDERR_FILENO) == 0 && + posix_spawn_file_actions_addclose(&fileActions, stdoutReadFd) == 0 && + posix_spawn_file_actions_addclose(&fileActions, stdoutWriteFd) == 0 && + posix_spawn_file_actions_addclose(&fileActions, stderrReadFd) == 0 && + posix_spawn_file_actions_addclose(&fileActions, stderrWriteFd) == 0; + } + + if (!ready) { + posix_spawn_file_actions_destroy(&fileActions); + errorText = "failed to configure process spawn"; + return std::nullopt; + } + + std::vector argv; + argv.reserve(arguments.size() + 1); + for (const std::string& argument : arguments) { + argv.push_back(const_cast(argument.c_str())); + } + argv.push_back(nullptr); + + std::vector environmentStorage = reqpack_sanitized_process_environment(); + std::vector environmentPointers; + environmentPointers.reserve(environmentStorage.size() + 1); + for (std::string& entry : environmentStorage) { + environmentPointers.push_back(entry.data()); + } + environmentPointers.push_back(nullptr); + + pid_t pid = 0; + const int spawnResult = posix_spawnp( + &pid, + arguments.front().c_str(), + &fileActions, + nullptr, + argv.data(), + environmentPointers.data() + ); + posix_spawn_file_actions_destroy(&fileActions); + if (spawnResult != 0) { + errorText = std::string{"spawn failed: "} + std::strerror(spawnResult); + return std::nullopt; + } + + return pid; +} + +#endif + +} // namespace + +bool reqpack_run_process( + const std::vector& arguments, + const std::filesystem::path& workingDirectory +) { +#if defined(_WIN32) + std::string errorText; + std::optional handles = start_windows_process(arguments, workingDirectory, false, errorText); + if (!handles.has_value()) { + return false; + } + + int exitCode = 1; + const bool waited = wait_for_windows_process(handles->process, exitCode, errorText); + close_windows_process_handles(handles.value()); + return waited && exitCode == 0; +#else + std::string errorText; + const std::optional pid = start_posix_process(arguments, workingDirectory, -1, -1, -1, -1, errorText); + if (!pid.has_value()) { + return false; + } + + int exitCode = 1; + const bool waited = wait_for_posix_process(pid.value(), exitCode, errorText); + return waited && exitCode == 0; +#endif +} + +ReqpackProcessResult reqpack_run_process_capture( + const std::vector& arguments, + const std::filesystem::path& workingDirectory +) { + ReqpackProcessResult result; + if (arguments.empty()) { + result.stderrText = "empty command"; + return result; + } + +#if defined(_WIN32) + std::string errorText; + std::optional handles = start_windows_process(arguments, workingDirectory, true, errorText); + if (!handles.has_value()) { + result.stderrText = errorText; + return result; + } + + (void)read_windows_pipe(handles->stdoutRead, result.stdoutText, result.stderrText); + (void)read_windows_pipe(handles->stderrRead, result.stderrText, result.stderrText); + + int exitCode = 1; + if (!wait_for_windows_process(handles->process, exitCode, errorText)) { + if (result.stderrText.empty()) { + result.stderrText = errorText; + } else { + result.stderrText += "\n" + errorText; + } + } + result.exitCode = exitCode; + close_windows_process_handles(handles.value()); + return result; +#else + int stdoutPipe[2]; + if (!create_pipe_cloexec(stdoutPipe)) { + result.stderrText = std::string{"pipe(stdout) failed: "} + std::strerror(errno); + return result; + } + + int stderrPipe[2]; + if (!create_pipe_cloexec(stderrPipe)) { + result.stderrText = std::string{"pipe(stderr) failed: "} + std::strerror(errno); + (void)::close(stdoutPipe[0]); + (void)::close(stdoutPipe[1]); + return result; + } + + std::string errorText; + const std::optional pid = start_posix_process( + arguments, + workingDirectory, + stdoutPipe[0], + stdoutPipe[1], + stderrPipe[0], + stderrPipe[1], + errorText + ); + (void)::close(stdoutPipe[1]); + (void)::close(stderrPipe[1]); + if (!pid.has_value()) { + result.stderrText = errorText; + (void)::close(stdoutPipe[0]); + (void)::close(stderrPipe[0]); + return result; + } + + (void)read_posix_fd(stdoutPipe[0], result.stdoutText, result.stderrText); + (void)::close(stdoutPipe[0]); + (void)read_posix_fd(stderrPipe[0], result.stderrText, result.stderrText); + (void)::close(stderrPipe[0]); + + int exitCode = 1; + if (!wait_for_posix_process(pid.value(), exitCode, errorText)) { + if (result.stderrText.empty()) { + result.stderrText = errorText; + } else { + result.stderrText += "\n" + errorText; + } + } + result.exitCode = exitCode; + return result; +#endif +} diff --git a/src/main/cpp/core/config/configuration_cli_overrides.cpp b/src/main/cpp/core/config/configuration_cli_overrides.cpp index d2bdbb5..4af5ea6 100644 --- a/src/main/cpp/core/config/configuration_cli_overrides.cpp +++ b/src/main/cpp/core/config/configuration_cli_overrides.cpp @@ -39,6 +39,7 @@ ReqPackConfig apply_config_overrides(const ReqPackConfig& base, const ReqPackCon if (overrides.reportOutputPath.has_value()) config.reports.outputPath = configuration_internal::expand_user_path(overrides.reportOutputPath.value()).string(); if (overrides.dryRun.has_value()) config.execution.dryRun = overrides.dryRun.value(); + if (overrides.jsonOutput.has_value()) config.display.jsonOutput = overrides.jsonOutput.value(); if (overrides.stopOnFirstFailure.has_value()) config.execution.stopOnFirstFailure = overrides.stopOnFirstFailure.value(); if (overrides.useTransactionDb.has_value()) config.execution.useTransactionDb = overrides.useTransactionDb.value(); if (overrides.jobs.has_value()) config.execution.jobs = std::max(1u, overrides.jobs.value()); @@ -196,6 +197,10 @@ bool consume_cli_config_flag(const std::vector& arguments, std::siz overrides.dryRun = true; return true; } + if (argument == "--json") { + overrides.jsonOutput = true; + return true; + } if (argument == "--jobs") { if (!require_value(value)) { set_error("missing value for --jobs"); diff --git a/src/main/cpp/core/config/configuration_internal.h b/src/main/cpp/core/config/configuration_internal.h index b3a70bf..3a0093c 100644 --- a/src/main/cpp/core/config/configuration_internal.h +++ b/src/main/cpp/core/config/configuration_internal.h @@ -46,6 +46,31 @@ inline std::optional unsigned_int_from_string(const std::string& v } } +inline constexpr const char* default_osv_feed_url() { + return "https://storage.googleapis.com/osv-vulnerabilities"; +} + +inline std::string resolve_osv_feed_url(const ReqPackConfig& config, const std::string& backendFeedUrl) { + const std::string& topLevel = config.security.osvFeedUrl; + if (!backendFeedUrl.empty()) { + if (backendFeedUrl == default_osv_feed_url() && topLevel != default_osv_feed_url()) { + return topLevel; + } + return backendFeedUrl; + } + return topLevel; +} + +inline void sync_osv_backend_feed_url(ReqPackConfig& config) { + if (!config.security.backends.contains("osv")) { + return; + } + SecurityBackendConfig& osvBackend = config.security.backends["osv"]; + if (osvBackend.feedUrl == default_osv_feed_url() && config.security.osvFeedUrl != default_osv_feed_url()) { + osvBackend.feedUrl = config.security.osvFeedUrl; + } +} + inline std::vector normalize_string_list(std::vector values) { for (std::string& value : values) { value = to_lower_copy(value); diff --git a/src/main/cpp/core/config/configuration_loader.cpp b/src/main/cpp/core/config/configuration_loader.cpp index 3056ec0..2041c12 100644 --- a/src/main/cpp/core/config/configuration_loader.cpp +++ b/src/main/cpp/core/config/configuration_loader.cpp @@ -115,6 +115,55 @@ void load_security_section(const sol::table& root, ReqPackConfig& config) { osvBackend.overlayPath = config.security.osvOverlayPath; } } + configuration_internal::sync_osv_backend_feed_url(config); + if (!config.security.backends.contains("snyk")) { + SecurityBackendConfig snykBackend; + snykBackend.apiBaseUrl = "https://api.snyk.io/rest"; + snykBackend.apiVersion = "2024-10-15"; + snykBackend.tokenEnv = "SNYK_TOKEN"; + snykBackend.dataset = "issues"; + config.security.backends["snyk"] = std::move(snykBackend); + } else { + SecurityBackendConfig& snykBackend = config.security.backends["snyk"]; + if (snykBackend.apiBaseUrl.empty()) { + snykBackend.apiBaseUrl = "https://api.snyk.io/rest"; + } + if (snykBackend.apiVersion.empty()) { + snykBackend.apiVersion = "2024-10-15"; + } + if (snykBackend.tokenEnv.empty()) { + snykBackend.tokenEnv = "SNYK_TOKEN"; + } + if (snykBackend.dataset.empty()) { + snykBackend.dataset = "issues"; + } + } + if (!config.security.backends.contains("trivy")) { + SecurityBackendConfig trivyBackend; + trivyBackend.dbRepositories = { + "mirror.gcr.io/aquasec/trivy-db:2", + "ghcr.io/aquasecurity/trivy-db:2", + }; + config.security.backends["trivy"] = std::move(trivyBackend); + } else { + SecurityBackendConfig& trivyBackend = config.security.backends["trivy"]; + if (trivyBackend.dbRepositories.empty()) { + trivyBackend.dbRepositories = { + "mirror.gcr.io/aquasec/trivy-db:2", + "ghcr.io/aquasecurity/trivy-db:2", + }; + } + } + if (!config.security.backends.contains("gh-advisory")) { + SecurityBackendConfig ghAdvisoryBackend; + ghAdvisoryBackend.feedUrl = "https://codeload.github.com/github/advisory-database/tar.gz/refs/heads/main"; + config.security.backends["gh-advisory"] = std::move(ghAdvisoryBackend); + } else { + SecurityBackendConfig& ghAdvisoryBackend = config.security.backends["gh-advisory"]; + if (ghAdvisoryBackend.feedUrl.empty()) { + ghAdvisoryBackend.feedUrl = "https://codeload.github.com/github/advisory-database/tar.gz/refs/heads/main"; + } + } } void load_reports_section(const sol::table& root, ReqPackConfig& config) { diff --git a/src/main/cpp/core/config/configuration_lua_values.cpp b/src/main/cpp/core/config/configuration_lua_values.cpp index f9596ee..f1de2e4 100644 --- a/src/main/cpp/core/config/configuration_lua_values.cpp +++ b/src/main/cpp/core/config/configuration_lua_values.cpp @@ -31,7 +31,7 @@ std::optional repository_checksum_policy_from_string(c return RepositoryChecksumPolicy::WARN; } if (normalized == "ignore") { - return RepositoryChecksumPolicy::IGNORE; + return RepositoryChecksumPolicy::SKIP; } return std::nullopt; @@ -492,6 +492,17 @@ std::map load_security_backend_map(const sol assign_if_present(backendTable, "refreshMode", osv_refresh_mode_from_string, backend.refreshMode); assign_if_present(backendTable, "refreshIntervalSeconds", backend.refreshIntervalSeconds); assign_if_present(backendTable, "overlayPath", backend.overlayPath); + assign_if_present(backendTable, "apiBaseUrl", backend.apiBaseUrl); + assign_if_present(backendTable, "apiVersion", backend.apiVersion); + assign_if_present(backendTable, "tokenEnv", backend.tokenEnv); + assign_if_present(backendTable, "orgId", backend.orgId); + assign_if_present(backendTable, "groupId", backend.groupId); + assign_if_present(backendTable, "dataset", backend.dataset); + assign_if_present(backendTable, "helperPath", backend.helperPath); + const std::vector dbRepositories = load_string_array(backendTable["dbRepositories"]); + if (!dbRepositories.empty()) { + backend.dbRepositories = dbRepositories; + } if (!backend.feedUrl.empty()) { backend.feedUrl = expand_user_path(backend.feedUrl).string(); @@ -499,6 +510,12 @@ std::map load_security_backend_map(const sol if (!backend.overlayPath.empty()) { backend.overlayPath = expand_user_path(backend.overlayPath).string(); } + if (!backend.apiBaseUrl.empty()) { + backend.apiBaseUrl = expand_user_path(backend.apiBaseUrl).string(); + } + if (!backend.helperPath.empty()) { + backend.helperPath = expand_user_path(backend.helperPath).string(); + } values[to_lower_copy(key.as())] = std::move(backend); } diff --git a/src/main/cpp/core/config/configuration_paths.cpp b/src/main/cpp/core/config/configuration_paths.cpp index 13ca91b..0b63b59 100644 --- a/src/main/cpp/core/config/configuration_paths.cpp +++ b/src/main/cpp/core/config/configuration_paths.cpp @@ -1,14 +1,21 @@ #include "configuration_internal.h" #include +#include +#include + +#if !defined(_WIN32) #include #include #include +#endif namespace { constexpr const char* ARCHIVE_PASSWORD_ENV = "REQPACK_ARCHIVE_PASSWORD"; +#if !defined(_WIN32) + std::optional passwd_home_from_user(const std::string& username) { if (username.empty()) { return std::nullopt; @@ -33,6 +40,8 @@ std::optional passwd_home_from_uid(uid_t uid) { return std::nullopt; } +#endif + std::filesystem::path xdg_directory(const char* envName, const std::filesystem::path& fallback) { const char* value = std::getenv(envName); if (value != nullptr && std::string(value).size() > 0) { @@ -46,6 +55,19 @@ std::filesystem::path xdg_directory(const char* envName, const std::filesystem:: namespace configuration_internal { std::filesystem::path invoking_user_home_directory() { +#if defined(_WIN32) + const char* userProfile = std::getenv("USERPROFILE"); + if (userProfile != nullptr && std::string(userProfile).size() > 0) { + return std::filesystem::path(userProfile); + } + + const char* home = std::getenv("HOME"); + if (home != nullptr && std::string(home).size() > 0) { + return std::filesystem::path(home); + } + + return std::filesystem::current_path(); +#else const char* sudoUser = std::getenv("SUDO_USER"); if (sudoUser != nullptr && std::string(sudoUser).size() > 0) { if (const auto home = passwd_home_from_user(sudoUser)) { @@ -73,6 +95,7 @@ std::filesystem::path invoking_user_home_directory() { } return std::filesystem::current_path(); +#endif } std::filesystem::path expand_user_path(const std::filesystem::path& path) { @@ -151,6 +174,36 @@ ReqPackConfig::ReqPackConfig() }) { version = reqpack_build_release_id(); downloader.userAgent = reqpack_user_agent(); + + if (!security.backends.contains("osv")) { + SecurityBackendConfig osvBackend; + osvBackend.feedUrl = security.osvFeedUrl; + osvBackend.refreshMode = security.osvRefreshMode; + osvBackend.refreshIntervalSeconds = security.osvRefreshIntervalSeconds; + osvBackend.overlayPath = security.osvOverlayPath; + security.backends["osv"] = std::move(osvBackend); + } + if (!security.backends.contains("snyk")) { + SecurityBackendConfig snykBackend; + snykBackend.apiBaseUrl = "https://api.snyk.io/rest"; + snykBackend.apiVersion = "2024-10-15"; + snykBackend.tokenEnv = "SNYK_TOKEN"; + snykBackend.dataset = "issues"; + security.backends["snyk"] = std::move(snykBackend); + } + if (!security.backends.contains("trivy")) { + SecurityBackendConfig trivyBackend; + trivyBackend.dbRepositories = { + "mirror.gcr.io/aquasec/trivy-db:2", + "ghcr.io/aquasecurity/trivy-db:2", + }; + security.backends["trivy"] = std::move(trivyBackend); + } + if (!security.backends.contains("gh-advisory")) { + SecurityBackendConfig ghAdvisoryBackend; + ghAdvisoryBackend.feedUrl = "https://codeload.github.com/github/advisory-database/tar.gz/refs/heads/main"; + security.backends["gh-advisory"] = std::move(ghAdvisoryBackend); + } } ReqPackConfig default_reqpack_config() { diff --git a/src/main/cpp/core/execution/executor_dispatch.cpp b/src/main/cpp/core/execution/executor_dispatch.cpp index 3d248c7..f4481e3 100644 --- a/src/main/cpp/core/execution/executor_dispatch.cpp +++ b/src/main/cpp/core/execution/executor_dispatch.cpp @@ -3,6 +3,7 @@ #include "executor_internal.h" #include "core/host/host_info.h" +#include "core/planning/planner_platform_policy.h" #include "output/logger.h" #include @@ -40,6 +41,44 @@ std::vector Executer::executeTaskGroup(const TaskGr return {}; } + if (planner_platform::softSkipNixInstalls() && + is_install_like_action(taskGroup.action) && + planner_platform::isNixInstallSystem(this->registry->resolvePluginName(taskGroup.system))) { + bool consumersReady = !taskGroup.nixSoftSkipConsumers.empty(); + for (const std::string& consumerSystem : taskGroup.nixSoftSkipConsumers) { + if (this->registry->getPlugin(consumerSystem) == nullptr || !this->registry->loadPlugin(consumerSystem)) { + consumersReady = false; + break; + } + } + const bool stillRequired = taskGroup.nixSoftSkipConsumers.empty() || !consumersReady; + + if (stillRequired) { + std::string packageList; + for (const Package& package : taskGroup.packages) { + if (!packageList.empty()) { + packageList += ", "; + } + packageList += package.name; + if (!package.version.empty()) { + packageList += "@" + package.version; + } + } + Logger::instance().diagnostic(make_warning_diagnostic( + "executor", + "Skipping nix install on Windows", + "ReqPack will not install nix packages on Windows and expects required tools to already be available on the host.", + "Install the required tools with a Windows package manager (for example Chocolatey or winget) or manually, then retry.", + packageList.empty() ? std::string{} : ("packages: " + packageList), + taskGroup.system, + "nix-soft-skip" + )); + } + + // Soft-skip: no plugin call and no success history records. + return {}; + } + if (this->securityGateway.isGatewaySystem(taskGroup.system)) { const bool ok = this->dispatchTaskGroupToSecurityGateway(taskGroup); std::vector records = ok ? this->buildSuccessRecords(taskGroup) : this->buildFailureRecords(taskGroup); @@ -175,6 +214,61 @@ std::vector Executer::buildSuccessRecords(const Tas return records; } +std::vector Executer::buildAlreadySatisfiedRecords( + const std::vector& allTaskGroups, + const std::vector& executableTaskGroups) const +{ + auto packageKey = [](const std::string& system, const std::string& name) { + return system + '\0' + name; + }; + + std::set executableKeys; + for (const TaskGroup& taskGroup : executableTaskGroups) { + for (const Package& package : taskGroup.packages) { + executableKeys.insert(packageKey(taskGroup.system, package.name)); + } + } + + std::vector installedState; + if (this->historyManager != nullptr) { + installedState = this->historyManager->loadInstalledState(); + } + + std::vector records; + for (const TaskGroup& taskGroup : allTaskGroups) { + if (!actionUsesMissingPackageFilter(taskGroup.action) || taskGroup.usesLocalTarget) { + continue; + } + + for (const Package& package : taskGroup.packages) { + if (executableKeys.find(packageKey(taskGroup.system, package.name)) != executableKeys.end()) { + continue; + } + + std::string version = package.version; + if (version.empty()) { + for (const InstalledEntry& entry : installedState) { + if (entry.system == taskGroup.system && entry.name == package.name) { + version = entry.version; + break; + } + } + } + + records.push_back(TransactionRecord{ + .runId = {}, + .system = taskGroup.system, + .action = taskGroup.action, + .packageName = package.name, + .packageVersion = version, + .status = "success" + }); + } + } + + return records; +} + std::vector Executer::buildFailureRecords(const TaskGroup& taskGroup) const { std::vector records; records.reserve(taskGroup.packages.size()); diff --git a/src/main/cpp/core/execution/executor_run.cpp b/src/main/cpp/core/execution/executor_run.cpp index 3531238..d0aaf92 100644 --- a/src/main/cpp/core/execution/executor_run.cpp +++ b/src/main/cpp/core/execution/executor_run.cpp @@ -3,6 +3,7 @@ #include "executor_internal.h" #include "output/logger.h" +#include "output/run_result_json.h" #include #include @@ -61,8 +62,9 @@ bool Executer::execute(Graph *graph) { } } + const bool jsonOutput = this->config.display.jsonOutput; bool sessionBegun = false; - if (!taskGroups.empty()) { + if (!taskGroups.empty() && !jsonOutput) { std::vector itemIds; for (const TaskGroup& tg : taskGroups) { if (tg.usesLocalTarget) { @@ -87,8 +89,33 @@ bool Executer::execute(Graph *graph) { ); std::vector allRecords = records; allRecords.insert(allRecords.end(), orphanRemovalRecords.begin(), orphanRemovalRecords.end()); + if (jsonOutput && !inputAlreadyFiltered) { + const std::vector alreadySatisfiedRecords = + this->buildAlreadySatisfiedRecords(allTaskGroups, taskGroups); + allRecords.insert(allRecords.end(), alreadySatisfiedRecords.begin(), alreadySatisfiedRecords.end()); + } + + const bool ok = std::none_of(allRecords.begin(), allRecords.end(), [](const TransactionRecord& record) { + return record.status != "success"; + }); - if (sessionBegun) { + if (jsonOutput) { + RunResultJsonDocument document; + document.ok = ok; + document.dryRun = this->config.execution.dryRun; + document.items.reserve(allRecords.size()); + for (const TransactionRecord& record : allRecords) { + const bool success = record.status == "success"; + RunResultJsonItem item; + item.system = record.system; + item.name = record.packageName; + item.version = record.packageVersion; + item.status = run_result_status_for_action(record.action, success, this->config.execution.dryRun); + item.errorMessage = record.errorMessage; + document.items.push_back(std::move(item)); + } + emit_run_result_json(document); + } else if (sessionBegun) { int succeeded = 0; int failed = 0; for (const TransactionRecord& r : allRecords) { @@ -117,7 +144,5 @@ bool Executer::execute(Graph *graph) { } } - return std::none_of(allRecords.begin(), allRecords.end(), [](const TransactionRecord& record) { - return record.status != "success"; - }); + return ok; } diff --git a/src/main/cpp/core/execution/executor_task_graph.cpp b/src/main/cpp/core/execution/executor_task_graph.cpp index 23d2642..f919016 100644 --- a/src/main/cpp/core/execution/executor_task_graph.cpp +++ b/src/main/cpp/core/execution/executor_task_graph.cpp @@ -2,6 +2,8 @@ #include "executor_internal.h" +#include "core/planning/planner_platform_policy.h" + #include #include @@ -75,6 +77,12 @@ std::vector Executer::createTaskGroups(const Graph& graph) } } + if constexpr (planner_platform::reorderNonNixBeforeNix) { + std::stable_partition(groups.begin(), groups.end(), [](const TaskGroup& group) { + return !planner_platform::isNixInstallSystem(group.system); + }); + } + return groups; } @@ -113,7 +121,23 @@ std::vector Executer::createTaskGroupPlans(const std::v } } + auto isInstallLike = [](const ActionType action) { + return action == ActionType::INSTALL || action == ActionType::ENSURE; + }; + std::set> seenEdges; + auto addScheduleEdge = [&](const std::size_t sourceGroup, const std::size_t targetGroup) { + if (sourceGroup == targetGroup) { + return; + } + const std::pair edge{sourceGroup, targetGroup}; + if (!seenEdges.insert(edge).second) { + return; + } + plans[sourceGroup].successors.push_back(targetGroup); + ++plans[targetGroup].pendingDependencies; + }; + auto [edgeBegin, edgeEnd] = boost::edges(*graph); for (auto edgeIt = edgeBegin; edgeIt != edgeEnd; ++edgeIt) { const Package& sourcePackage = (*graph)[boost::source(*edgeIt, *graph)]; @@ -123,15 +147,45 @@ std::vector Executer::createTaskGroupPlans(const std::v if (sourceGroupIt == packageToGroupIndex.end() || targetGroupIt == packageToGroupIndex.end()) { continue; } - if (sourceGroupIt->second == targetGroupIt->second) { - continue; + + const std::string resolvedSource = this->registry->resolvePluginName(sourcePackage.system); + const std::string resolvedTarget = this->registry->resolvePluginName(targetPackage.system); + + if (planner_platform::isNixInstallSystem(resolvedSource) && !planner_platform::isNixInstallSystem(resolvedTarget)) { + auto& consumers = plans[sourceGroupIt->second].taskGroup.nixSoftSkipConsumers; + if (std::find(consumers.begin(), consumers.end(), resolvedTarget) == consumers.end()) { + consumers.push_back(resolvedTarget); + } } - const std::pair edge{sourceGroupIt->second, targetGroupIt->second}; - if (!seenEdges.insert(edge).second) { + + if (planner_platform::shouldIgnoreScheduleEdge(resolvedSource, resolvedTarget)) { continue; } - plans[sourceGroupIt->second].successors.push_back(targetGroupIt->second); - ++plans[targetGroupIt->second].pendingDependencies; + + addScheduleEdge(sourceGroupIt->second, targetGroupIt->second); + } + + if (planner_platform::needsNonNixBeforeNixBarrier()) { + std::vector nonNixInstallGroups; + std::vector nixInstallGroups; + for (std::size_t groupIndex = 0; groupIndex < plans.size(); ++groupIndex) { + const TaskGroup& group = plans[groupIndex].taskGroup; + if (!isInstallLike(group.action)) { + continue; + } + const std::string resolvedSystem = this->registry->resolvePluginName(group.system); + if (planner_platform::isNixInstallSystem(resolvedSystem)) { + nixInstallGroups.push_back(groupIndex); + } else { + nonNixInstallGroups.push_back(groupIndex); + } + } + + for (const std::size_t nixGroup : nixInstallGroups) { + for (const std::size_t nonNixGroup : nonNixInstallGroups) { + addScheduleEdge(nonNixGroup, nixGroup); + } + } } return plans; diff --git a/src/main/cpp/core/execution/orchestrator.cpp b/src/main/cpp/core/execution/orchestrator.cpp index d106f75..62c2bb8 100644 --- a/src/main/cpp/core/execution/orchestrator.cpp +++ b/src/main/cpp/core/execution/orchestrator.cpp @@ -71,6 +71,7 @@ int Orchestrator::run() { if (this->requests.empty()) { return 0; } + orchestrator_internal::rewrite_security_gateway_package_requests(this->requests, this->registry, this->config); orchestrator_internal::rewrite_registry_package_requests(this->requests, this->registry->getDatabase()); if (this->shouldRunSystemWidePackageUpdates()) { return this->runSystemWidePackageUpdates(); diff --git a/src/main/cpp/core/execution/orchestrator_internal.h b/src/main/cpp/core/execution/orchestrator_internal.h index 700989a..9787290 100644 --- a/src/main/cpp/core/execution/orchestrator_internal.h +++ b/src/main/cpp/core/execution/orchestrator_internal.h @@ -25,6 +25,7 @@ bool prepare_requests_for_run( ); void rewrite_registry_package_requests(std::vector& requests, const RegistryDatabase* database); +void rewrite_security_gateway_package_requests(std::vector& requests, Registry* registry, const ReqPackConfig& config); bool requests_target_plugin_install(const std::vector& requests); bool requests_target_plugin_remove(const std::vector& requests); diff --git a/src/main/cpp/core/execution/orchestrator_plugin_flows.cpp b/src/main/cpp/core/execution/orchestrator_plugin_flows.cpp index 300bb8e..2683c58 100644 --- a/src/main/cpp/core/execution/orchestrator_plugin_flows.cpp +++ b/src/main/cpp/core/execution/orchestrator_plugin_flows.cpp @@ -6,6 +6,7 @@ #include "core/plugins/plugin_bundle.h" #include "output/diagnostic.h" #include "output/logger.h" +#include "output/run_result_json.h" #include #include @@ -235,30 +236,45 @@ int Orchestrator::runSystemWidePackageUpdates() { int Orchestrator::runPluginInstallRequests() { Logger& logger = Logger::instance(); + const bool jsonOutput = this->config.display.jsonOutput; std::vector itemIds; itemIds.reserve(this->requests.size()); for (const Request& request : this->requests) { itemIds.push_back(request.system); } - logger.displaySessionBegin(DisplayMode::INSTALL, itemIds); + if (!jsonOutput) { + logger.displaySessionBegin(DisplayMode::INSTALL, itemIds); + } int succeeded = 0; int failed = 0; + RunResultJsonDocument jsonDocument; + jsonDocument.dryRun = this->config.execution.dryRun; Downloader downloader(this->registry->getDatabase(), this->config); for (const Request& request : this->requests) { - logger.displayItemBegin(request.system, request.system); - logger.displayItemStep(request.system, "install plugin wrapper"); + if (!jsonOutput) { + logger.displayItemBegin(request.system, request.system); + logger.displayItemStep(request.system, "install plugin wrapper"); + } const std::string resolvedSystem = this->registry->resolvePluginName(request.system); + RunResultJsonItem jsonItem; + jsonItem.system = "rqp"; + jsonItem.name = request.system; if (resolvedSystem.empty()) { - logger.displayItemFailure(request.system, make_error_diagnostic( - "install", - "Plugin install failed", - "ReqPack could not resolve requested system to a plugin wrapper.", - "Check plugin name and registry sources, then retry.", - {}, - request.system, - "plugin-install" - )); + if (!jsonOutput) { + logger.displayItemFailure(request.system, make_error_diagnostic( + "install", + "Plugin install failed", + "ReqPack could not resolve requested system to a plugin wrapper.", + "Check plugin name and registry sources, then retry.", + {}, + request.system, + "plugin-install" + )); + } + jsonItem.status = run_result_status_for_action(ActionType::INSTALL, false, this->config.execution.dryRun); + jsonItem.errorMessage = "plugin resolve failed"; + jsonDocument.items.push_back(std::move(jsonItem)); ++failed; continue; } @@ -269,68 +285,103 @@ int Orchestrator::runPluginInstallRequests() { installed = plugin_bundle_exists(this->config, resolvedSystem); } if (installed) { - logger.displayItemSuccess(request.system); + if (!jsonOutput) { + logger.displayItemSuccess(request.system); + } + jsonItem.status = run_result_status_for_action(ActionType::INSTALL, true, this->config.execution.dryRun); + jsonDocument.items.push_back(std::move(jsonItem)); ++succeeded; continue; } - logger.displayItemFailure(request.system, make_error_diagnostic( - "install", - "Plugin install failed", - "ReqPack could not install requested plugin wrapper from configured local registry sources.", - "Check registry source path, plugin bundle contents, and auto-download settings, then retry.", - {}, - request.system, - "plugin-install" - )); + if (!jsonOutput) { + logger.displayItemFailure(request.system, make_error_diagnostic( + "install", + "Plugin install failed", + "ReqPack could not install requested plugin wrapper from configured local registry sources.", + "Check registry source path, plugin bundle contents, and auto-download settings, then retry.", + {}, + request.system, + "plugin-install" + )); + } + jsonItem.status = run_result_status_for_action(ActionType::INSTALL, false, this->config.execution.dryRun); + jsonItem.errorMessage = "plugin install failed"; + jsonDocument.items.push_back(std::move(jsonItem)); ++failed; } - logger.displaySessionEnd(failed == 0, succeeded, 0, failed); + jsonDocument.ok = failed == 0; + if (jsonOutput) { + emit_run_result_json(jsonDocument); + } else { + logger.displaySessionEnd(failed == 0, succeeded, 0, failed); + } return failed == 0 ? 0 : 1; } int Orchestrator::runPluginRemoveRequests() { Logger& logger = Logger::instance(); + const bool jsonOutput = this->config.display.jsonOutput; std::vector itemIds; itemIds.reserve(this->requests.size()); for (const Request& request : this->requests) { itemIds.push_back(request.system); } - logger.displaySessionBegin(DisplayMode::REMOVE, itemIds); + if (!jsonOutput) { + logger.displaySessionBegin(DisplayMode::REMOVE, itemIds); + } int succeeded = 0; int failed = 0; + RunResultJsonDocument jsonDocument; + jsonDocument.dryRun = this->config.execution.dryRun; for (const Request& request : this->requests) { - logger.displayItemBegin(request.system, request.system); - logger.displayItemStep(request.system, "remove plugin wrapper"); + if (!jsonOutput) { + logger.displayItemBegin(request.system, request.system); + logger.displayItemStep(request.system, "remove plugin wrapper"); + } + + RunResultJsonItem jsonItem; + jsonItem.system = "rqp"; + jsonItem.name = request.system; const std::string resolvedSystem = this->registry->resolvePluginName(request.system); if (resolvedSystem.empty()) { - logger.displayItemFailure(request.system, make_error_diagnostic( - "remove", - "Plugin remove failed", - "ReqPack could not resolve requested system to an installed plugin wrapper.", - "Check plugin name and local plugin state, then retry.", - {}, - request.system, - "plugin-remove" - )); + if (!jsonOutput) { + logger.displayItemFailure(request.system, make_error_diagnostic( + "remove", + "Plugin remove failed", + "ReqPack could not resolve requested system to an installed plugin wrapper.", + "Check plugin name and local plugin state, then retry.", + {}, + request.system, + "plugin-remove" + )); + } + jsonItem.status = run_result_status_for_action(ActionType::REMOVE, false, this->config.execution.dryRun); + jsonItem.errorMessage = "plugin resolve failed"; + jsonDocument.items.push_back(std::move(jsonItem)); ++failed; continue; } const std::optional layout = plugin_bundle_find_installed(this->config, resolvedSystem); if (!layout.has_value()) { - logger.displayItemFailure(request.system, make_error_diagnostic( - "remove", - "Plugin remove failed", - "Requested plugin wrapper is not installed in current plugin directory.", - "Install plugin first or verify configured plugin directory before retrying remove.", - {}, - request.system, - "plugin-remove" - )); + if (!jsonOutput) { + logger.displayItemFailure(request.system, make_error_diagnostic( + "remove", + "Plugin remove failed", + "Requested plugin wrapper is not installed in current plugin directory.", + "Install plugin first or verify configured plugin directory before retrying remove.", + {}, + request.system, + "plugin-remove" + )); + } + jsonItem.status = run_result_status_for_action(ActionType::REMOVE, false, this->config.execution.dryRun); + jsonItem.errorMessage = "plugin not installed"; + jsonDocument.items.push_back(std::move(jsonItem)); ++failed; continue; } @@ -370,15 +421,20 @@ int Orchestrator::runPluginRemoveRequests() { } if (!dependencyRemovalOk) { - logger.displayItemFailure(request.system, make_error_diagnostic( - "remove", - "Plugin remove failed", - "ReqPack could not remove one or more plugin dependency packages before deleting wrapper files.", - "Inspect dependency plugin output and retry once dependent packages can be removed cleanly.", - {}, - request.system, - "plugin-remove" - )); + if (!jsonOutput) { + logger.displayItemFailure(request.system, make_error_diagnostic( + "remove", + "Plugin remove failed", + "ReqPack could not remove one or more plugin dependency packages before deleting wrapper files.", + "Inspect dependency plugin output and retry once dependent packages can be removed cleanly.", + {}, + request.system, + "plugin-remove" + )); + } + jsonItem.status = run_result_status_for_action(ActionType::REMOVE, false, this->config.execution.dryRun); + jsonItem.errorMessage = "plugin dependency remove failed"; + jsonDocument.items.push_back(std::move(jsonItem)); ++failed; continue; } @@ -387,15 +443,20 @@ int Orchestrator::runPluginRemoveRequests() { std::error_code error; std::filesystem::remove_all(layout->rootDir, error); if (error) { - logger.displayItemFailure(request.system, make_error_diagnostic( - "remove", - "Plugin remove failed", - "ReqPack could not delete installed plugin wrapper files from disk.", - "Check filesystem permissions and retry remove after closing any process using that plugin directory.", - error.message(), - request.system, - "plugin-remove" - )); + if (!jsonOutput) { + logger.displayItemFailure(request.system, make_error_diagnostic( + "remove", + "Plugin remove failed", + "ReqPack could not delete installed plugin wrapper files from disk.", + "Check filesystem permissions and retry remove after closing any process using that plugin directory.", + error.message(), + request.system, + "plugin-remove" + )); + } + jsonItem.status = run_result_status_for_action(ActionType::REMOVE, false, this->config.execution.dryRun); + jsonItem.errorMessage = error.message(); + jsonDocument.items.push_back(std::move(jsonItem)); ++failed; continue; } @@ -403,10 +464,19 @@ int Orchestrator::runPluginRemoveRequests() { this->registry->scanDirectory(this->config.registry.pluginDirectory); } - logger.displayItemSuccess(request.system); + if (!jsonOutput) { + logger.displayItemSuccess(request.system); + } + jsonItem.status = run_result_status_for_action(ActionType::REMOVE, true, this->config.execution.dryRun); + jsonDocument.items.push_back(std::move(jsonItem)); ++succeeded; } - logger.displaySessionEnd(failed == 0, succeeded, 0, failed); + jsonDocument.ok = failed == 0; + if (jsonOutput) { + emit_run_result_json(jsonDocument); + } else { + logger.displaySessionEnd(failed == 0, succeeded, 0, failed); + } return failed == 0 ? 0 : 1; } diff --git a/src/main/cpp/core/execution/orchestrator_request_prep.cpp b/src/main/cpp/core/execution/orchestrator_request_prep.cpp index 5bf2d42..865cd07 100644 --- a/src/main/cpp/core/execution/orchestrator_request_prep.cpp +++ b/src/main/cpp/core/execution/orchestrator_request_prep.cpp @@ -4,6 +4,7 @@ #include "core/download/downloader.h" #include "core/planning/request_resolution.h" #include "core/registry/registry_database_core.h" +#include "core/security/security_gateway_service.h" #include "output/diagnostic.h" #include "output/logger.h" @@ -129,6 +130,37 @@ bool resolve_local_target( namespace orchestrator_internal { +void rewrite_security_gateway_package_requests(std::vector& requests, Registry* registry, const ReqPackConfig& config) { + if (registry == nullptr || requests.size() < 2) { + return; + } + + SecurityGatewayService gateway(registry, registry, config); + const Request& first = requests.front(); + if (first.system.empty() || !gateway.isGatewaySystem(first.system) || + (first.action != ActionType::INSTALL && first.action != ActionType::UPDATE && first.action != ActionType::ENSURE) || + first.usesLocalTarget || !first.packages.empty()) { + return; + } + + std::vector gatewayPackages; + gatewayPackages.reserve(requests.size() - 1); + for (std::size_t index = 1; index < requests.size(); ++index) { + const Request& request = requests[index]; + if (request.action != first.action || request.system.empty() || request.usesLocalTarget || !request.packages.empty()) { + return; + } + if (!registry->getPluginSecurityMetadata(request.system).has_value()) { + return; + } + gatewayPackages.push_back(request.system); + } + + Request rewritten = first; + rewritten.packages = std::move(gatewayPackages); + requests = {std::move(rewritten)}; +} + void rewrite_registry_package_requests(std::vector& requests, const RegistryDatabase* database) { if (database == nullptr) { return; @@ -200,7 +232,7 @@ bool prepare_requests_for_run( std::filesystem::create_directories(tempDir, ec); const std::filesystem::path tempFile = tempDir / filename; - Logger::instance().stdout("downloading " + request.localPath, request.system, "install"); + Logger::instance().logStdout("downloading " + request.localPath, request.system, "install"); Downloader downloader(registry->getDatabase(), config); if (!downloader.download(request.localPath, tempFile.string())) { diff --git a/src/main/cpp/core/export/audit_exporter_table.cpp b/src/main/cpp/core/export/audit_exporter_table.cpp index ba1ec90..228cc5e 100644 --- a/src/main/cpp/core/export/audit_exporter_table.cpp +++ b/src/main/cpp/core/export/audit_exporter_table.cpp @@ -1,5 +1,7 @@ #include "audit_exporter_internal.h" +#include "core/common/terminal_width.h" +#include "core/common/tty_helpers.h" #include "output/ansi_color.h" #include @@ -12,8 +14,6 @@ #include #include #include -#include -#include #include namespace { @@ -65,12 +65,7 @@ std::size_t terminal_width() { } } - winsize size{}; - if (isatty(STDOUT_FILENO) && ioctl(STDOUT_FILENO, TIOCGWINSZ, &size) == 0 && size.ws_col > 0) { - return static_cast(size.ws_col); - } - - return 120; + return reqpack_terminal_column_count(120); } std::string fit_table_cell(const std::string& value, const std::size_t width) { @@ -286,7 +281,7 @@ bool table_colors_enabled() { if (force_color_enabled()) { return true; } - return isatty(STDOUT_FILENO); + return reqpack_stdout_is_tty(); } } // namespace audit_exporter_internal diff --git a/src/main/cpp/core/export/audit_exporter_write.cpp b/src/main/cpp/core/export/audit_exporter_write.cpp index b77c033..d30f8d3 100644 --- a/src/main/cpp/core/export/audit_exporter_write.cpp +++ b/src/main/cpp/core/export/audit_exporter_write.cpp @@ -41,11 +41,11 @@ bool write_output_file(const std::string& rendered, const Request& request, cons if (!interactive) { return false; } - logger.stdout(resolvedOutputPath + " already exists. Overwrite? [y/N]"); + logger.logStdout(resolvedOutputPath + " already exists. Overwrite? [y/N]"); logger.flushSync(); std::string answer; if (!std::getline(std::cin, answer) || (answer != "y" && answer != "Y")) { - logger.stdout("aborted."); + logger.logStdout("aborted."); logger.flushSync(); return false; } diff --git a/src/main/cpp/core/export/sbom_exporter_table.cpp b/src/main/cpp/core/export/sbom_exporter_table.cpp index 1ba1b66..00fa458 100644 --- a/src/main/cpp/core/export/sbom_exporter_table.cpp +++ b/src/main/cpp/core/export/sbom_exporter_table.cpp @@ -1,5 +1,7 @@ #include "sbom_exporter_internal.h" +#include "core/common/terminal_width.h" +#include "core/common/tty_helpers.h" #include "output/ansi_color.h" #include @@ -9,8 +11,6 @@ #include #include #include -#include -#include #include namespace { @@ -59,12 +59,7 @@ std::size_t terminal_width() { } } - winsize size{}; - if (isatty(STDOUT_FILENO) && ioctl(STDOUT_FILENO, TIOCGWINSZ, &size) == 0 && size.ws_col > 0) { - return static_cast(size.ws_col); - } - - return 120; + return reqpack_terminal_column_count(120); } std::string fit_table_cell(const std::string& value, const std::size_t width) { @@ -262,7 +257,7 @@ bool table_colors_enabled() { if (force_color_enabled()) { return true; } - return isatty(STDOUT_FILENO); + return reqpack_stdout_is_tty(); } } // namespace sbom_exporter_internal diff --git a/src/main/cpp/core/export/sbom_exporter_write.cpp b/src/main/cpp/core/export/sbom_exporter_write.cpp index eb9c382..bce8e00 100644 --- a/src/main/cpp/core/export/sbom_exporter_write.cpp +++ b/src/main/cpp/core/export/sbom_exporter_write.cpp @@ -46,11 +46,11 @@ bool write_output_file( if (!interactive) { return false; } - logger.stdout(resolvedOutputPath + " already exists. Overwrite? [y/N]"); + logger.logStdout(resolvedOutputPath + " already exists. Overwrite? [y/N]"); logger.flushSync(); std::string answer; if (!std::getline(std::cin, answer) || (answer != "y" && answer != "Y")) { - logger.stdout("aborted."); + logger.logStdout("aborted."); logger.flushSync(); return false; } diff --git a/src/main/cpp/core/export/snapshot_exporter_write.cpp b/src/main/cpp/core/export/snapshot_exporter_write.cpp index c06e50f..e6a9ebd 100644 --- a/src/main/cpp/core/export/snapshot_exporter_write.cpp +++ b/src/main/cpp/core/export/snapshot_exporter_write.cpp @@ -55,11 +55,11 @@ bool write_snapshot_output( const bool force = std::find(request.flags.begin(), request.flags.end(), "force") != request.flags.end(); if (!force) { Logger& logger = Logger::instance(); - logger.stdout(resolvedPath + " already exists. Overwrite? [y/N]"); + logger.logStdout(resolvedPath + " already exists. Overwrite? [y/N]"); logger.flushSync(); std::string answer; if (!std::getline(std::cin, answer) || (answer != "y" && answer != "Y")) { - logger.stdout("aborted."); + logger.logStdout("aborted."); logger.flushSync(); return false; } diff --git a/src/main/cpp/core/history/history_manager_log.cpp b/src/main/cpp/core/history/history_manager_log.cpp index ae170bc..93044b5 100644 --- a/src/main/cpp/core/history/history_manager_log.cpp +++ b/src/main/cpp/core/history/history_manager_log.cpp @@ -1,5 +1,7 @@ #include "history_manager_internal.h" +#include "core/common/time_helpers.h" + #include #include #include @@ -65,7 +67,7 @@ std::string utc_timestamp_now() { const auto now = std::chrono::system_clock::now(); const std::time_t t = std::chrono::system_clock::to_time_t(now); std::tm tm{}; - gmtime_r(&t, &tm); + (void)reqpack_gmtime_utc(&tm, &t); std::ostringstream oss; oss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ"); return oss.str(); diff --git a/src/main/cpp/core/host/host_info_probe.cpp b/src/main/cpp/core/host/host_info_probe.cpp index aecd9ef..3eb31c1 100644 --- a/src/main/cpp/core/host/host_info_probe.cpp +++ b/src/main/cpp/core/host/host_info_probe.cpp @@ -11,8 +11,10 @@ #include #include +#if !defined(_WIN32) #include #include +#endif #if defined(__linux__) #include @@ -112,6 +114,17 @@ HostPlatformInfo detect_platform_info() { platform.osFamily = "unknown"; #endif +#if defined(_WIN32) +#if defined(_M_ARM64) || defined(__aarch64__) + platform.arch = normalize_host_architecture("aarch64"); +#elif defined(_M_X64) || defined(__amd64__) || defined(__x86_64__) + platform.arch = normalize_host_architecture("x86_64"); +#elif defined(_M_IX86) || defined(__i386__) + platform.arch = normalize_host_architecture("x86"); +#else + platform.arch = "unknown"; +#endif +#else struct utsname uts{}; if (::uname(&uts) == 0) { platform.arch = normalize_host_architecture(uts.machine); @@ -119,6 +132,7 @@ HostPlatformInfo detect_platform_info() { if (platform.arch.empty()) { platform.arch = normalize_host_architecture(exec_read_first_line("uname -m 2>/dev/null").value_or(std::string{})); } +#endif if (platform.arch.empty()) { platform.arch = "unknown"; } @@ -141,6 +155,11 @@ HostPlatformInfo detect_platform_info() { } void fill_uname_fields(HostKernelInfo& kernel, HostCpuInfo& cpu) { +#if defined(_WIN32) + (void)kernel; + (void)cpu; + return; +#else struct utsname uts{}; if (::uname(&uts) != 0) { return; @@ -152,6 +171,7 @@ void fill_uname_fields(HostKernelInfo& kernel, HostCpuInfo& cpu) { if (cpu.arch.empty()) { cpu.arch = normalize_host_architecture(uts.machine); } +#endif } void fill_linux_os_info(HostInfoSnapshot& snapshot) { @@ -376,6 +396,11 @@ bool should_skip_mount_type(const std::string& fsType) { } void fill_mount_capacity(const std::string& mountPoint, HostMountInfo& mount) { +#if defined(_WIN32) + (void)mountPoint; + (void)mount; + return; +#else struct statvfs info{}; if (::statvfs(mountPoint.c_str(), &info) != 0) { return; @@ -390,6 +415,7 @@ void fill_mount_capacity(const std::string& mountPoint, HostMountInfo& mount) { mount.usedBytes = total - freeBytes; } mount.readOnly = (info.f_flag & ST_RDONLY) != 0; +#endif } #if defined(__linux__) diff --git a/src/main/cpp/core/packages/rq_package_reader.cpp b/src/main/cpp/core/packages/rq_package_reader.cpp index 800b90f..9f5c041 100644 --- a/src/main/cpp/core/packages/rq_package_reader.cpp +++ b/src/main/cpp/core/packages/rq_package_reader.cpp @@ -1,5 +1,7 @@ #include "rq_package_internal.h" +#include "core/common/temp_directory.h" + #include #include #include @@ -7,21 +9,10 @@ #include #include -#include - namespace { std::filesystem::path make_unique_directory(const std::filesystem::path& parent, const std::string& prefix) { - std::filesystem::create_directories(parent); - std::filesystem::path pattern = parent / (prefix + "-XXXXXX"); - std::string value = pattern.string(); - std::vector buffer(value.begin(), value.end()); - buffer.push_back('\0'); - char* created = ::mkdtemp(buffer.data()); - if (created == nullptr) { - throw std::runtime_error("failed to create temporary directory"); - } - return std::filesystem::path(created); + return reqpack_make_unique_directory(parent, prefix); } } // namespace diff --git a/src/main/cpp/core/plugins/plugin_bundle.cpp b/src/main/cpp/core/plugins/plugin_bundle.cpp index f82cf63..f3a0466 100644 --- a/src/main/cpp/core/plugins/plugin_bundle.cpp +++ b/src/main/cpp/core/plugins/plugin_bundle.cpp @@ -239,8 +239,14 @@ std::optional plugin_bundle_read_directory(const std::filesy std::optional plugin_bundle_find_root(const std::filesystem::path& basePath, const std::string& expectedPluginId) { std::vector candidates; if (!expectedPluginId.empty()) { + const std::string normalized = to_lower_copy(expectedPluginId); + const std::string group = normalized.substr(0, 1); candidates.push_back(basePath / "plugins" / expectedPluginId); + candidates.push_back(basePath / "plugins" / group / expectedPluginId); candidates.push_back(basePath / expectedPluginId); + candidates.push_back(basePath / group / expectedPluginId); + candidates.push_back(basePath / "rqp-plugins" / expectedPluginId); + candidates.push_back(basePath / "rqp-plugins" / group / expectedPluginId); } candidates.push_back(basePath); diff --git a/src/main/cpp/core/plugins/plugin_test_runner_case.cpp b/src/main/cpp/core/plugins/plugin_test_runner_case.cpp index 3672b92..116a0a2 100644 --- a/src/main/cpp/core/plugins/plugin_test_runner_case.cpp +++ b/src/main/cpp/core/plugins/plugin_test_runner_case.cpp @@ -8,8 +8,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -83,6 +85,11 @@ struct FakeExecResponse { ExecResult result; }; +struct PluginTestFixtureFile { + std::string path; + std::string content; +}; + struct PluginTestCase { std::string name; std::string sourcePath; @@ -92,6 +99,10 @@ struct PluginTestCase { std::vector packages{}; std::string localPath; std::string prompt; + std::filesystem::path fixtureRoot; + std::vector fixtureDirs{}; + std::vector fixtureFiles{}; + std::map environment{}; std::vector fakeExec{}; std::optional expectSuccess; std::vector expectStdout{}; @@ -116,6 +127,238 @@ class PluginTestError : public std::runtime_error { : std::runtime_error(message) {} }; +std::filesystem::path default_fixture_root_for_case(const std::filesystem::path& casePath) { + const std::string stem = casePath.stem().string(); + std::string sanitized; + sanitized.reserve(stem.size()); + for (unsigned char ch : stem) { + if (std::isalnum(ch) || ch == '-' || ch == '_') { + sanitized.push_back(static_cast(ch)); + } else { + sanitized.push_back('-'); + } + } + if (sanitized.empty()) { + sanitized = "case"; + } + + const std::size_t pathHash = std::hash{}(casePath.string()); + std::ostringstream suffix; + suffix << std::hex << pathHash; + return (std::filesystem::temp_directory_path() / ("reqpack-plugin-test-" + sanitized + "-" + suffix.str())).lexically_normal(); +} + +std::map string_map_from_table(const sol::object& object) { + if (!object.valid() || object.is() || object.get_type() != sol::type::table) { + return {}; + } + + std::map values; + for (const auto& [key, entry] : object.as()) { + if (key.get_type() != sol::type::string || entry.get_type() != sol::type::string) { + continue; + } + values.emplace(key.as(), entry.as()); + } + return values; +} + +std::vector fixture_files_from_table(const sol::object& object, const std::filesystem::path& path) { + if (!object.valid() || object.is() || object.get_type() != sol::type::table) { + return {}; + } + + std::vector files; + for (const auto& [_, entry] : object.as()) { + if (entry.get_type() != sol::type::table) { + continue; + } + const sol::table file = entry.as(); + const sol::optional fixturePath = file["path"]; + if (!fixturePath.has_value() || fixturePath->empty()) { + throw PluginTestError("fixtureFiles.path is required: " + path.string()); + } + const sol::optional content = file["content"]; + files.push_back(PluginTestFixtureFile{ + .path = fixturePath.value(), + .content = content.has_value() ? content.value() : std::string{}, + }); + } + return files; +} + +std::filesystem::path resolve_fixture_path(const PluginTestCase& testCase, const std::string& rawPath) { + const std::filesystem::path path(rawPath); + if (path.is_absolute()) { + return path.lexically_normal(); + } + return (testCase.fixtureRoot / path).lexically_normal(); +} + +std::string replace_all(std::string value, const std::string& needle, const std::string& replacement) { + if (needle.empty()) { + return value; + } + std::size_t position = 0; + while ((position = value.find(needle, position)) != std::string::npos) { + value.replace(position, needle.size(), replacement); + position += replacement.size(); + } + return value; +} + +std::map resolved_case_environment(const PluginTestCase& testCase) { + std::map values = testCase.environment; + const std::string fixtureRoot = testCase.fixtureRoot.string(); + const std::string caseDirectory = std::filesystem::path(testCase.sourcePath).parent_path().string(); + for (auto& [_, value] : values) { + value = replace_all(value, "${fixtureRoot}", fixtureRoot); + value = replace_all(value, "${caseDir}", caseDirectory); + } + if (!fixtureRoot.empty() && values.find("REQPACK_TEST_FIXTURE_ROOT") == values.end()) { + values.emplace("REQPACK_TEST_FIXTURE_ROOT", fixtureRoot); + } + return values; +} + +void apply_case_placeholders(PluginTestCase& testCase) { + const std::string fixtureRoot = testCase.fixtureRoot.string(); + const std::string caseDirectory = std::filesystem::path(testCase.sourcePath).parent_path().string(); + const auto expand = [&](const std::string& value) { + return replace_all(replace_all(value, "${fixtureRoot}", fixtureRoot), "${caseDir}", caseDirectory); + }; + + testCase.localPath = expand(testCase.localPath); + testCase.prompt = expand(testCase.prompt); + for (FakeExecResponse& response : testCase.fakeExec) { + response.match = expand(response.match); + response.result.stdoutText = expand(response.result.stdoutText); + response.result.stderrText = expand(response.result.stderrText); + } + for (std::string& value : testCase.expectCommands) { + value = expand(value); + } + for (std::string& flag : testCase.flags) { + flag = expand(flag); + } + for (Package& package : testCase.packages) { + package.name = expand(package.name); + package.version = expand(package.version); + package.sourcePath = expand(package.sourcePath); + for (std::string& flag : package.flags) { + flag = expand(flag); + } + } +} + +void write_fixture_file(const std::filesystem::path& path, const std::string& content) { + std::error_code error; + std::filesystem::create_directories(path.parent_path(), error); + if (error) { + throw PluginTestError("failed to create fixture directory: " + path.parent_path().string()); + } + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output.is_open()) { + throw PluginTestError("failed to write fixture file: " + path.string()); + } + output << content; +} + +void prepare_case_fixture_files(const PluginTestCase& testCase) { + if (testCase.fixtureRoot.empty()) { + return; + } + + const std::string fixtureRoot = testCase.fixtureRoot.string(); + const std::string caseDirectory = std::filesystem::path(testCase.sourcePath).parent_path().string(); + + std::error_code error; + std::filesystem::remove_all(testCase.fixtureRoot, error); + error.clear(); + std::filesystem::create_directories(testCase.fixtureRoot, error); + if (error) { + throw PluginTestError("failed to create fixture root: " + testCase.fixtureRoot.string()); + } + + for (const std::string& directory : testCase.fixtureDirs) { + const std::filesystem::path resolved = resolve_fixture_path(testCase, directory); + std::filesystem::create_directories(resolved, error); + if (error) { + throw PluginTestError("failed to create fixture directory: " + resolved.string()); + } + } + + for (const PluginTestFixtureFile& file : testCase.fixtureFiles) { + write_fixture_file( + resolve_fixture_path(testCase, file.path), + replace_all(replace_all(file.content, "${fixtureRoot}", fixtureRoot), "${caseDir}", caseDirectory) + ); + } +} + +class PreparedCaseFixtures { +public: + explicit PreparedCaseFixtures(const std::vector& cases) { + roots_.reserve(cases.size()); + for (const PluginTestCase& testCase : cases) { + prepare_case_fixture_files(testCase); + if (!testCase.fixtureRoot.empty()) { + roots_.push_back(testCase.fixtureRoot); + } + } + } + + ~PreparedCaseFixtures() { + for (const std::filesystem::path& root : roots_) { + std::error_code error; + std::filesystem::remove_all(root, error); + } + } + +private: + std::vector roots_{}; +}; + +std::optional environment_value(const std::string& name) { + const char* value = std::getenv(name.c_str()); + if (value == nullptr) { + return std::nullopt; + } + return std::string{value}; +} + +void set_environment_value(const std::string& name, const std::optional& value) { +#ifdef _WIN32 + _putenv_s(name.c_str(), value.has_value() ? value->c_str() : ""); +#else + if (value.has_value()) { + setenv(name.c_str(), value->c_str(), 1); + } else { + unsetenv(name.c_str()); + } +#endif +} + +class ScopedEnvironmentOverride { +public: + explicit ScopedEnvironmentOverride(const std::map& values) { + previous_.reserve(values.size()); + for (const auto& [name, value] : values) { + previous_.push_back({name, environment_value(name)}); + set_environment_value(name, value); + } + } + + ~ScopedEnvironmentOverride() { + for (auto it = previous_.rbegin(); it != previous_.rend(); ++it) { + set_environment_value(it->first, it->second); + } + } + +private: + std::vector>> previous_{}; +}; + class FakeExecHost : public LuaBridge { public: FakeExecHost(const std::string& scriptPath, const ReqPackConfig& config) @@ -338,6 +581,7 @@ PluginTestCase load_case_file(const std::filesystem::path& path) { PluginTestCase testCase; testCase.sourcePath = path.string(); testCase.name = optional_string_field(root, "name").value_or(path.stem().string()); + testCase.fixtureRoot = default_fixture_root_for_case(path); const sol::object requestObject = root["request"]; if (!requestObject.valid() || requestObject.get_type() != sol::type::table) { @@ -390,6 +634,16 @@ PluginTestCase load_case_file(const std::filesystem::path& path) { } } + if (const sol::optional fixtureRoot = root["fixtureRoot"]; fixtureRoot.has_value() && !fixtureRoot->empty()) { + const std::filesystem::path customRoot(fixtureRoot.value()); + testCase.fixtureRoot = customRoot.is_absolute() + ? customRoot.lexically_normal() + : (path.parent_path() / customRoot).lexically_normal(); + } + testCase.fixtureDirs = string_list_from_table(root["fixtureDirs"]); + testCase.fixtureFiles = fixture_files_from_table(root["fixtureFiles"], path); + testCase.environment = string_map_from_table(root["environment"]); + const sol::object expectObject = root["expect"]; if (expectObject.valid() && expectObject.get_type() == sol::type::table) { const sol::table expect = expectObject.as(); @@ -417,6 +671,8 @@ PluginTestCase load_case_file(const std::filesystem::path& path) { } } + apply_case_placeholders(testCase); + return testCase; } @@ -443,6 +699,7 @@ std::string package_display_name(const PackageInfo& info) { PluginTestRuntimeCaseResult run_single_case(FakeExecHost& plugin, const ReqPackConfig& config, const PluginTestCase& testCase) { plugin.setCaseResponses(testCase.fakeExec); + ScopedEnvironmentOverride environmentOverride(resolved_case_environment(testCase)); PluginTestRuntimeCaseResult result; result.summary.name = testCase.name; @@ -493,8 +750,8 @@ PluginTestRuntimeCaseResult run_single_case(FakeExecHost& plugin, const ReqPackC result.events = plugin.takeRecentEvents(); result.summary.commands = plugin.commands(); - result.summary.stdout = plugin.stdout_lines(); - result.summary.stderr = plugin.stderr_lines(); + result.summary.stdoutLines = plugin.stdout_lines(); + result.summary.stderrLines = plugin.stderr_lines(); result.summary.artifacts = plugin.artifacts(); for (const PluginEventRecord& event : result.events) { result.summary.events.push_back(event.name); @@ -514,13 +771,13 @@ PluginTestRuntimeCaseResult run_single_case(FakeExecHost& plugin, const ReqPackC } if (!testCase.expectStdout.empty()) { - if (result.summary.stdout != testCase.expectStdout) { + if (result.summary.stdoutLines != testCase.expectStdout) { failures.push_back("stdout did not match expectation"); } } if (!testCase.expectStderr.empty()) { - if (result.summary.stderr != testCase.expectStderr) { + if (result.summary.stderrLines != testCase.expectStderr) { failures.push_back("stderr did not match expectation"); } } @@ -621,6 +878,8 @@ PluginTestRunReport run_plugin_test_cases_impl(const ReqPackConfig& config, cons loadedCases.push_back(std::move(testCase)); } + PreparedCaseFixtures preparedFixtures(loadedCases); + FakeExecHost plugin(pluginScript.string(), config); plugin.setCaseResponses(std::move(bootstrapResponses)); if (!plugin.init()) { diff --git a/src/main/cpp/core/plugins/plugin_test_runner_report.cpp b/src/main/cpp/core/plugins/plugin_test_runner_report.cpp index c4a75f3..921348a 100644 --- a/src/main/cpp/core/plugins/plugin_test_runner_report.cpp +++ b/src/main/cpp/core/plugins/plugin_test_runner_report.cpp @@ -73,19 +73,19 @@ std::string plugin_test_report_to_json(const PluginTestRunReport& report) { } stream << "],\n"; stream << " \"stdout\": ["; - for (std::size_t stdoutIndex = 0; stdoutIndex < entry.stdout.size(); ++stdoutIndex) { + for (std::size_t stdoutIndex = 0; stdoutIndex < entry.stdoutLines.size(); ++stdoutIndex) { if (stdoutIndex != 0) { stream << ", "; } - stream << '"' << json_escape(entry.stdout[stdoutIndex]) << '"'; + stream << '"' << json_escape(entry.stdoutLines[stdoutIndex]) << '"'; } stream << "],\n"; stream << " \"stderr\": ["; - for (std::size_t stderrIndex = 0; stderrIndex < entry.stderr.size(); ++stderrIndex) { + for (std::size_t stderrIndex = 0; stderrIndex < entry.stderrLines.size(); ++stderrIndex) { if (stderrIndex != 0) { stream << ", "; } - stream << '"' << json_escape(entry.stderr[stderrIndex]) << '"'; + stream << '"' << json_escape(entry.stderrLines[stderrIndex]) << '"'; } stream << "],\n"; stream << " \"events\": ["; diff --git a/src/main/cpp/core/registry/registry_database_git.cpp b/src/main/cpp/core/registry/registry_database_git.cpp index 621aaa3..1ac563b 100644 --- a/src/main/cpp/core/registry/registry_database_git.cpp +++ b/src/main/cpp/core/registry/registry_database_git.cpp @@ -1,17 +1,11 @@ #include "registry_database_internal.h" -#include "core/common/network_environment.h" +#include "core/common/process_runner.h" #include "core/common/version_compare.h" -#include #include -#include -#include -#include #include #include -#include -#include namespace { @@ -34,140 +28,12 @@ std::string trim_copy(const std::string& value) { } ProcessResult run_process_capture(const std::vector& arguments) { - ProcessResult result; - if (arguments.empty()) { - result.stderrText = "empty command"; - return result; - } - - int stdoutPipe[2]; - if (::pipe(stdoutPipe) != 0) { - result.stderrText = std::string{"pipe(stdout) failed: "} + std::strerror(errno); - return result; - } - - int stderrPipe[2]; - if (::pipe(stderrPipe) != 0) { - result.stderrText = std::string{"pipe(stderr) failed: "} + std::strerror(errno); - (void)::close(stdoutPipe[0]); - (void)::close(stdoutPipe[1]); - return result; - } - - posix_spawn_file_actions_t fileActions; - if (posix_spawn_file_actions_init(&fileActions) != 0) { - result.stderrText = "posix_spawn_file_actions_init failed"; - (void)::close(stdoutPipe[0]); - (void)::close(stdoutPipe[1]); - (void)::close(stderrPipe[0]); - (void)::close(stderrPipe[1]); - return result; - } - - const bool actionsReady = - posix_spawn_file_actions_addopen(&fileActions, STDIN_FILENO, "/dev/null", O_RDONLY, 0) == 0 && - posix_spawn_file_actions_adddup2(&fileActions, stdoutPipe[1], STDOUT_FILENO) == 0 && - posix_spawn_file_actions_adddup2(&fileActions, stderrPipe[1], STDERR_FILENO) == 0 && - posix_spawn_file_actions_addclose(&fileActions, stdoutPipe[0]) == 0 && - posix_spawn_file_actions_addclose(&fileActions, stdoutPipe[1]) == 0 && - posix_spawn_file_actions_addclose(&fileActions, stderrPipe[0]) == 0 && - posix_spawn_file_actions_addclose(&fileActions, stderrPipe[1]) == 0; - if (!actionsReady) { - posix_spawn_file_actions_destroy(&fileActions); - result.stderrText = "failed to configure process pipes"; - (void)::close(stdoutPipe[0]); - (void)::close(stdoutPipe[1]); - (void)::close(stderrPipe[0]); - (void)::close(stderrPipe[1]); - return result; - } - - std::vector argv; - argv.reserve(arguments.size() + 1); - for (const std::string& argument : arguments) { - argv.push_back(const_cast(argument.c_str())); - } - argv.push_back(nullptr); - - std::vector environmentStorage = reqpack_sanitized_process_environment(); - std::vector environmentPointers; - environmentPointers.reserve(environmentStorage.size() + 1); - for (std::string& entry : environmentStorage) { - environmentPointers.push_back(entry.data()); - } - environmentPointers.push_back(nullptr); - - pid_t pid = 0; - const int spawnResult = posix_spawnp(&pid, arguments.front().c_str(), &fileActions, nullptr, argv.data(), environmentPointers.data()); - posix_spawn_file_actions_destroy(&fileActions); - (void)::close(stdoutPipe[1]); - (void)::close(stderrPipe[1]); - if (spawnResult != 0) { - result.stderrText = std::string{"spawn failed: "} + std::strerror(spawnResult); - (void)::close(stdoutPipe[0]); - (void)::close(stderrPipe[0]); - return result; - } - - char buffer[4096]; - while (true) { - const ssize_t bytesRead = ::read(stdoutPipe[0], buffer, sizeof(buffer)); - if (bytesRead > 0) { - result.stdoutText.append(buffer, static_cast(bytesRead)); - continue; - } - if (bytesRead == 0) { - break; - } - if (errno == EINTR) { - continue; - } - result.stderrText += std::string{"read(stdout) failed: "} + std::strerror(errno); - break; - } - (void)::close(stdoutPipe[0]); - - while (true) { - const ssize_t bytesRead = ::read(stderrPipe[0], buffer, sizeof(buffer)); - if (bytesRead > 0) { - result.stderrText.append(buffer, static_cast(bytesRead)); - continue; - } - if (bytesRead == 0) { - break; - } - if (errno == EINTR) { - continue; - } - result.stderrText += std::string{"read(stderr) failed: "} + std::strerror(errno); - break; - } - (void)::close(stderrPipe[0]); - - int status = 0; - while (waitpid(pid, &status, 0) == -1) { - if (errno != EINTR) { - result.stderrText += std::string{"waitpid failed: "} + std::strerror(errno); - return result; - } - } - - if (WIFEXITED(status)) { - result.exitCode = WEXITSTATUS(status); - return result; - } - - if (WIFSIGNALED(status)) { - result.exitCode = 128 + WTERMSIG(status); - if (!result.stderrText.empty() && result.stderrText.back() != '\n') { - result.stderrText.push_back('\n'); - } - result.stderrText += "terminated by signal " + std::to_string(WTERMSIG(status)); - return result; - } - - result.stderrText += "process ended unexpectedly"; - return result; + const ReqpackProcessResult captured = reqpack_run_process_capture(arguments); + return ProcessResult{ + .exitCode = captured.exitCode, + .stdoutText = captured.stdoutText, + .stderrText = captured.stderrText, + }; } bool run_process_quiet(const std::vector& arguments) { diff --git a/src/main/cpp/core/remote/remote_client.cpp b/src/main/cpp/core/remote/remote_client.cpp index 2fcd896..26fbf3e 100644 --- a/src/main/cpp/core/remote/remote_client.cpp +++ b/src/main/cpp/core/remote/remote_client.cpp @@ -2,14 +2,10 @@ #include "cli/cli.h" +#include "core/common/socket_platform.h" #include "output/command_output.h" #include "output/logger.h" -#include -#include -#include -#include - #include #include #include @@ -176,10 +172,10 @@ std::optional detect_upload_install_request( return request; } -bool send_all(int fd, const std::string& data) { +bool send_all(ReqpackSocket fd, const std::string& data) { std::size_t offset = 0; while (offset < data.size()) { - const ssize_t written = ::send(fd, data.data() + offset, data.size() - offset, 0); + const auto written = ::send(fd, data.data() + offset, static_cast(data.size() - offset), 0); if (written <= 0) { return false; } @@ -188,11 +184,11 @@ bool send_all(int fd, const std::string& data) { return true; } -std::optional read_line(int fd) { +std::optional read_line(ReqpackSocket fd) { std::string line; char c = '\0'; for (;;) { - const ssize_t received = ::recv(fd, &c, 1, 0); + const auto received = ::recv(fd, &c, 1, 0); if (received == 0) { if (line.empty()) { return std::nullopt; @@ -212,11 +208,11 @@ std::optional read_line(int fd) { return line; } -std::string read_bytes(int fd, std::size_t count) { +std::string read_bytes(ReqpackSocket fd, std::size_t count) { std::string out(count, '\0'); std::size_t offset = 0; while (offset < count) { - const ssize_t received = ::recv(fd, out.data() + offset, count - offset, 0); + const auto received = ::recv(fd, out.data() + offset, static_cast(count - offset), 0); if (received <= 0) { throw std::runtime_error("failed to read remote response body"); } @@ -225,7 +221,7 @@ std::string read_bytes(int fd, std::size_t count) { return out; } -std::pair read_text_response(int fd) { +std::pair read_text_response(ReqpackSocket fd) { const std::optional header = read_line(fd); if (!header.has_value()) { throw std::runtime_error("remote server closed connection"); @@ -294,7 +290,9 @@ std::optional extract_json_string_field(const std::string& json, co return std::nullopt; } -int connect_remote(const RemoteProfile& profile) { +ReqpackSocket connect_remote(const RemoteProfile& profile) { + reqpack_ensure_socket_runtime(); + addrinfo hints{}; hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; @@ -305,27 +303,27 @@ int connect_remote(const RemoteProfile& profile) { throw std::runtime_error("invalid remote host: " + profile.host); } - int fd = -1; + ReqpackSocket fd = REQPACK_INVALID_SOCKET; for (addrinfo* address = addresses; address != nullptr; address = address->ai_next) { fd = ::socket(address->ai_family, address->ai_socktype, address->ai_protocol); - if (fd == -1) { + if (fd == REQPACK_INVALID_SOCKET) { continue; } - if (::connect(fd, address->ai_addr, address->ai_addrlen) == 0) { + if (::connect(fd, address->ai_addr, static_cast(address->ai_addrlen)) == 0) { break; } - ::close(fd); - fd = -1; + reqpack_close_socket(fd); + fd = REQPACK_INVALID_SOCKET; } ::freeaddrinfo(addresses); - if (fd == -1) { + if (fd == REQPACK_INVALID_SOCKET) { throw std::runtime_error("failed to connect to remote profile '" + profile.name + "'"); } return fd; } -int send_text_command_and_render_response(int fd, const std::string& command, IDisplay* display) { +int send_text_command_and_render_response(ReqpackSocket fd, const std::string& command, IDisplay* display) { (void)display; if (!send_all(fd, command + "\n")) { throw std::runtime_error("failed to send remote command"); @@ -335,7 +333,7 @@ int send_text_command_and_render_response(int fd, const std::string& command, ID return response.first == "OK" ? 0 : 1; } -int send_upload_install_request(int fd, const UploadInstallRequest& request, IDisplay* display) { +int send_upload_install_request(ReqpackSocket fd, const UploadInstallRequest& request, IDisplay* display) { (void)display; const std::string header = join_arguments({ REMOTE_UPLOAD_INSTALL_COMMAND, @@ -382,8 +380,8 @@ int run_text_client_session(const RemoteProfile& profile, const std::vector -#include -#include -#include - #include #include #include @@ -39,8 +34,8 @@ int run_remote_serve( return 1; } - const int serverFd = create_server_socket(options, logger); - if (serverFd == -1) { + const ReqpackSocket serverFd = create_server_socket(options, logger); + if (serverFd == REQPACK_INVALID_SOCKET) { logger.flushSync(); return 1; } @@ -76,8 +71,8 @@ int run_remote_serve( while (!state.shutdownRequested.load() && !signalHandlers.shutdownRequested()) { sockaddr_storage clientAddress{}; socklen_t clientLength = sizeof(clientAddress); - const int clientFd = ::accept(serverFd, reinterpret_cast(&clientAddress), &clientLength); - if (clientFd == -1) { + const ReqpackSocket clientFd = ::accept(serverFd, reinterpret_cast(&clientAddress), &clientLength); + if (clientFd == REQPACK_INVALID_SOCKET) { if (state.shutdownRequested.load() || signalHandlers.shutdownRequested()) { break; } @@ -91,7 +86,7 @@ int run_remote_serve( ? json_response(false, output) : text_response(false, output); (void)send_all(clientFd, response); - ::close(clientFd); + reqpack_close_socket(clientFd); continue; } @@ -136,7 +131,7 @@ int run_remote_serve( (void)send_all(clientFd, text_response(false, command_output_message(DisplayMode::SERVE, "unsupported negotiated protocol", false))); } } - ::close(clientFd); + reqpack_close_socket(clientFd); std::lock_guard lock(state.mutex); state.sessions.erase(sessionId); }).detach(); @@ -145,7 +140,7 @@ int run_remote_serve( if (signalHandlers.shutdownRequested()) { state.shutdownRequested.store(true); std::lock_guard lock(state.mutex); - state.serverFd = -1; + state.serverFd = REQPACK_INVALID_SOCKET; } logger.flushSync(); diff --git a/src/main/cpp/core/remote/serve_remote_command.cpp b/src/main/cpp/core/remote/serve_remote_command.cpp index cbac986..ebf2406 100644 --- a/src/main/cpp/core/remote/serve_remote_command.cpp +++ b/src/main/cpp/core/remote/serve_remote_command.cpp @@ -290,7 +290,7 @@ RemoteResponse execute_command( } RemoteResponse execute_upload_install_command( - int clientFd, + ReqpackSocket clientFd, Cli& cli, RemoteServerState& state, Logger& logger, diff --git a/src/main/cpp/core/remote/serve_remote_internal.h b/src/main/cpp/core/remote/serve_remote_internal.h index 809d443..eca58d7 100644 --- a/src/main/cpp/core/remote/serve_remote_internal.h +++ b/src/main/cpp/core/remote/serve_remote_internal.h @@ -2,6 +2,7 @@ #include "core/remote/serve_remote.h" #include "core/remote/remote_profiles.h" +#include "core/common/socket_platform.h" #include #include @@ -73,7 +74,7 @@ struct RemoteServerState { std::filesystem::path configPath; ReqPackConfigOverrides configOverrides; std::filesystem::path remoteUsersPath; - int serverFd{-1}; + ReqpackSocket serverFd{REQPACK_INVALID_SOCKET}; int nextSessionId{1}; std::atomic shutdownRequested{false}; }; @@ -151,26 +152,31 @@ inline std::vector merged_command_arguments( class ScopedRemoteSignalHandlers { public: - explicit ScopedRemoteSignalHandlers(int serverFd); + explicit ScopedRemoteSignalHandlers(ReqpackSocket serverFd); ~ScopedRemoteSignalHandlers(); bool shutdownRequested() const; private: +#if defined(_WIN32) + void (*oldTerm_)(int){SIG_DFL}; + void (*oldInt_)(int){SIG_DFL}; +#else struct sigaction oldTerm_ {}; struct sigaction oldInt_ {}; +#endif bool installed_{false}; }; -bool send_all(int fd, const std::string& data); -bool read_exact_bytes(int fd, char* buffer, std::size_t count); -bool discard_bytes(int fd, std::uintmax_t count); -std::optional read_line_from_socket(int fd); +bool send_all(ReqpackSocket fd, const std::string& data); +bool read_exact_bytes(ReqpackSocket fd, char* buffer, std::size_t count); +bool discard_bytes(ReqpackSocket fd, std::uintmax_t count); +std::optional read_line_from_socket(ReqpackSocket fd); std::optional detect_connection_protocol( const ServeRuntimeOptions& options, const std::string& firstLine ); -int create_server_socket(const ServeRuntimeOptions& options, Logger& logger); +ReqpackSocket create_server_socket(const ServeRuntimeOptions& options, Logger& logger); RemoteStateSnapshot snapshot_remote_state(RemoteServerState& state); std::string connection_protocol_name(ConnectionProtocol protocol); @@ -222,7 +228,7 @@ class ScopedPathCleanup { std::filesystem::path path_; }; -ScopedPathCleanup write_uploaded_file_to_temp(int clientFd, const UploadInstallEnvelope& envelope); +ScopedPathCleanup write_uploaded_file_to_temp(ReqpackSocket clientFd, const UploadInstallEnvelope& envelope); std::string substitute_upload_path(const std::string& commandTemplate, const std::filesystem::path& path); bool reload_remote_state(RemoteServerState& state, Logger& logger, std::string& error); @@ -238,7 +244,7 @@ RemoteResponse execute_command( std::mutex& commandMutex ); RemoteResponse execute_upload_install_command( - int clientFd, + ReqpackSocket clientFd, Cli& cli, RemoteServerState& state, Logger& logger, @@ -252,7 +258,7 @@ std::optional parse_json_command(const std::string& line); std::string json_response(bool ok, const CommandOutput& output); std::string text_response(bool ok, const CommandOutput& output); void handle_text_client( - int clientFd, + ReqpackSocket clientFd, Cli& cli, RemoteServerState& state, Logger& logger, @@ -262,7 +268,7 @@ void handle_text_client( std::optional pendingLine = std::nullopt ); void handle_json_client( - int clientFd, + ReqpackSocket clientFd, Cli& cli, RemoteServerState& state, Logger& logger, diff --git a/src/main/cpp/core/remote/serve_remote_protocol.cpp b/src/main/cpp/core/remote/serve_remote_protocol.cpp index 2dc5ab2..d8b0e6c 100644 --- a/src/main/cpp/core/remote/serve_remote_protocol.cpp +++ b/src/main/cpp/core/remote/serve_remote_protocol.cpp @@ -90,7 +90,7 @@ std::string text_response(bool ok, const CommandOutput& output) { } void handle_text_client( - int clientFd, + ReqpackSocket clientFd, Cli& cli, RemoteServerState& state, Logger& logger, @@ -160,7 +160,7 @@ void handle_text_client( } void handle_json_client( - int clientFd, + ReqpackSocket clientFd, Cli& cli, RemoteServerState& state, Logger& logger, diff --git a/src/main/cpp/core/remote/serve_remote_session.cpp b/src/main/cpp/core/remote/serve_remote_session.cpp index 9c4fdc4..94e76d8 100644 --- a/src/main/cpp/core/remote/serve_remote_session.cpp +++ b/src/main/cpp/core/remote/serve_remote_session.cpp @@ -1,7 +1,6 @@ #include "serve_remote_internal.h" -#include -#include +#include "core/common/time_helpers.h" #include #include @@ -14,7 +13,7 @@ std::string connection_protocol_name(ConnectionProtocol protocol) { std::string format_timestamp(const std::chrono::system_clock::time_point& timePoint) { const std::time_t time = std::chrono::system_clock::to_time_t(timePoint); std::tm tm{}; - localtime_r(&time, &tm); + (void)reqpack_localtime(&tm, &time); std::ostringstream output; output << std::put_time(&tm, "%Y-%m-%d %H:%M:%S"); return output.str(); @@ -88,15 +87,15 @@ int active_connection_count(RemoteServerState& state) { } void request_server_shutdown(RemoteServerState& state) { - int serverFd = -1; + ReqpackSocket serverFd = REQPACK_INVALID_SOCKET; { std::lock_guard lock(state.mutex); serverFd = state.serverFd; - state.serverFd = -1; + state.serverFd = REQPACK_INVALID_SOCKET; } - if (serverFd != -1) { - ::shutdown(serverFd, SHUT_RDWR); - ::close(serverFd); + if (serverFd != REQPACK_INVALID_SOCKET) { + (void)reqpack_shutdown_socket(serverFd); + reqpack_close_socket(serverFd); } } diff --git a/src/main/cpp/core/remote/serve_remote_socket.cpp b/src/main/cpp/core/remote/serve_remote_socket.cpp index 27f0c36..476e2c9 100644 --- a/src/main/cpp/core/remote/serve_remote_socket.cpp +++ b/src/main/cpp/core/remote/serve_remote_socket.cpp @@ -1,10 +1,5 @@ #include "serve_remote_internal.h" -#include -#include -#include -#include - #include #include #include @@ -12,24 +7,35 @@ namespace { volatile std::sig_atomic_t g_remote_signal_shutdown_requested = 0; +#if !defined(_WIN32) volatile std::sig_atomic_t g_remote_signal_server_fd = -1; +#endif void handle_remote_serve_signal(int) { g_remote_signal_shutdown_requested = 1; +#if !defined(_WIN32) const int serverFd = static_cast(g_remote_signal_server_fd); if (serverFd != -1) { ::shutdown(serverFd, SHUT_RDWR); ::close(serverFd); g_remote_signal_server_fd = -1; } +#endif } } // namespace -ScopedRemoteSignalHandlers::ScopedRemoteSignalHandlers(int serverFd) { +ScopedRemoteSignalHandlers::ScopedRemoteSignalHandlers(ReqpackSocket serverFd) { g_remote_signal_shutdown_requested = 0; - g_remote_signal_server_fd = serverFd; - +#if !defined(_WIN32) + g_remote_signal_server_fd = static_cast(serverFd); +#endif + +#if defined(_WIN32) + oldTerm_ = std::signal(SIGTERM, handle_remote_serve_signal); + oldInt_ = std::signal(SIGINT, handle_remote_serve_signal); + installed_ = oldTerm_ != SIG_ERR && oldInt_ != SIG_ERR; +#else struct sigaction action {}; action.sa_handler = handle_remote_serve_signal; sigemptyset(&action.sa_mask); @@ -42,14 +48,22 @@ ScopedRemoteSignalHandlers::ScopedRemoteSignalHandlers(int serverFd) { return; } installed_ = true; +#endif } ScopedRemoteSignalHandlers::~ScopedRemoteSignalHandlers() { +#if !defined(_WIN32) g_remote_signal_server_fd = -1; +#endif g_remote_signal_shutdown_requested = 0; if (installed_) { +#if defined(_WIN32) + std::signal(SIGTERM, oldTerm_); + std::signal(SIGINT, oldInt_); +#else ::sigaction(SIGTERM, &oldTerm_, nullptr); ::sigaction(SIGINT, &oldInt_, nullptr); +#endif } } @@ -57,10 +71,10 @@ bool ScopedRemoteSignalHandlers::shutdownRequested() const { return g_remote_signal_shutdown_requested != 0; } -bool send_all(int fd, const std::string& data) { +bool send_all(ReqpackSocket fd, const std::string& data) { std::size_t offset = 0; while (offset < data.size()) { - const ssize_t written = ::send(fd, data.data() + offset, data.size() - offset, 0); + const auto written = ::send(fd, data.data() + offset, static_cast(data.size() - offset), 0); if (written <= 0) { return false; } @@ -69,10 +83,10 @@ bool send_all(int fd, const std::string& data) { return true; } -bool read_exact_bytes(int fd, char* buffer, std::size_t count) { +bool read_exact_bytes(ReqpackSocket fd, char* buffer, std::size_t count) { std::size_t offset = 0; while (offset < count) { - const ssize_t received = ::recv(fd, buffer + offset, count - offset, 0); + const auto received = ::recv(fd, buffer + offset, static_cast(count - offset), 0); if (received <= 0) { return false; } @@ -81,7 +95,7 @@ bool read_exact_bytes(int fd, char* buffer, std::size_t count) { return true; } -bool discard_bytes(int fd, std::uintmax_t count) { +bool discard_bytes(ReqpackSocket fd, std::uintmax_t count) { char buffer[8192]; std::uintmax_t remaining = count; while (remaining > 0) { @@ -94,11 +108,11 @@ bool discard_bytes(int fd, std::uintmax_t count) { return true; } -std::optional read_line_from_socket(int fd) { +std::optional read_line_from_socket(ReqpackSocket fd) { std::string line; char c = '\0'; for (;;) { - const ssize_t received = ::recv(fd, &c, 1, 0); + const auto received = ::recv(fd, &c, 1, 0); if (received == 0) { if (line.empty()) { return std::nullopt; @@ -135,7 +149,9 @@ std::optional detect_connection_protocol( return std::nullopt; } -int create_server_socket(const ServeRuntimeOptions& options, Logger& logger) { +ReqpackSocket create_server_socket(const ServeRuntimeOptions& options, Logger& logger) { + reqpack_ensure_socket_runtime(); + addrinfo hints{}; hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; @@ -145,30 +161,31 @@ int create_server_socket(const ServeRuntimeOptions& options, Logger& logger) { const std::string portString = std::to_string(options.port); if (::getaddrinfo(options.bind.c_str(), portString.c_str(), &hints, &addresses) != 0) { logger.err("failed to resolve bind address '" + options.bind + "'"); - return -1; + return REQPACK_INVALID_SOCKET; } - int serverFd = -1; + ReqpackSocket serverFd = REQPACK_INVALID_SOCKET; for (addrinfo* address = addresses; address != nullptr; address = address->ai_next) { serverFd = ::socket(address->ai_family, address->ai_socktype, address->ai_protocol); - if (serverFd == -1) { + if (serverFd == REQPACK_INVALID_SOCKET) { continue; } int reuse = 1; - (void)::setsockopt(serverFd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + (void)::setsockopt(serverFd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&reuse), sizeof(reuse)); - if (::bind(serverFd, address->ai_addr, address->ai_addrlen) == 0 && ::listen(serverFd, SOMAXCONN) == 0) { + if (::bind(serverFd, address->ai_addr, static_cast(address->ai_addrlen)) == 0 && + ::listen(serverFd, SOMAXCONN) == 0) { break; } - ::close(serverFd); - serverFd = -1; + reqpack_close_socket(serverFd); + serverFd = REQPACK_INVALID_SOCKET; } ::freeaddrinfo(addresses); - if (serverFd == -1) { + if (serverFd == REQPACK_INVALID_SOCKET) { logger.err("failed to bind remote server on " + options.bind + ":" + portString); } return serverFd; diff --git a/src/main/cpp/core/remote/serve_remote_upload.cpp b/src/main/cpp/core/remote/serve_remote_upload.cpp index 04ddd01..67bb3ed 100644 --- a/src/main/cpp/core/remote/serve_remote_upload.cpp +++ b/src/main/cpp/core/remote/serve_remote_upload.cpp @@ -76,7 +76,7 @@ void ScopedPathCleanup::reset() { path_.clear(); } -ScopedPathCleanup write_uploaded_file_to_temp(int clientFd, const UploadInstallEnvelope& envelope) { +ScopedPathCleanup write_uploaded_file_to_temp(ReqpackSocket clientFd, const UploadInstallEnvelope& envelope) { const std::filesystem::path tempRoot = std::filesystem::temp_directory_path() / "reqpack" / "remote-upload"; std::error_code dirError; std::filesystem::create_directories(tempRoot, dirError); diff --git a/src/main/cpp/core/security/gh_advisory_vulnerability_sync_service.cpp b/src/main/cpp/core/security/gh_advisory_vulnerability_sync_service.cpp new file mode 100644 index 0000000..8d675bc --- /dev/null +++ b/src/main/cpp/core/security/gh_advisory_vulnerability_sync_service.cpp @@ -0,0 +1,442 @@ +#include "core/security/gh_advisory_vulnerability_sync_service.h" + +#include "core/archive/archive_resolver.h" +#include "core/common/network_environment.h" +#include "core/security/osv_core.h" +#include "core/security/vulnerability_database.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr std::size_t GH_ADVISORY_IMPORT_BATCH_SIZE = 512; + +constexpr const char* GH_ADVISORY_SYNC_KEY_LAST_SUCCESS = "gh-advisory:last_successful_sync_at"; +constexpr const char* GH_ADVISORY_SYNC_KEY_SCOPE = "gh-advisory:scope"; +constexpr const char* GH_ADVISORY_SYNC_KEY_SOURCE_URL = "gh-advisory:source_url"; + +struct MaterializedSource { + std::filesystem::path root; + std::vector cleanupPaths; +}; + +struct ScanResults { + std::map canonicalByLower; + std::map> advisoriesByEcosystem; +}; + +std::string to_lower_copy(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return value; +} + +std::string trim_copy(const std::string& value) { + const std::size_t first = value.find_first_not_of(" \t\r\n"); + if (first == std::string::npos) { + return {}; + } + const std::size_t last = value.find_last_not_of(" \t\r\n"); + return value.substr(first, last - first + 1); +} + +bool starts_with(const std::string& value, const std::string& prefix) { + return value.rfind(prefix, 0) == 0; +} + +std::string now_epoch_seconds() { + const auto now = std::chrono::system_clock::now().time_since_epoch(); + return std::to_string(std::chrono::duration_cast(now).count()); +} + +std::string join_scope_key(const std::set& ecosystems) { + std::ostringstream stream; + bool first = true; + for (const std::string& ecosystem : ecosystems) { + if (!first) { + stream << '\n'; + } + first = false; + stream << ecosystem; + } + return stream.str(); +} + +std::filesystem::path security_index_root(const ReqPackConfig& config) { + std::filesystem::path root = config.security.indexPath; + const std::filesystem::path defaultIndexPath = default_reqpack_security_index_path(); + const std::filesystem::path defaultOsvDatabasePath = default_reqpack_osv_database_path(); + if (root == defaultIndexPath && std::filesystem::path(config.security.osvDatabasePath) != defaultOsvDatabasePath) { + root = config.security.osvDatabasePath; + } + return root; +} + +ReqPackConfig ecosystem_db_config(const ReqPackConfig& config, const std::string& ecosystem) { + ReqPackConfig dbConfig = config; + dbConfig.security.osvDatabasePath = (security_index_root(config) / ecosystem).string(); + return dbConfig; +} + +bool ensure_directory(const std::filesystem::path& path) { + if (path.empty()) { + return true; + } + std::error_code error; + std::filesystem::create_directories(path, error); + return !error; +} + +void cleanup_paths_quietly(const std::vector& cleanupPaths) { + for (const std::filesystem::path& cleanupPath : cleanupPaths) { + std::error_code error; + std::filesystem::remove_all(cleanupPath, error); + } +} + +std::size_t write_to_file(void* contents, std::size_t size, std::size_t nmemb, void* userp) { + const std::size_t bytes = size * nmemb; + auto* output = static_cast(userp); + output->write(static_cast(contents), static_cast(bytes)); + return output->good() ? bytes : 0; +} + +bool fetch_file(const ReqPackConfig& config, const std::string& url, const std::filesystem::path& path) { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output.is_open()) { + return false; + } + + CURL* curl = curl_easy_init(); + if (curl == nullptr) { + return false; + } + + reqpack_apply_curl_ca_bundle(curl); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, config.downloader.followRedirects ? 1L : 0L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, config.downloader.connectTimeoutSeconds); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, config.downloader.requestTimeoutSeconds); + curl_easy_setopt(curl, CURLOPT_USERAGENT, config.downloader.userAgent.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_to_file); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &output); + + long statusCode = 0; + const CURLcode result = curl_easy_perform(curl); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode); + curl_easy_cleanup(curl); + + output.flush(); + const bool ok = result == CURLE_OK && statusCode < 400 && output.good(); + output.close(); + if (!ok) { + std::error_code cleanupError; + std::filesystem::remove(path, cleanupError); + } + return ok; +} + +std::optional materialize_source_from_url(const ReqPackConfig& config, const std::string& url) { + if (starts_with(url, "file://")) { + const std::filesystem::path localPath(url.substr(std::string{"file://"}.size())); + std::error_code error; + if (std::filesystem::is_directory(localPath, error) && !error) { + return MaterializedSource{.root = localPath, .cleanupPaths = {}}; + } + error.clear(); + if (std::filesystem::is_regular_file(localPath, error) && !error) { + try { + const ArchiveResolution extracted = extract_archive_to_temp_directory(localPath); + return MaterializedSource{.root = extracted.installPath, .cleanupPaths = extracted.cleanupPaths}; + } catch (...) { + return std::nullopt; + } + } + return std::nullopt; + } + + const std::filesystem::path cacheRoot = std::filesystem::path(config.security.cachePath) / "gh-advisory"; + if (!ensure_directory(cacheRoot)) { + return std::nullopt; + } + + const std::filesystem::path archivePath = cacheRoot / "advisory-database.tar.gz"; + if (!fetch_file(config, url, archivePath)) { + return std::nullopt; + } + + try { + const ArchiveResolution extracted = extract_archive_to_temp_directory(archivePath); + return MaterializedSource{.root = extracted.installPath, .cleanupPaths = extracted.cleanupPaths}; + } catch (...) { + return std::nullopt; + } +} + +std::optional find_local_canonical_ecosystem(const ReqPackConfig& config, const std::string& token) { + const std::string normalized = to_lower_copy(trim_copy(token)); + if (normalized.empty()) { + return std::nullopt; + } + + const std::filesystem::path root = security_index_root(config); + std::error_code error; + if (!std::filesystem::is_directory(root, error) || error) { + return std::nullopt; + } + + for (const auto& entry : std::filesystem::directory_iterator(root, error)) { + if (error || !entry.is_directory()) { + continue; + } + const std::string name = entry.path().filename().string(); + if (to_lower_copy(name) == normalized) { + return name; + } + } + return std::nullopt; +} + +bool should_refresh_ecosystem( + const ReqPackConfig& config, + const SecurityBackendConfig& backend, + const std::optional& canonicalEcosystem, + bool forceRefresh +) { + if (!canonicalEcosystem.has_value()) { + return true; + } + + ReqPackConfig dbConfig = ecosystem_db_config(config, canonicalEcosystem.value()); + VulnerabilityDatabase database(dbConfig); + if (!database.ensureReady() || !database.hasAdvisories()) { + return true; + } + if (forceRefresh || backend.refreshMode == OsvRefreshMode::ALWAYS) { + return true; + } + if (backend.refreshMode == OsvRefreshMode::MANUAL) { + return false; + } + + const auto lastSync = database.getSyncState(GH_ADVISORY_SYNC_KEY_LAST_SUCCESS); + if (!lastSync.has_value()) { + return true; + } + try { + const long last = std::stol(lastSync.value()); + const long now = std::stol(now_epoch_seconds()); + return (now - last) >= backend.refreshIntervalSeconds; + } catch (...) { + return true; + } +} + +std::vector> collect_package_keys_for_lower_ecosystem( + const OsvAdvisory& advisory, + const std::string& lowerEcosystem +) { + std::vector> keys; + std::set seen; + for (const OsvAffectedPackage& package : advisory.affected) { + if (to_lower_copy(package.ecosystem) != lowerEcosystem || package.name.empty()) { + continue; + } + + const std::string normalizedName = osv_normalize_package_name(package.name); + const std::string dedupeKey = to_lower_copy(package.ecosystem) + "\n" + normalizedName; + if (!seen.insert(dedupeKey).second) { + continue; + } + keys.emplace_back(package.ecosystem, normalizedName); + } + return keys; +} + +ScanResults scan_source_for_ecosystems(const std::filesystem::path& root, const std::set& requestedLower) { + ScanResults results; + std::error_code error; + if (!std::filesystem::exists(root, error) || error) { + return results; + } + + for (const auto& entry : std::filesystem::recursive_directory_iterator(root, error)) { + if (error || !entry.is_regular_file() || entry.path().extension() != ".json") { + continue; + } + + for (const OsvAdvisory& advisory : osv_load_advisories_from_path(entry.path())) { + std::set matchedLowerEcosystems; + for (const OsvAffectedPackage& package : advisory.affected) { + const std::string lower = to_lower_copy(package.ecosystem); + if (!requestedLower.contains(lower)) { + continue; + } + matchedLowerEcosystems.insert(lower); + if (!results.canonicalByLower.contains(lower)) { + results.canonicalByLower[lower] = package.ecosystem; + } + } + + for (const std::string& lower : matchedLowerEcosystems) { + const auto packageKeys = collect_package_keys_for_lower_ecosystem(advisory, lower); + if (packageKeys.empty()) { + continue; + } + results.advisoriesByEcosystem[results.canonicalByLower[lower]].push_back(VulnerabilityDatabase::IndexedAdvisory{ + .advisory = advisory, + .packageKeys = packageKeys, + }); + } + } + } + + return results; +} + +} // namespace + +GhAdvisoryVulnerabilitySyncService::GhAdvisoryVulnerabilitySyncService( + std::set ecosystems, + const ReqPackConfig& config +) : config(config), ecosystems(std::move(ecosystems)) {} + +std::vector GhAdvisoryVulnerabilitySyncService::ensureReady(bool forceRefresh) const { + if (this->ecosystems.empty()) { + return {}; + } + + const auto backendIt = this->config.security.backends.find("gh-advisory"); + const SecurityBackendConfig backend = backendIt == this->config.security.backends.end() + ? SecurityBackendConfig{} + : backendIt->second; + const std::string sourceUrl = backend.feedUrl.empty() + ? "https://codeload.github.com/github/advisory-database/tar.gz/refs/heads/main" + : backend.feedUrl; + + struct RequestedState { + std::string raw; + std::string lower; + std::optional localCanonical; + bool needsRefresh{true}; + }; + + std::map requestedStates; + for (const std::string& ecosystem : this->ecosystems) { + const std::string raw = trim_copy(ecosystem); + const std::string lower = to_lower_copy(raw); + if (lower.empty()) { + continue; + } + RequestedState state; + state.raw = raw; + state.lower = lower; + state.localCanonical = find_local_canonical_ecosystem(this->config, raw); + state.needsRefresh = should_refresh_ecosystem(this->config, backend, state.localCanonical, forceRefresh); + requestedStates[lower] = std::move(state); + } + if (requestedStates.empty()) { + return {}; + } + + bool anyRefreshNeeded = false; + std::set refreshTargets; + for (const auto& [lower, state] : requestedStates) { + if (state.needsRefresh) { + anyRefreshNeeded = true; + refreshTargets.insert(lower); + } + } + if (!anyRefreshNeeded) { + return {}; + } + + const auto source = materialize_source_from_url(this->config, sourceUrl); + if (!source.has_value()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "gh-advisory", + .severity = "high", + .message = "failed to materialize GitHub advisory database source", + }}; + } + + const ScanResults scanResults = scan_source_for_ecosystems(source->root, refreshTargets); + cleanup_paths_quietly(source->cleanupPaths); + + std::vector findings; + std::set importedEcosystems; + + for (const auto& [lower, state] : requestedStates) { + if (!state.needsRefresh) { + continue; + } + if (!scanResults.canonicalByLower.contains(lower)) { + findings.push_back(ValidationFinding{ + .kind = "sync_error", + .source = "gh-advisory", + .severity = "high", + .message = "requested GitHub advisory ecosystem not found: " + state.raw, + }); + continue; + } + + const std::string& canonicalEcosystem = scanResults.canonicalByLower.at(lower); + if (!importedEcosystems.insert(canonicalEcosystem).second) { + continue; + } + + ReqPackConfig dbConfig = ecosystem_db_config(this->config, canonicalEcosystem); + VulnerabilityDatabase database(dbConfig); + if (!database.ensureReady()) { + findings.push_back(ValidationFinding{ + .kind = "sync_error", + .source = "gh-advisory", + .severity = "high", + .message = "failed to initialize GitHub advisory database for ecosystem '" + canonicalEcosystem + "'", + }); + continue; + } + if (!database.clearAll()) { + findings.push_back(ValidationFinding{ + .kind = "sync_error", + .source = "gh-advisory", + .severity = "high", + .message = "failed to reset local GitHub advisory database for ecosystem '" + canonicalEcosystem + "'", + }); + continue; + } + + const auto advisoriesIt = scanResults.advisoriesByEcosystem.find(canonicalEcosystem); + if (advisoriesIt != scanResults.advisoriesByEcosystem.end() && + !advisoriesIt->second.empty() && + !database.upsertIndexedAdvisories(advisoriesIt->second, GH_ADVISORY_IMPORT_BATCH_SIZE)) { + findings.push_back(ValidationFinding{ + .kind = "sync_error", + .source = "gh-advisory", + .severity = "high", + .message = "failed to import GitHub advisory records for ecosystem '" + canonicalEcosystem + "'", + }); + continue; + } + + (void)database.setSyncState(GH_ADVISORY_SYNC_KEY_SCOPE, join_scope_key({canonicalEcosystem})); + (void)database.setSyncState(GH_ADVISORY_SYNC_KEY_LAST_SUCCESS, now_epoch_seconds()); + (void)database.setSyncState(GH_ADVISORY_SYNC_KEY_SOURCE_URL, sourceUrl); + } + + return findings; +} diff --git a/src/main/cpp/core/security/security_gateway_service.cpp b/src/main/cpp/core/security/security_gateway_service.cpp index cd91a3e..ba6cb91 100644 --- a/src/main/cpp/core/security/security_gateway_service.cpp +++ b/src/main/cpp/core/security/security_gateway_service.cpp @@ -1,7 +1,10 @@ #include "core/security/security_gateway_service.h" #include "core/registry/registry.h" +#include "core/security/gh_advisory_vulnerability_sync_service.h" #include "core/security/vulnerability_database.h" +#include "core/security/snyk_vulnerability_sync_service.h" +#include "core/security/trivy_vulnerability_sync_service.h" #include "core/security/vulnerability_sync_service.h" #include "core/security/osv_core.h" @@ -27,6 +30,19 @@ std::string package_index_path_for_ecosystem(const ReqPackConfig& config, const return (root / ecosystem).string(); } +constexpr const char* kDefaultOsvFeedUrl = "https://storage.googleapis.com/osv-vulnerabilities"; + +std::string resolve_osv_feed_url(const ReqPackConfig& config, const std::string& backendFeedUrl) { + const std::string& topLevel = config.security.osvFeedUrl; + if (!backendFeedUrl.empty()) { + if (backendFeedUrl == kDefaultOsvFeedUrl && topLevel != kDefaultOsvFeedUrl) { + return topLevel; + } + return backendFeedUrl; + } + return topLevel; +} + } // namespace SecurityGatewayService::SecurityGatewayService( @@ -63,6 +79,8 @@ std::set SecurityGatewayService::configuredGatewayNames() const { std::set names; if (this->config.security.gateways.empty()) { names.insert(this->normalizeGatewayName(this->config.security.defaultGateway)); + const std::set discoveredProviders = this->discoverSecurityProviders(); + names.insert(discoveredProviders.begin(), discoveredProviders.end()); return names; } for (const auto& [name, gateway] : this->config.security.gateways) { @@ -70,6 +88,8 @@ std::set SecurityGatewayService::configuredGatewayNames() const { names.insert(name); } } + const std::set discoveredProviders = this->discoverSecurityProviders(); + names.insert(discoveredProviders.begin(), discoveredProviders.end()); return names; } @@ -80,6 +100,9 @@ bool SecurityGatewayService::isGatewaySystem(const std::string& system) const { std::set SecurityGatewayService::resolveGatewayBackends(const std::string& gateway) const { const std::string normalizedGateway = this->normalizeGatewayName(gateway); std::set discoveredProviders = this->discoverSecurityProviders(); + if (discoveredProviders.contains(normalizedGateway)) { + return {normalizedGateway}; + } if (this->config.security.gateways.empty()) { if (discoveredProviders.empty()) { return {"osv"}; @@ -178,29 +201,42 @@ std::vector SecurityGatewayService::ensureEcosystemsReady( std::vector findings; bool anySuccess = false; for (const std::string& backend : backends) { - if (backend != "osv") { - findings.push_back(ValidationFinding{ - .kind = "sync_warning", - .source = backend, - .severity = "low", - .message = "security backend not implemented yet", - }); - continue; - } - ReqPackConfig backendConfig = this->config; - if (const auto backendIt = this->config.security.backends.find("osv"); backendIt != this->config.security.backends.end()) { - backendConfig.security.osvFeedUrl = backendIt->second.feedUrl.empty() - ? backendConfig.security.osvFeedUrl - : backendIt->second.feedUrl; - backendConfig.security.osvRefreshMode = backendIt->second.refreshMode; - backendConfig.security.osvRefreshIntervalSeconds = backendIt->second.refreshIntervalSeconds; - if (!backendIt->second.overlayPath.empty()) { - backendConfig.security.osvOverlayPath = backendIt->second.overlayPath; + if (const auto backendIt = this->config.security.backends.find(backend); backendIt != this->config.security.backends.end()) { + if (backend == "osv") { + backendConfig.security.osvFeedUrl = resolve_osv_feed_url( + this->config, + backendIt->second.feedUrl + ); + backendConfig.security.osvRefreshMode = backendIt->second.refreshMode; + backendConfig.security.osvRefreshIntervalSeconds = backendIt->second.refreshIntervalSeconds; + if (!backendIt->second.overlayPath.empty()) { + backendConfig.security.osvOverlayPath = backendIt->second.overlayPath; + } + } else if (backend == "gh-advisory") { + backendConfig.security.backends[backend].feedUrl = backendIt->second.feedUrl; + backendConfig.security.backends[backend].refreshMode = backendIt->second.refreshMode; + backendConfig.security.backends[backend].refreshIntervalSeconds = backendIt->second.refreshIntervalSeconds; } } - if (forceRefresh) { + if (forceRefresh && (backend == "osv" || backend == "gh-advisory")) { backendConfig.security.osvRefreshMode = OsvRefreshMode::ALWAYS; + if (backend == "gh-advisory") { + backendConfig.security.backends[backend].refreshMode = OsvRefreshMode::ALWAYS; + } + } + + if (backend == "gh-advisory") { + GhAdvisoryVulnerabilitySyncService syncService(ecosystems, backendConfig); + std::vector backendFindings = syncService.ensureReady(forceRefresh); + const bool failed = std::any_of(backendFindings.begin(), backendFindings.end(), [](const ValidationFinding& finding) { + return finding.kind == "sync_error"; + }); + if (!failed) { + anySuccess = true; + } + findings.insert(findings.end(), backendFindings.begin(), backendFindings.end()); + continue; } std::vector backendFindings; @@ -208,8 +244,24 @@ std::vector SecurityGatewayService::ensureEcosystemsReady( ReqPackConfig ecosystemConfig = backendConfig; ecosystemConfig.security.osvDatabasePath = package_index_path_for_ecosystem(backendConfig, ecosystem); VulnerabilityDatabase database(ecosystemConfig); - VulnerabilitySyncService syncService(&database, this->metadataProvider, ecosystemConfig, {ecosystem}); - std::vector syncFindings = syncService.ensureReady(); + std::vector syncFindings; + if (backend == "osv") { + VulnerabilitySyncService syncService(&database, this->metadataProvider, ecosystemConfig, {ecosystem}); + syncFindings = syncService.ensureReady(); + } else if (backend == "snyk") { + SnykVulnerabilitySyncService syncService(&database, {ecosystem}, ecosystemConfig); + syncFindings = syncService.ensureReady(forceRefresh); + } else if (backend == "trivy") { + TrivyVulnerabilitySyncService syncService(&database, {ecosystem}, ecosystemConfig); + syncFindings = syncService.ensureReady(forceRefresh); + } else { + syncFindings.push_back(ValidationFinding{ + .kind = "sync_warning", + .source = backend, + .severity = "low", + .message = "security backend not implemented yet", + }); + } const bool failed = std::any_of(syncFindings.begin(), syncFindings.end(), [](const ValidationFinding& finding) { return finding.kind == "sync_error"; }); @@ -247,10 +299,16 @@ std::vector SecurityGatewayService::executeGatewayRequest( }}; } + const std::string gatewayName = gateway.empty() ? this->config.security.defaultGateway : gateway; + const std::set backends = this->resolveGatewayBackends(gatewayName); + const bool allowRawGatewayEcosystems = backends.size() == 1 && backends.contains("gh-advisory"); + std::set ecosystems; for (const Package& package : packages) { if (const auto resolved = this->resolveCanonicalEcosystem(package.name); resolved.has_value()) { ecosystems.insert(resolved.value()); + } else if (allowRawGatewayEcosystems && !package.name.empty()) { + ecosystems.insert(package.name); } } return this->ensureEcosystemsReady(ecosystems, gateway, action == ActionType::UPDATE); diff --git a/src/main/cpp/core/security/snyk_vulnerability_sync_service.cpp b/src/main/cpp/core/security/snyk_vulnerability_sync_service.cpp new file mode 100644 index 0000000..79d6485 --- /dev/null +++ b/src/main/cpp/core/security/snyk_vulnerability_sync_service.cpp @@ -0,0 +1,870 @@ +#include "core/security/snyk_vulnerability_sync_service.h" + +#include "core/common/network_environment.h" +#include "core/security/osv_core.h" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using boost::property_tree::ptree; + +constexpr std::size_t SNYK_IMPORT_BATCH_SIZE = 256; + +constexpr const char* SNYK_SYNC_KEY_LAST_SUCCESS = "snyk:last_successful_sync_at"; +constexpr const char* SNYK_SYNC_KEY_SCOPE = "snyk:scope"; +constexpr const char* SNYK_SYNC_KEY_LAST_UPDATED = "snyk:last_export_updated_at"; + +struct SnykGroupedAdvisory { + std::string id; + std::string title; + std::string cve; + std::string severity; + double score{0.0}; + std::string updatedAt; + std::string fixedInVersion; + std::string semverVulnerableRange; + std::string productName; + std::map> versionsByPackage; +}; + +std::string to_lower_copy(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return value; +} + +std::string trim_copy(const std::string& value) { + const std::size_t first = value.find_first_not_of(" \t\r\n"); + if (first == std::string::npos) { + return {}; + } + const std::size_t last = value.find_last_not_of(" \t\r\n"); + return value.substr(first, last - first + 1); +} + +bool starts_with(const std::string& value, const std::string& prefix) { + return value.rfind(prefix, 0) == 0; +} + +std::string json_escape(const std::string& value) { + std::string escaped; + escaped.reserve(value.size() + 8); + for (unsigned char c : value) { + switch (c) { + case '\\': escaped += "\\\\"; break; + case '"': escaped += "\\\""; break; + case '\n': escaped += "\\n"; break; + case '\r': escaped += "\\r"; break; + case '\t': escaped += "\\t"; break; + default: escaped.push_back(static_cast(c)); break; + } + } + return escaped; +} + +std::string now_epoch_seconds() { + const auto now = std::chrono::system_clock::now().time_since_epoch(); + return std::to_string(std::chrono::duration_cast(now).count()); +} + +std::string join_scope_key(const std::set& ecosystems) { + std::ostringstream stream; + bool first = true; + for (const std::string& ecosystem : ecosystems) { + if (!first) { + stream << '\n'; + } + first = false; + stream << ecosystem; + } + return stream.str(); +} + +bool parse_json_tree(const std::string& json, ptree& tree) { + try { + std::istringstream input(json); + boost::property_tree::read_json(input, tree); + return true; + } catch (...) { + return false; + } +} + +std::optional resolve_file_fixture_path(const std::string& url) { + if (!starts_with(url, "file://")) { + return std::nullopt; + } + + const std::filesystem::path rawPath(url.substr(std::string{"file://"}.size())); + std::error_code error; + if (std::filesystem::is_regular_file(rawPath, error) && !error) { + return rawPath; + } + error.clear(); + const std::filesystem::path indexPath = rawPath / "index.json"; + if (std::filesystem::is_regular_file(indexPath, error) && !error) { + return indexPath; + } + error.clear(); + const std::filesystem::path jsonPath = rawPath.string() + ".json"; + if (std::filesystem::is_regular_file(jsonPath, error) && !error) { + return jsonPath; + } + return std::nullopt; +} + +std::size_t write_to_string(void* contents, std::size_t size, std::size_t nmemb, void* userp) { + const std::size_t bytes = size * nmemb; + auto* buffer = static_cast(userp); + buffer->append(static_cast(contents), bytes); + return bytes; +} + +std::size_t write_to_file(void* contents, std::size_t size, std::size_t nmemb, void* userp) { + const std::size_t bytes = size * nmemb; + auto* output = static_cast(userp); + output->write(static_cast(contents), static_cast(bytes)); + return output->good() ? bytes : 0; +} + +bool configure_curl(CURL* curl, const ReqPackConfig& config, const std::string& url) { + if (curl == nullptr) { + return false; + } + + reqpack_apply_curl_ca_bundle(curl); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, config.downloader.followRedirects ? 1L : 0L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, config.downloader.connectTimeoutSeconds); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, config.downloader.requestTimeoutSeconds); + curl_easy_setopt(curl, CURLOPT_USERAGENT, config.downloader.userAgent.c_str()); + return true; +} + +std::optional http_request( + const ReqPackConfig& config, + const std::string& method, + const std::string& url, + const std::vector& headers, + const std::optional& body = std::nullopt +) { + if (starts_with(url, "file://")) { + const auto resolved = resolve_file_fixture_path(url); + if (!resolved.has_value()) { + return std::nullopt; + } + const std::filesystem::path& path = resolved.value(); + std::ifstream input(path, std::ios::binary); + if (!input.is_open()) { + return std::nullopt; + } + std::ostringstream buffer; + buffer << input.rdbuf(); + return buffer.str(); + } + + std::string buffer; + CURL* curl = curl_easy_init(); + if (!configure_curl(curl, config, url)) { + return std::nullopt; + } + + struct curl_slist* headerList = nullptr; + for (const std::string& header : headers) { + headerList = curl_slist_append(headerList, header.c_str()); + } + if (headerList != nullptr) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerList); + } + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_to_string); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); + if (body.has_value()) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body->c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(body->size())); + } + + long statusCode = 0; + const CURLcode result = curl_easy_perform(curl); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode); + if (headerList != nullptr) { + curl_slist_free_all(headerList); + } + curl_easy_cleanup(curl); + + if (result != CURLE_OK || statusCode >= 400) { + return std::nullopt; + } + return buffer; +} + +bool fetch_file(const ReqPackConfig& config, const std::string& url, const std::filesystem::path& path) { + if (starts_with(url, "file://")) { + std::ifstream input(url.substr(std::string{"file://"}.size()), std::ios::binary); + if (!input.is_open()) { + return false; + } + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output.is_open()) { + return false; + } + output << input.rdbuf(); + return output.good(); + } + + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output.is_open()) { + return false; + } + + CURL* curl = curl_easy_init(); + if (!configure_curl(curl, config, url)) { + return false; + } + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_to_file); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &output); + + long statusCode = 0; + const CURLcode result = curl_easy_perform(curl); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode); + curl_easy_cleanup(curl); + output.flush(); + const bool ok = result == CURLE_OK && statusCode < 400 && output.good(); + output.close(); + if (!ok) { + std::error_code cleanupError; + std::filesystem::remove(path, cleanupError); + } + return ok; +} + +double parse_score(const std::string& value) { + const std::string trimmed = trim_copy(value); + if (trimmed.empty()) { + return 0.0; + } + try { + return std::stod(trimmed); + } catch (...) { + return 0.0; + } +} + +std::string severity_from_score(const double score) { + if (score >= 9.0) return "critical"; + if (score >= 7.0) return "high"; + if (score >= 4.0) return "medium"; + if (score > 0.0) return "low"; + return "unassigned"; +} + +int severity_rank(const std::string& severity) { + if (severity == "critical") return 4; + if (severity == "high") return 3; + if (severity == "medium") return 2; + if (severity == "low") return 1; + return 0; +} + +std::string severity_from_row(const std::string& value, const double score) { + const std::string normalized = to_lower_copy(trim_copy(value)); + if (normalized == "critical" || normalized == "high" || normalized == "medium" || normalized == "low") { + return normalized; + } + if (normalized == "moderate") { + return "medium"; + } + return severity_from_score(score); +} + +bool is_maven_purl(const std::string& value) { + return value.rfind("pkg:maven/", 0) == 0; +} + +std::optional parse_maven_package_name_and_version(const std::string& value) { + const std::string trimmed = trim_copy(value); + if (!is_maven_purl(trimmed)) { + return std::nullopt; + } + + const std::size_t at = trimmed.rfind('@'); + if (at == std::string::npos || at <= std::string{"pkg:maven/"}.size() || at == trimmed.size() - 1) { + return std::nullopt; + } + + const std::string packagePath = trimmed.substr(std::string{"pkg:maven/"}.size(), at - std::string{"pkg:maven/"}.size()); + const std::size_t slash = packagePath.rfind('/'); + if (slash == std::string::npos || slash == 0 || slash == packagePath.size() - 1) { + return std::nullopt; + } + + return ParsedSnykPackage{ + .name = packagePath.substr(0, slash) + ":" + packagePath.substr(slash + 1), + .version = trimmed.substr(at + 1), + }; +} + +std::vector parse_csv_line_impl(std::string_view line) { + std::vector fields; + std::string current; + bool quoted = false; + + for (std::size_t index = 0; index < line.size(); ++index) { + const char ch = line[index]; + if (quoted) { + if (ch == '"') { + if (index + 1 < line.size() && line[index + 1] == '"') { + current.push_back('"'); + ++index; + } else { + quoted = false; + } + } else { + current.push_back(ch); + } + continue; + } + + if (ch == '"') { + quoted = true; + continue; + } + if (ch == ',') { + fields.push_back(current); + current.clear(); + continue; + } + current.push_back(ch); + } + + fields.push_back(current); + return fields; +} + +std::optional parse_export_row_impl( + const std::vector& header, + const std::vector& values +) { + if (header.size() != values.size()) { + return std::nullopt; + } + + std::map row; + for (std::size_t index = 0; index < header.size(); ++index) { + row[header[index]] = values[index]; + } + + SnykExportRow parsed; + parsed.problemId = trim_copy(row["PROBLEM_ID"]); + parsed.problemTitle = trim_copy(row["PROBLEM_TITLE"]); + parsed.cve = trim_copy(row["CVE"]); + parsed.packageNameAndVersion = trim_copy(row["PACKAGE_NAME_AND_VERSION"]); + parsed.semverVulnerableRange = trim_copy(row["SEMVER_VULNERABLE_RANGE"]); + parsed.issueSeverity = trim_copy(row["ISSUE_SEVERITY"]); + parsed.nvdScore = parse_score(row["NVD_SCORE"]); + parsed.snykCvssScore = parse_score(row["SNYK_CVSS_SCORE"]); + parsed.updatedAt = trim_copy(row["UPDATED_AT"]); + parsed.fixedInVersion = trim_copy(row["FIXED_IN_VERSION"]); + parsed.productName = trim_copy(row["PRODUCT_NAME"]); + + if (parsed.problemId.empty() || parsed.packageNameAndVersion.empty()) { + return std::nullopt; + } + return parsed; +} + +std::string build_export_body(const std::string& dataset) { + return std::string{"{\"data\":{\"attributes\":{\"dataset\":\""} + json_escape(dataset) + + "\",\"filters\":{\"updated\":{\"from\":\"1970-01-01T00:00:00Z\",\"to\":\"9999-12-31T23:59:59Z\"}}," + "\"columns\":[\"PROBLEM_ID\",\"PROBLEM_TITLE\",\"CVE\",\"PACKAGE_NAME_AND_VERSION\",\"SEMVER_VULNERABLE_RANGE\",\"ISSUE_SEVERITY\",\"NVD_SCORE\",\"SNYK_CVSS_SCORE\",\"UPDATED_AT\",\"FIXED_IN_VERSION\",\"PRODUCT_NAME\"]," + "\"url_expiration_seconds\":3600}}}"; +} + +std::vector build_headers(const SecurityBackendConfig& backend, const std::string& token) { + return { + "Authorization: token " + token, + "Version: " + backend.apiVersion, + "Content-Type: application/vnd.api+json", + "Accept: application/vnd.api+json", + }; +} + +std::string endpoint_for(const SecurityBackendConfig& backend, const std::string& path) { + const std::string base = backend.apiBaseUrl.empty() ? std::string{"https://api.snyk.io/rest"} : backend.apiBaseUrl; + if (base.back() == '/' && !path.empty() && path.front() == '/') { + return base.substr(0, base.size() - 1) + path; + } + if (base.back() != '/' && !path.empty() && path.front() != '/') { + return base + "/" + path; + } + return base + path; +} + +std::optional read_env_value(const std::string& name) { + if (name.empty()) { + return std::nullopt; + } + const char* value = std::getenv(name.c_str()); + if (value == nullptr || value[0] == '\0') { + return std::nullopt; + } + return std::string(value); +} + +std::optional create_export_job(const ReqPackConfig& config, const SecurityBackendConfig& backend, const std::string& token) { + const std::vector headers = build_headers(backend, token); + const std::string scopePath = !backend.orgId.empty() + ? "/orgs/" + backend.orgId + "/export" + : "/groups/" + backend.groupId + "/export"; + const auto payload = http_request(config, "POST", endpoint_for(backend, scopePath), headers, build_export_body(backend.dataset)); + if (!payload.has_value()) { + return std::nullopt; + } + ptree tree; + if (!parse_json_tree(payload.value(), tree)) { + return std::nullopt; + } + const std::string exportId = trim_copy(tree.get("export_id", {})); + return exportId.empty() ? std::nullopt : std::optional(exportId); +} + +std::optional export_status(const ReqPackConfig& config, const SecurityBackendConfig& backend, const std::string& token, const std::string& exportId) { + const std::vector headers = build_headers(backend, token); + const std::string scopePath = !backend.orgId.empty() + ? "/orgs/" + backend.orgId + "/jobs/export/" + exportId + : "/groups/" + backend.groupId + "/jobs/export/" + exportId; + const auto payload = http_request(config, "GET", endpoint_for(backend, scopePath), headers); + if (!payload.has_value()) { + return std::nullopt; + } + ptree tree; + if (!parse_json_tree(payload.value(), tree)) { + return std::nullopt; + } + const std::string status = trim_copy(tree.get("status", {})); + return status.empty() ? std::nullopt : std::optional(status); +} + +std::optional export_download_url(const ReqPackConfig& config, const SecurityBackendConfig& backend, const std::string& token, const std::string& exportId) { + const std::vector headers = build_headers(backend, token); + const std::string scopePath = !backend.orgId.empty() + ? "/orgs/" + backend.orgId + "/export/" + exportId + : "/groups/" + backend.groupId + "/export/" + exportId; + const auto payload = http_request(config, "GET", endpoint_for(backend, scopePath), headers); + if (!payload.has_value()) { + return std::nullopt; + } + ptree tree; + if (!parse_json_tree(payload.value(), tree)) { + return std::nullopt; + } + const std::string url = trim_copy(tree.get("download_url", {})); + return url.empty() ? std::nullopt : std::optional(url); +} + +std::optional download_export_csv( + const ReqPackConfig& config, + const std::filesystem::path& root, + const std::string& downloadUrl +) { + std::error_code error; + std::filesystem::create_directories(root, error); + if (error) { + return std::nullopt; + } + const std::filesystem::path path = root / "snyk-export.csv"; + if (!fetch_file(config, downloadUrl, path)) { + return std::nullopt; + } + return path; +} + +std::optional> load_export_rows(const std::filesystem::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input.is_open()) { + return std::nullopt; + } + + std::string line; + if (!std::getline(input, line)) { + return std::nullopt; + } + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + const std::vector header = parse_csv_line_impl(line); + std::vector rows; + while (std::getline(input, line)) { + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + if (trim_copy(line).empty()) { + continue; + } + const auto parsed = parse_export_row_impl(header, parse_csv_line_impl(line)); + if (parsed.has_value()) { + rows.push_back(parsed.value()); + } + } + return rows; +} + +bool row_matches_ecosystem(const SnykExportRow& row, const std::string& ecosystem) { + return ecosystem == "Maven" && is_maven_purl(row.packageNameAndVersion); +} + +std::optional build_indexed_advisory(const SnykGroupedAdvisory& grouped, const std::string& ecosystem) { + std::ostringstream raw; + raw << '{'; + raw << "\"id\":\"" << json_escape(grouped.id) << "\","; + raw << "\"modified\":\"" << json_escape(grouped.updatedAt) << "\","; + raw << "\"summary\":\"" << json_escape(grouped.title.empty() ? grouped.id : grouped.title) << "\","; + raw << "\"details\":\"" << json_escape(grouped.fixedInVersion.empty() + ? std::string{"Imported from Snyk issues export"} + : "Imported from Snyk issues export. Fixed in: " + grouped.fixedInVersion) << "\","; + if (!grouped.cve.empty()) { + raw << "\"aliases\":[\"" << json_escape(grouped.cve) << "\"],"; + } + raw << "\"severity\":[{\"type\":\"CVSS_V3\",\"score\":\"" << grouped.score << "\"}],"; + raw << "\"database_specific\":{\"severity\":\"" << json_escape(grouped.severity) << "\"},"; + raw << "\"affected\":["; + + bool firstPackage = true; + for (const auto& [packageName, versions] : grouped.versionsByPackage) { + if (!firstPackage) { + raw << ','; + } + firstPackage = false; + raw << '{'; + raw << "\"package\":{\"ecosystem\":\"" << json_escape(ecosystem) << "\",\"name\":\"" << json_escape(packageName) << "\"},"; + raw << "\"database_specific\":{\"severity\":\"" << json_escape(grouped.severity) << "\"},"; + raw << "\"versions\":["; + bool firstVersion = true; + for (const std::string& version : versions) { + if (!firstVersion) { + raw << ','; + } + firstVersion = false; + raw << "\"" << json_escape(version) << "\""; + } + raw << ']'; + raw << '}'; + } + raw << "]}"; + + const auto parsed = osv_parse_advisory(raw.str()); + if (!parsed.has_value()) { + return std::nullopt; + } + + std::vector> packageKeys; + for (const OsvAffectedPackage& package : parsed->affected) { + packageKeys.emplace_back(package.ecosystem, osv_normalize_package_name(package.name)); + } + + return VulnerabilityDatabase::IndexedAdvisory{ + .advisory = parsed.value(), + .packageKeys = packageKeys, + }; +} + +std::vector import_rows( + VulnerabilityDatabase* database, + const std::set& ecosystems, + const std::vector& rows +) { + std::vector findings; + if (database == nullptr || ecosystems.empty()) { + return findings; + } + std::map grouped; + + for (const SnykExportRow& row : rows) { + for (const std::string& ecosystem : ecosystems) { + if (!row_matches_ecosystem(row, ecosystem)) { + continue; + } + const auto parsed = parse_maven_package_name_and_version(row.packageNameAndVersion); + if (!parsed.has_value()) { + findings.push_back(ValidationFinding{ + .kind = "sync_warning", + .source = "snyk", + .severity = "low", + .message = "skipping Snyk row with unsupported package format", + }); + continue; + } + + SnykGroupedAdvisory& advisory = grouped[row.problemId]; + advisory.id = row.problemId; + advisory.title = row.problemTitle; + advisory.cve = row.cve; + advisory.updatedAt = std::max(advisory.updatedAt, row.updatedAt); + advisory.fixedInVersion = advisory.fixedInVersion.empty() ? row.fixedInVersion : advisory.fixedInVersion; + advisory.semverVulnerableRange = advisory.semverVulnerableRange.empty() ? row.semverVulnerableRange : advisory.semverVulnerableRange; + advisory.productName = advisory.productName.empty() ? row.productName : advisory.productName; + const double score = row.snykCvssScore > 0.0 ? row.snykCvssScore : row.nvdScore; + if (score > advisory.score) { + advisory.score = score; + } + const std::string severity = severity_from_row(row.issueSeverity, score); + if (severity_rank(severity) > severity_rank(advisory.severity)) { + advisory.severity = severity; + } + advisory.versionsByPackage[parsed->name].insert(parsed->version); + } + } + + std::vector advisories; + advisories.reserve(grouped.size()); + std::string latestUpdated; + for (const auto& [_, groupedAdvisory] : grouped) { + if (!groupedAdvisory.updatedAt.empty() && groupedAdvisory.updatedAt > latestUpdated) { + latestUpdated = groupedAdvisory.updatedAt; + } + const auto advisory = build_indexed_advisory(groupedAdvisory, *ecosystems.begin()); + if (advisory.has_value()) { + advisories.push_back(advisory.value()); + } + } + + if (advisories.empty()) { + if (!database->clearAll()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "failed to reset local Snyk database", + }}; + } + (void)database->setSyncState(SNYK_SYNC_KEY_SCOPE, join_scope_key(ecosystems)); + (void)database->setSyncState(SNYK_SYNC_KEY_LAST_SUCCESS, now_epoch_seconds()); + return findings; + } + + if (!database->clearAll()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "failed to reset local Snyk database", + }}; + } + if (!database->upsertIndexedAdvisories(advisories, SNYK_IMPORT_BATCH_SIZE)) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "failed to import Snyk advisory batch", + }}; + } + + if (!latestUpdated.empty()) { + (void)database->setSyncState(SNYK_SYNC_KEY_LAST_UPDATED, latestUpdated); + } + (void)database->setSyncState(SNYK_SYNC_KEY_SCOPE, join_scope_key(ecosystems)); + (void)database->setSyncState(SNYK_SYNC_KEY_LAST_SUCCESS, now_epoch_seconds()); + return findings; +} + +bool should_refresh(const VulnerabilityDatabase* database, const SecurityBackendConfig& backend, bool forceRefresh) { + if (forceRefresh || database == nullptr) { + return true; + } + if (backend.refreshMode == OsvRefreshMode::ALWAYS) { + return true; + } + if (backend.refreshMode == OsvRefreshMode::MANUAL) { + return !database->hasAdvisories(); + } + + const auto lastSync = database->getSyncState(SNYK_SYNC_KEY_LAST_SUCCESS); + if (!lastSync.has_value()) { + return true; + } + try { + const long last = std::stol(lastSync.value()); + const long now = std::stol(now_epoch_seconds()); + return (now - last) >= backend.refreshIntervalSeconds; + } catch (...) { + return true; + } +} + +bool scope_matches(const VulnerabilityDatabase* database, const std::set& ecosystems) { + if (database == nullptr) { + return false; + } + return database->getSyncState(SNYK_SYNC_KEY_SCOPE).value_or(std::string{}) == join_scope_key(ecosystems); +} + +} // namespace + +std::vector snyk_parse_csv_line(std::string_view line) { + return parse_csv_line_impl(line); +} + +std::optional snyk_parse_export_row( + const std::vector& header, + const std::vector& values +) { + return parse_export_row_impl(header, values); +} + +std::optional snyk_parse_maven_package_name_and_version(const std::string& value) { + const auto parsed = parse_maven_package_name_and_version(value); + if (!parsed.has_value()) { + return std::nullopt; + } + return ParsedSnykPackage{.name = parsed->name, .version = parsed->version}; +} + +std::string snyk_normalize_severity(const std::string& value, double score) { + return severity_from_row(value, score); +} + +SnykVulnerabilitySyncService::SnykVulnerabilitySyncService( + VulnerabilityDatabase* database, + std::set ecosystems, + const ReqPackConfig& config +) : config(config), database(database), ecosystems(std::move(ecosystems)) {} + +std::vector SnykVulnerabilitySyncService::ensureReady(bool forceRefresh) const { + if (this->database == nullptr || !this->database->ensureReady()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "vulnerability database unavailable", + }}; + } + + const auto backendIt = this->config.security.backends.find("snyk"); + if (backendIt == this->config.security.backends.end()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "missing Snyk backend configuration", + }}; + } + const SecurityBackendConfig& backend = backendIt->second; + const auto token = read_env_value(backend.tokenEnv); + if (!token.has_value()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "missing Snyk API token in environment '" + backend.tokenEnv + "'", + }}; + } + const bool hasOrg = !backend.orgId.empty(); + const bool hasGroup = !backend.groupId.empty(); + if (hasOrg == hasGroup) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "configure exactly one of Snyk orgId or groupId", + }}; + } + + if (scope_matches(this->database, this->ecosystems) && !should_refresh(this->database, backend, forceRefresh)) { + return {}; + } + + const auto exportId = create_export_job(this->config, backend, token.value()); + if (!exportId.has_value()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "failed to create Snyk export job", + }}; + } + + for (int attempt = 0; attempt < 20; ++attempt) { + const auto status = export_status(this->config, backend, token.value(), exportId.value()); + if (!status.has_value()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "failed to poll Snyk export status", + }}; + } + if (status.value() == "FINISHED") { + break; + } + if (status.value() == "ERROR") { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "Snyk export job failed", + }}; + } + if (attempt == 19) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "timed out waiting for Snyk export job", + }}; + } + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + } + + const auto downloadUrl = export_download_url(this->config, backend, token.value(), exportId.value()); + if (!downloadUrl.has_value()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "failed to retrieve Snyk export download URL", + }}; + } + + const std::filesystem::path tempRoot = std::filesystem::path(this->config.security.cachePath) / "snyk"; + const auto csvPath = download_export_csv(this->config, tempRoot, downloadUrl.value()); + if (!csvPath.has_value()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "failed to download Snyk export CSV", + }}; + } + + const auto rows = load_export_rows(csvPath.value()); + if (!rows.has_value()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "snyk", + .severity = "high", + .message = "failed to parse Snyk export CSV", + }}; + } + + return import_rows(this->database, this->ecosystems, rows.value()); +} diff --git a/src/main/cpp/core/security/trivy_vulnerability_sync_service.cpp b/src/main/cpp/core/security/trivy_vulnerability_sync_service.cpp new file mode 100644 index 0000000..c18e109 --- /dev/null +++ b/src/main/cpp/core/security/trivy_vulnerability_sync_service.cpp @@ -0,0 +1,1129 @@ +#include "core/security/trivy_vulnerability_sync_service.h" + +#include "core/archive/archive_resolver.h" +#include "core/common/network_environment.h" +#include "core/common/process_runner.h" +#include "core/security/osv_core.h" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using boost::property_tree::ptree; + +constexpr std::size_t TRIVY_IMPORT_BATCH_SIZE = 256; + +constexpr const char* TRIVY_SYNC_KEY_LAST_SUCCESS = "trivy:last_successful_sync_at"; +constexpr const char* TRIVY_SYNC_KEY_SOURCE_REPOSITORY = "trivy:source_repository"; +constexpr const char* TRIVY_SYNC_KEY_METADATA_VERSION = "trivy:db_metadata_version"; +constexpr const char* TRIVY_SYNC_KEY_SCOPE = "trivy:scope"; + +constexpr const char* TRIVY_DB_LAYER_MEDIA_TYPE = "application/vnd.aquasec.trivy.db.layer.v1.tar+gzip"; + +#ifndef REQPACK_TRIVY_HELPER_DEFAULT_PATH +#define REQPACK_TRIVY_HELPER_DEFAULT_PATH "" +#endif + +struct HttpResponse { + long statusCode{0}; + std::string body{}; + std::map headers{}; +}; + +struct ProcessResult { + int exitCode{1}; + std::string stdoutText{}; + std::string stderrText{}; +}; + +struct OciReference { + std::string original; + std::string registry; + std::string repository; + std::string reference; +}; + +struct BearerChallenge { + std::string realm; + std::string service; + std::string scope; +}; + +struct TrivyDbMetadata { + std::string version; +}; + +struct TrivyGroupedPackage { + std::set versions; + std::vector ranges; +}; + +struct TrivyGroupedAdvisory { + std::string ecosystem; + std::string id; + std::set aliases; + std::string modified; + std::string published; + std::string summary; + std::string description; + std::string severity{"unassigned"}; + double score{0.0}; + std::set references; + std::map packages; +}; + +std::string to_lower_copy(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return value; +} + +std::string trim_copy(const std::string& value) { + const std::size_t first = value.find_first_not_of(" \t\r\n"); + if (first == std::string::npos) { + return {}; + } + const std::size_t last = value.find_last_not_of(" \t\r\n"); + return value.substr(first, last - first + 1); +} + +bool starts_with(const std::string& value, const std::string& prefix) { + return value.rfind(prefix, 0) == 0; +} + +std::string json_escape(const std::string& value) { + std::string escaped; + escaped.reserve(value.size() + 8); + for (unsigned char c : value) { + switch (c) { + case '\\': escaped += "\\\\"; break; + case '"': escaped += "\\\""; break; + case '\n': escaped += "\\n"; break; + case '\r': escaped += "\\r"; break; + case '\t': escaped += "\\t"; break; + default: escaped.push_back(static_cast(c)); break; + } + } + return escaped; +} + +std::string now_epoch_seconds() { + const auto now = std::chrono::system_clock::now().time_since_epoch(); + return std::to_string(std::chrono::duration_cast(now).count()); +} + +std::string join_scope_key(const std::set& ecosystems) { + std::ostringstream stream; + bool first = true; + for (const std::string& ecosystem : ecosystems) { + if (!first) { + stream << '\n'; + } + first = false; + stream << ecosystem; + } + return stream.str(); +} + +bool parse_json_tree(const std::string& json, ptree& tree) { + try { + std::istringstream input(json); + boost::property_tree::read_json(input, tree); + return true; + } catch (...) { + return false; + } +} + +std::string normalize_trivy_severity(const std::string& value, double score) { + const std::string normalized = to_lower_copy(trim_copy(value)); + if (normalized == "critical" || normalized == "high" || normalized == "medium" || normalized == "low") { + return normalized; + } + if (score >= 9.0) { + return "critical"; + } + if (score >= 7.0) { + return "high"; + } + if (score >= 4.0) { + return "medium"; + } + if (score > 0.0) { + return "low"; + } + return "unassigned"; +} + +int severity_rank(const std::string& severity) { + if (severity == "critical") return 4; + if (severity == "high") return 3; + if (severity == "medium") return 2; + if (severity == "low") return 1; + return 0; +} + +std::string canonicalize_header_name(std::string name) { + name = trim_copy(name); + std::transform(name.begin(), name.end(), name.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return name; +} + +std::size_t write_to_string(void* contents, std::size_t size, std::size_t nmemb, void* userp) { + const std::size_t bytes = size * nmemb; + auto* buffer = static_cast(userp); + buffer->append(static_cast(contents), bytes); + return bytes; +} + +std::size_t write_to_file(void* contents, std::size_t size, std::size_t nmemb, void* userp) { + const std::size_t bytes = size * nmemb; + auto* output = static_cast(userp); + output->write(static_cast(contents), static_cast(bytes)); + return output->good() ? bytes : 0; +} + +std::size_t capture_headers(void* contents, std::size_t size, std::size_t nmemb, void* userp) { + const std::size_t bytes = size * nmemb; + std::string line(static_cast(contents), bytes); + auto* headers = static_cast*>(userp); + const std::size_t separator = line.find(':'); + if (separator != std::string::npos) { + const std::string key = canonicalize_header_name(line.substr(0, separator)); + const std::string value = trim_copy(line.substr(separator + 1)); + if (!key.empty() && !value.empty()) { + (*headers)[key] = value; + } + } + return bytes; +} + +bool configure_curl(CURL* curl, const ReqPackConfig& config, const std::string& url) { + if (curl == nullptr) { + return false; + } + + reqpack_apply_curl_ca_bundle(curl); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, config.downloader.followRedirects ? 1L : 0L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, config.downloader.connectTimeoutSeconds); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, config.downloader.requestTimeoutSeconds); + curl_easy_setopt(curl, CURLOPT_USERAGENT, config.downloader.userAgent.c_str()); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &capture_headers); + return true; +} + +HttpResponse http_request( + const ReqPackConfig& config, + const std::string& method, + const std::string& url, + const std::vector& headers, + const std::optional& body = std::nullopt +) { + HttpResponse response; + CURL* curl = curl_easy_init(); + if (!configure_curl(curl, config, url)) { + return response; + } + + struct curl_slist* headerList = nullptr; + for (const std::string& header : headers) { + headerList = curl_slist_append(headerList, header.c_str()); + } + if (headerList != nullptr) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerList); + } + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_to_string); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response.body); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &response.headers); + if (body.has_value()) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body->c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(body->size())); + } + + const CURLcode result = curl_easy_perform(curl); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response.statusCode); + if (headerList != nullptr) { + curl_slist_free_all(headerList); + } + curl_easy_cleanup(curl); + if (result != CURLE_OK) { + response.statusCode = 0; + } + return response; +} + +bool http_download_to_path( + const ReqPackConfig& config, + const std::string& url, + const std::vector& headers, + const std::filesystem::path& path, + long& statusCode +) { + statusCode = 0; + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output.is_open()) { + return false; + } + + CURL* curl = curl_easy_init(); + if (!configure_curl(curl, config, url)) { + return false; + } + + struct curl_slist* headerList = nullptr; + for (const std::string& header : headers) { + headerList = curl_slist_append(headerList, header.c_str()); + } + if (headerList != nullptr) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerList); + } + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_to_file); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &output); + std::map responseHeaders; + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &responseHeaders); + + const CURLcode result = curl_easy_perform(curl); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode); + if (headerList != nullptr) { + curl_slist_free_all(headerList); + } + curl_easy_cleanup(curl); + output.flush(); + const bool ok = result == CURLE_OK && statusCode < 400 && output.good(); + output.close(); + if (!ok) { + std::error_code cleanupError; + std::filesystem::remove(path, cleanupError); + } + return ok; +} + +std::optional parse_oci_reference(const std::string& value) { + const std::size_t firstSlash = value.find('/'); + if (firstSlash == std::string::npos || firstSlash == 0 || firstSlash == value.size() - 1) { + return std::nullopt; + } + + const std::size_t lastColon = value.rfind(':'); + if (lastColon == std::string::npos || lastColon < firstSlash || lastColon == value.size() - 1) { + return std::nullopt; + } + + return OciReference{ + .original = value, + .registry = value.substr(0, firstSlash), + .repository = value.substr(firstSlash + 1, lastColon - firstSlash - 1), + .reference = value.substr(lastColon + 1), + }; +} + +std::string manifest_url(const OciReference& reference) { + return "https://" + reference.registry + "/v2/" + reference.repository + "/manifests/" + reference.reference; +} + +std::string blob_url(const OciReference& reference, const std::string& digest) { + return "https://" + reference.registry + "/v2/" + reference.repository + "/blobs/" + digest; +} + +std::string manifest_digest_url(const OciReference& reference, const std::string& digest) { + return "https://" + reference.registry + "/v2/" + reference.repository + "/manifests/" + digest; +} + +std::optional parse_bearer_challenge(const std::string& header) { + if (header.empty()) { + return std::nullopt; + } + const std::string normalized = to_lower_copy(header); + if (!starts_with(normalized, "bearer ")) { + return std::nullopt; + } + + BearerChallenge challenge; + const std::regex fieldRegex(R"field(([A-Za-z]+)="([^"]*)")field"); + auto begin = std::sregex_iterator(header.begin(), header.end(), fieldRegex); + auto end = std::sregex_iterator(); + for (auto it = begin; it != end; ++it) { + const std::string key = to_lower_copy((*it)[1].str()); + const std::string value = (*it)[2].str(); + if (key == "realm") { + challenge.realm = value; + } else if (key == "service") { + challenge.service = value; + } else if (key == "scope") { + challenge.scope = value; + } + } + + if (challenge.realm.empty()) { + return std::nullopt; + } + return challenge; +} + +std::string url_encode(const std::string& value) { + CURL* curl = curl_easy_init(); + if (curl == nullptr) { + return value; + } + char* escaped = curl_easy_escape(curl, value.c_str(), static_cast(value.size())); + std::string result = escaped == nullptr ? value : std::string(escaped); + if (escaped != nullptr) { + curl_free(escaped); + } + curl_easy_cleanup(curl); + return result; +} + +std::optional request_bearer_token(const ReqPackConfig& config, const BearerChallenge& challenge) { + std::string url = challenge.realm; + const char separator = challenge.realm.find('?') == std::string::npos ? '?' : '&'; + url.push_back(separator); + url += "service=" + url_encode(challenge.service); + if (!challenge.scope.empty()) { + url += "&scope=" + url_encode(challenge.scope); + } + const HttpResponse response = http_request(config, "GET", url, {}); + if (response.statusCode >= 400 || response.body.empty()) { + return std::nullopt; + } + ptree tree; + if (!parse_json_tree(response.body, tree)) { + return std::nullopt; + } + const std::string token = trim_copy(tree.get("token", tree.get("access_token", {}))); + return token.empty() ? std::nullopt : std::optional(token); +} + +std::vector manifest_headers(const std::optional& token) { + std::vector headers{ + "Accept: application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json", + }; + if (token.has_value()) { + headers.push_back("Authorization: Bearer " + token.value()); + } + return headers; +} + +std::vector blob_headers(const std::optional& token) { + if (!token.has_value()) { + return {}; + } + return {"Authorization: Bearer " + token.value()}; +} + +std::optional select_db_layer_digest(const std::string& manifestJson) { + ptree tree; + if (!parse_json_tree(manifestJson, tree)) { + return std::nullopt; + } + + if (const auto layers = tree.get_child_optional("layers")) { + for (const auto& [_, layer] : layers.value()) { + const std::string mediaType = trim_copy(layer.get("mediaType", {})); + std::string title; + if (const auto annotations = layer.get_child_optional("annotations")) { + for (const auto& [annotationKey, annotationValue] : annotations.value()) { + if (annotationKey == "org.opencontainers.image.title") { + title = trim_copy(annotationValue.get_value()); + break; + } + } + } + if (mediaType == TRIVY_DB_LAYER_MEDIA_TYPE || title == "db.tar.gz") { + const std::string digest = trim_copy(layer.get("digest", {})); + if (!digest.empty()) { + return digest; + } + } + } + for (const auto& [_, layer] : layers.value()) { + const std::string digest = trim_copy(layer.get("digest", {})); + if (!digest.empty()) { + return digest; + } + } + } + return std::nullopt; +} + +std::optional load_manifest_with_auth( + const ReqPackConfig& config, + const OciReference& reference, + std::optional& token +) { + HttpResponse response = http_request(config, "GET", manifest_url(reference), manifest_headers(token)); + if (response.statusCode == 401 && !token.has_value()) { + const auto challenge = parse_bearer_challenge(response.headers["www-authenticate"]); + if (!challenge.has_value()) { + return std::nullopt; + } + token = request_bearer_token(config, challenge.value()); + if (!token.has_value()) { + return std::nullopt; + } + response = http_request(config, "GET", manifest_url(reference), manifest_headers(token)); + } + if (response.statusCode >= 400 || response.body.empty()) { + return std::nullopt; + } + + ptree tree; + if (!parse_json_tree(response.body, tree)) { + return std::nullopt; + } + if (!tree.get_child_optional("layers").has_value()) { + std::string nestedDigest; + if (const auto manifests = tree.get_child_optional("manifests")) { + for (const auto& [_, manifestNode] : manifests.value()) { + nestedDigest = trim_copy(manifestNode.get("digest", {})); + if (!nestedDigest.empty()) { + break; + } + } + } + if (nestedDigest.empty()) { + return std::nullopt; + } + const HttpResponse nested = http_request(config, "GET", manifest_digest_url(reference, nestedDigest), manifest_headers(token)); + if (nested.statusCode >= 400 || nested.body.empty()) { + return std::nullopt; + } + return nested.body; + } + return response.body; +} + +bool ensure_directory(const std::filesystem::path& path) { + if (path.empty()) { + return true; + } + std::error_code error; + std::filesystem::create_directories(path, error); + return !error; +} + +bool copy_file_replace(const std::filesystem::path& source, const std::filesystem::path& target) { + if (!ensure_directory(target.parent_path())) { + return false; + } + std::error_code error; + std::filesystem::copy_file(source, target, std::filesystem::copy_options::overwrite_existing, error); + return !error; +} + +void remove_cleanup_paths_quietly(const std::vector& cleanupPaths) { + for (const std::filesystem::path& cleanupPath : cleanupPaths) { + std::error_code cleanupError; + std::filesystem::remove_all(cleanupPath, cleanupError); + } +} + +std::optional materialize_extracted_database( + const std::filesystem::path& sourceDirectory, + const std::filesystem::path& destinationDirectory +) { + const std::filesystem::path metadataPath = sourceDirectory / "metadata.json"; + const std::filesystem::path dbPath = sourceDirectory / "trivy.db"; + std::error_code error; + if (!std::filesystem::is_regular_file(metadataPath, error) || error) { + return std::nullopt; + } + error.clear(); + if (!std::filesystem::is_regular_file(dbPath, error) || error) { + return std::nullopt; + } + + std::filesystem::remove_all(destinationDirectory, error); + error.clear(); + if (!ensure_directory(destinationDirectory)) { + return std::nullopt; + } + if (!copy_file_replace(metadataPath, destinationDirectory / "metadata.json")) { + return std::nullopt; + } + if (!copy_file_replace(dbPath, destinationDirectory / "trivy.db")) { + return std::nullopt; + } + return destinationDirectory; +} + +std::optional materialize_extracted_archive( + const std::filesystem::path& archivePath, + const std::filesystem::path& destinationDirectory +) { + try { + const ArchiveResolution extracted = extract_archive_to_temp_directory(archivePath); + const auto materialized = materialize_extracted_database(extracted.installPath, destinationDirectory); + remove_cleanup_paths_quietly(extracted.cleanupPaths); + return materialized; + } catch (...) { + return std::nullopt; + } +} + +std::optional prepare_local_fixture_database( + const std::string& repository, + const std::filesystem::path& cacheRoot +) { + const std::filesystem::path localPath(repository.substr(std::string{"file://"}.size())); + const std::filesystem::path destinationDirectory = cacheRoot / "input"; + std::error_code error; + error.clear(); + if (std::filesystem::is_directory(localPath, error) && !error) { + if (const auto extracted = materialize_extracted_database(localPath, destinationDirectory); extracted.has_value()) { + return extracted; + } + error.clear(); + const std::filesystem::path archivePath = localPath / "db.tar.gz"; + if (std::filesystem::is_regular_file(archivePath, error) && !error) { + return materialize_extracted_archive(archivePath, destinationDirectory); + } + return std::nullopt; + } + + error.clear(); + if (std::filesystem::is_regular_file(localPath, error) && !error) { + return materialize_extracted_archive(localPath, destinationDirectory); + } + return std::nullopt; +} + +std::optional download_trivy_database_from_oci( + const ReqPackConfig& config, + const std::string& repository, + const std::filesystem::path& cacheRoot +) { + const auto reference = parse_oci_reference(repository); + if (!reference.has_value()) { + return std::nullopt; + } + + std::optional token; + const auto manifest = load_manifest_with_auth(config, reference.value(), token); + if (!manifest.has_value()) { + return std::nullopt; + } + const auto digest = select_db_layer_digest(manifest.value()); + if (!digest.has_value()) { + return std::nullopt; + } + + if (!ensure_directory(cacheRoot)) { + return std::nullopt; + } + const std::filesystem::path archivePath = cacheRoot / "db.tar.gz"; + long statusCode = 0; + if (!http_download_to_path(config, blob_url(reference.value(), digest.value()), blob_headers(token), archivePath, statusCode)) { + return std::nullopt; + } + return materialize_extracted_archive(archivePath, cacheRoot / "input"); +} + +std::optional prepare_trivy_database( + const ReqPackConfig& config, + const SecurityBackendConfig& backend, + std::string& selectedRepository +) { + const std::filesystem::path cacheRoot = std::filesystem::path(config.security.cachePath) / "trivy"; + for (const std::string& repository : backend.dbRepositories) { + std::optional prepared; + if (starts_with(repository, "file://")) { + prepared = prepare_local_fixture_database(repository, cacheRoot); + } else { + prepared = download_trivy_database_from_oci(config, repository, cacheRoot); + } + if (prepared.has_value()) { + selectedRepository = repository; + return prepared; + } + } + return std::nullopt; +} + +std::optional load_trivy_metadata(const std::filesystem::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input.is_open()) { + return std::nullopt; + } + std::ostringstream buffer; + buffer << input.rdbuf(); + ptree tree; + if (!parse_json_tree(buffer.str(), tree)) { + return std::nullopt; + } + const long version = tree.get("Version", 0L); + return TrivyDbMetadata{.version = version > 0 ? std::to_string(version) : std::string{}}; +} + +ProcessResult run_process_capture(const std::vector& arguments) { + const ReqpackProcessResult captured = reqpack_run_process_capture(arguments); + return ProcessResult{ + .exitCode = captured.exitCode, + .stdoutText = captured.stdoutText, + .stderrText = captured.stderrText, + }; +} + +std::filesystem::path resolve_helper_path(const SecurityBackendConfig& backend) { + if (!backend.helperPath.empty()) { + return backend.helperPath; + } + if (const char* overridePath = std::getenv("REQPACK_TRIVY_HELPER"); overridePath != nullptr && overridePath[0] != '\0') { + return std::filesystem::path(overridePath); + } + if (std::string_view(REQPACK_TRIVY_HELPER_DEFAULT_PATH).empty()) { + return {}; + } + return std::filesystem::path(REQPACK_TRIVY_HELPER_DEFAULT_PATH); +} + +bool scope_matches(const VulnerabilityDatabase* database, const std::set& ecosystems) { + if (database == nullptr) { + return false; + } + return database->getSyncState(TRIVY_SYNC_KEY_SCOPE).value_or(std::string{}) == join_scope_key(ecosystems); +} + +bool should_refresh(const VulnerabilityDatabase* database, const SecurityBackendConfig& backend, bool forceRefresh) { + if (forceRefresh || database == nullptr) { + return true; + } + if (backend.refreshMode == OsvRefreshMode::ALWAYS) { + return true; + } + if (backend.refreshMode == OsvRefreshMode::MANUAL) { + return !database->hasAdvisories(); + } + + const auto lastSync = database->getSyncState(TRIVY_SYNC_KEY_LAST_SUCCESS); + if (!lastSync.has_value()) { + return true; + } + try { + const long last = std::stol(lastSync.value()); + const long now = std::stol(now_epoch_seconds()); + return (now - last) >= backend.refreshIntervalSeconds; + } catch (...) { + return true; + } +} + +bool append_unique_range(std::vector& ranges, const TrivyHelperRange& candidate) { + const auto duplicate = std::find_if(ranges.begin(), ranges.end(), [&](const TrivyHelperRange& existing) { + return existing.introduced == candidate.introduced && + existing.fixed == candidate.fixed && + existing.lastAffected == candidate.lastAffected && + existing.limit == candidate.limit; + }); + if (duplicate != ranges.end()) { + return false; + } + ranges.push_back(candidate); + return true; +} + +std::optional build_indexed_advisory(const TrivyGroupedAdvisory& grouped, const std::string& ecosystem) { + std::ostringstream raw; + raw << '{'; + raw << "\"id\":\"" << json_escape(grouped.id) << "\","; + if (!grouped.modified.empty()) { + raw << "\"modified\":\"" << json_escape(grouped.modified) << "\","; + } + if (!grouped.published.empty()) { + raw << "\"published\":\"" << json_escape(grouped.published) << "\","; + } + raw << "\"summary\":\"" << json_escape(grouped.summary.empty() ? grouped.id : grouped.summary) << "\","; + raw << "\"details\":\"" << json_escape(grouped.description.empty() ? std::string{"Imported from Trivy vulnerability database"} : grouped.description) << "\","; + if (!grouped.aliases.empty()) { + raw << "\"aliases\":["; + bool firstAlias = true; + for (const std::string& alias : grouped.aliases) { + if (!firstAlias) { + raw << ','; + } + firstAlias = false; + raw << "\"" << json_escape(alias) << "\""; + } + raw << "],"; + } + if (!grouped.references.empty()) { + raw << "\"references\":["; + bool firstReference = true; + for (const std::string& reference : grouped.references) { + if (!firstReference) { + raw << ','; + } + firstReference = false; + raw << "{\"type\":\"WEB\",\"url\":\"" << json_escape(reference) << "\"}"; + } + raw << "],"; + } + if (grouped.score > 0.0) { + raw << "\"severity\":[{\"type\":\"CVSS_V3\",\"score\":\"" << grouped.score << "\"}],"; + } + raw << "\"database_specific\":{\"severity\":\"" << json_escape(grouped.severity) << "\"},"; + raw << "\"affected\":["; + + bool firstPackage = true; + for (const auto& [packageName, package] : grouped.packages) { + if (!firstPackage) { + raw << ','; + } + firstPackage = false; + raw << '{'; + raw << "\"package\":{\"ecosystem\":\"" << json_escape(ecosystem) << "\",\"name\":\"" << json_escape(packageName) << "\"},"; + raw << "\"database_specific\":{\"severity\":\"" << json_escape(grouped.severity) << "\"}"; + if (grouped.score > 0.0) { + raw << ",\"severity\":[{\"type\":\"CVSS_V3\",\"score\":\"" << grouped.score << "\"}]"; + } + if (!package.versions.empty()) { + raw << ",\"versions\":["; + bool firstVersion = true; + for (const std::string& version : package.versions) { + if (!firstVersion) { + raw << ','; + } + firstVersion = false; + raw << "\"" << json_escape(version) << "\""; + } + raw << ']'; + } + if (!package.ranges.empty()) { + raw << ",\"ranges\":[{\"type\":\"ECOSYSTEM\",\"events\":["; + bool firstEvent = true; + for (const TrivyHelperRange& range : package.ranges) { + if (!firstEvent) { + raw << ','; + } + firstEvent = false; + raw << '{'; + bool hasField = false; + if (!range.introduced.empty()) { + raw << "\"introduced\":\"" << json_escape(range.introduced) << "\""; + hasField = true; + } + if (!range.fixed.empty()) { + if (hasField) raw << ','; + raw << "\"fixed\":\"" << json_escape(range.fixed) << "\""; + hasField = true; + } + if (!range.lastAffected.empty()) { + if (hasField) raw << ','; + raw << "\"last_affected\":\"" << json_escape(range.lastAffected) << "\""; + hasField = true; + } + if (!range.limit.empty()) { + if (hasField) raw << ','; + raw << "\"limit\":\"" << json_escape(range.limit) << "\""; + } + raw << '}'; + } + raw << "]}]"; + } + raw << '}'; + } + raw << "]}"; + + const auto parsed = osv_parse_advisory(raw.str()); + if (!parsed.has_value()) { + return std::nullopt; + } + + std::vector> packageKeys; + for (const OsvAffectedPackage& package : parsed->affected) { + packageKeys.emplace_back(package.ecosystem, osv_normalize_package_name(package.name)); + } + return VulnerabilityDatabase::IndexedAdvisory{ + .advisory = parsed.value(), + .packageKeys = packageKeys, + }; +} + +std::optional> load_helper_records(const std::string& output) { + std::vector records; + std::istringstream stream(output); + std::string line; + while (std::getline(stream, line)) { + if (trim_copy(line).empty()) { + continue; + } + const auto record = trivy_parse_helper_record(line); + if (!record.has_value()) { + return std::nullopt; + } + records.push_back(record.value()); + } + return records; +} + +std::vector import_records( + VulnerabilityDatabase* database, + const std::set& ecosystems, + const std::vector& records, + const std::string& sourceRepository, + const TrivyDbMetadata& metadata +) { + if (database == nullptr || ecosystems.empty()) { + return {}; + } + + std::vector findings; + std::map grouped; + + for (const TrivyHelperRecord& record : records) { + if (!ecosystems.contains(record.ecosystem)) { + continue; + } + TrivyGroupedAdvisory& advisory = grouped[record.ecosystem + "\n" + record.advisoryId]; + advisory.ecosystem = record.ecosystem; + advisory.id = record.advisoryId; + advisory.summary = advisory.summary.empty() ? record.summary : advisory.summary; + advisory.description = advisory.description.empty() ? record.description : advisory.description; + advisory.modified = std::max(advisory.modified, record.modified); + advisory.published = std::max(advisory.published, record.published); + if (record.score > advisory.score) { + advisory.score = record.score; + } + if (severity_rank(record.severity) > severity_rank(advisory.severity)) { + advisory.severity = record.severity; + } + advisory.aliases.insert(record.aliases.begin(), record.aliases.end()); + advisory.references.insert(record.references.begin(), record.references.end()); + + TrivyGroupedPackage& package = advisory.packages[record.packageName]; + package.versions.insert(record.versions.begin(), record.versions.end()); + for (const TrivyHelperRange& range : record.ranges) { + (void)append_unique_range(package.ranges, range); + } + } + + std::vector advisories; + advisories.reserve(grouped.size()); + for (const auto& [_, groupedAdvisory] : grouped) { + const auto advisory = build_indexed_advisory(groupedAdvisory, groupedAdvisory.ecosystem); + if (!advisory.has_value()) { + findings.push_back(ValidationFinding{ + .kind = "sync_warning", + .source = "trivy", + .severity = "low", + .message = "skipping Trivy advisory that could not be normalized", + }); + continue; + } + advisories.push_back(advisory.value()); + } + + if (!database->clearAll()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "trivy", + .severity = "high", + .message = "failed to reset local Trivy database", + }}; + } + if (!advisories.empty() && !database->upsertIndexedAdvisories(advisories, TRIVY_IMPORT_BATCH_SIZE)) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "trivy", + .severity = "high", + .message = "failed to import Trivy advisory batch", + }}; + } + + (void)database->setSyncState(TRIVY_SYNC_KEY_SCOPE, join_scope_key(ecosystems)); + (void)database->setSyncState(TRIVY_SYNC_KEY_LAST_SUCCESS, now_epoch_seconds()); + if (!sourceRepository.empty()) { + (void)database->setSyncState(TRIVY_SYNC_KEY_SOURCE_REPOSITORY, sourceRepository); + } + if (!metadata.version.empty()) { + (void)database->setSyncState(TRIVY_SYNC_KEY_METADATA_VERSION, metadata.version); + } + return findings; +} + +} // namespace + +std::optional trivy_parse_helper_record(const std::string& line) { + ptree tree; + if (!parse_json_tree(line, tree)) { + return std::nullopt; + } + + TrivyHelperRecord record; + record.ecosystem = trim_copy(tree.get("ecosystem", {})); + record.packageName = trim_copy(tree.get("packageName", {})); + record.advisoryId = trim_copy(tree.get("advisoryId", {})); + record.summary = trim_copy(tree.get("summary", {})); + record.description = trim_copy(tree.get("description", {})); + record.score = tree.get("score", 0.0); + record.severity = normalize_trivy_severity(tree.get("severity", {}), record.score); + record.modified = trim_copy(tree.get("modified", {})); + record.published = trim_copy(tree.get("published", {})); + + if (const auto aliases = tree.get_child_optional("aliases")) { + for (const auto& [_, item] : aliases.value()) { + record.aliases.push_back(trim_copy(item.get_value())); + } + } + if (const auto references = tree.get_child_optional("references")) { + for (const auto& [_, item] : references.value()) { + record.references.push_back(trim_copy(item.get_value())); + } + } + if (const auto versions = tree.get_child_optional("versions")) { + for (const auto& [_, item] : versions.value()) { + record.versions.push_back(trim_copy(item.get_value())); + } + } + if (const auto ranges = tree.get_child_optional("ranges")) { + for (const auto& [_, rangeNode] : ranges.value()) { + TrivyHelperRange range; + range.introduced = trim_copy(rangeNode.get("introduced", {})); + range.fixed = trim_copy(rangeNode.get("fixed", {})); + range.lastAffected = trim_copy(rangeNode.get("lastAffected", rangeNode.get("last_affected", {}))); + range.limit = trim_copy(rangeNode.get("limit", {})); + record.ranges.push_back(std::move(range)); + } + } + + if (record.ecosystem.empty() || record.packageName.empty() || record.advisoryId.empty()) { + return std::nullopt; + } + return record; +} + +TrivyVulnerabilitySyncService::TrivyVulnerabilitySyncService( + VulnerabilityDatabase* database, + std::set ecosystems, + const ReqPackConfig& config +) : config(config), database(database), ecosystems(std::move(ecosystems)) {} + +std::vector TrivyVulnerabilitySyncService::ensureReady(bool forceRefresh) const { + if (this->database == nullptr || !this->database->ensureReady()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "trivy", + .severity = "high", + .message = "vulnerability database unavailable", + }}; + } + + const auto backendIt = this->config.security.backends.find("trivy"); + if (backendIt == this->config.security.backends.end()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "trivy", + .severity = "high", + .message = "missing Trivy backend configuration", + }}; + } + const SecurityBackendConfig& backend = backendIt->second; + if (backend.dbRepositories.empty()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "trivy", + .severity = "high", + .message = "missing Trivy database repositories configuration", + }}; + } + + if (scope_matches(this->database, this->ecosystems) && !should_refresh(this->database, backend, forceRefresh)) { + return {}; + } + + std::string selectedRepository; + const auto extractedDirectory = prepare_trivy_database(this->config, backend, selectedRepository); + if (!extractedDirectory.has_value()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "trivy", + .severity = "high", + .message = "failed to download or extract Trivy database", + }}; + } + + const std::filesystem::path metadataPath = extractedDirectory.value() / "metadata.json"; + const std::filesystem::path dbPath = extractedDirectory.value() / "trivy.db"; + std::error_code error; + if (!std::filesystem::is_regular_file(metadataPath, error) || error) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "trivy", + .severity = "high", + .message = "Trivy metadata.json missing after extraction", + }}; + } + error.clear(); + if (!std::filesystem::is_regular_file(dbPath, error) || error) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "trivy", + .severity = "high", + .message = "Trivy trivy.db missing after extraction", + }}; + } + + const std::filesystem::path helperPath = resolve_helper_path(backend); + if (helperPath.empty() || !std::filesystem::exists(helperPath)) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "trivy", + .severity = "high", + .message = "Trivy helper binary not found", + }}; + } + + std::vector arguments{helperPath.string(), "--db-path", dbPath.string()}; + for (const std::string& ecosystem : this->ecosystems) { + arguments.push_back("--ecosystem"); + arguments.push_back(ecosystem); + } + + const ProcessResult helperResult = run_process_capture(arguments); + if (helperResult.exitCode != 0) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "trivy", + .severity = "high", + .message = "Trivy helper execution failed" + (helperResult.stderrText.empty() ? std::string{} : ": " + trim_copy(helperResult.stderrText)), + }}; + } + + const auto records = load_helper_records(helperResult.stdoutText); + if (!records.has_value()) { + return {ValidationFinding{ + .kind = "sync_error", + .source = "trivy", + .severity = "high", + .message = "failed to parse Trivy helper output", + }}; + } + + const TrivyDbMetadata metadata = load_trivy_metadata(metadataPath).value_or(TrivyDbMetadata{}); + return import_records(this->database, this->ecosystems, records.value(), selectedRepository, metadata); +} diff --git a/src/main/cpp/core/security/validator_prompt.cpp b/src/main/cpp/core/security/validator_prompt.cpp index 302e9bd..0fbf250 100644 --- a/src/main/cpp/core/security/validator_prompt.cpp +++ b/src/main/cpp/core/security/validator_prompt.cpp @@ -53,7 +53,7 @@ std::vector disposition_findings(const Graph& graph, const st bool Validator::requestUserDecision(const std::vector& findings) const { Logger& logger = Logger::instance(); - logger.stdout("unsafe findings require confirmation:"); + logger.logStdout("unsafe findings require confirmation:"); std::size_t shown = 0; for (const ValidationFinding& finding : findings) { std::string message = finding.message.empty() ? finding.id : finding.message; @@ -63,7 +63,7 @@ bool Validator::requestUserDecision(const std::vector& findin if (!finding.package.system.empty() && !finding.package.name.empty()) { message += " [" + finding.package.system + ":" + finding.package.name + "]"; } - logger.stdout("- " + message); + logger.logStdout("- " + message); ++shown; if (shown >= 5) { break; @@ -81,11 +81,11 @@ bool Validator::requestUserDecision(const std::vector& findin return false; } - logger.stdout("Continue? [y/N]"); + logger.logStdout("Continue? [y/N]"); logger.flushSync(); std::string answer; if (!std::getline(std::cin, answer)) { - logger.stdout("aborted."); + logger.logStdout("aborted."); logger.flushSync(); return false; } @@ -95,7 +95,7 @@ bool Validator::requestUserDecision(const std::vector& findin return true; } - logger.stdout("aborted."); + logger.logStdout("aborted."); logger.flushSync(); return false; } diff --git a/src/main/cpp/main.cpp b/src/main/cpp/main.cpp index ef8facc..3fc46d5 100644 --- a/src/main/cpp/main.cpp +++ b/src/main/cpp/main.cpp @@ -41,13 +41,13 @@ void configure_logger_from_config(Logger& logger, const ReqPackConfig& config) { } // namespace int main(int argc, char* argv[]) { - std::vector earlyArguments; - earlyArguments.reserve(static_cast(argc > 1 ? argc - 1 : 0)); + std::vector rawArguments; + rawArguments.reserve(static_cast(argc > 1 ? argc - 1 : 0)); for (int i = 1; i < argc; ++i) { - earlyArguments.emplace_back(argv[i]); + rawArguments.emplace_back(argv[i]); } - const PluginTestCliParseResult earlyPluginTest = parse_plugin_test_invocation(earlyArguments); + const PluginTestCliParseResult earlyPluginTest = parse_plugin_test_invocation(rawArguments); if (earlyPluginTest.matched && earlyPluginTest.helpRequested) { print_plugin_test_help(std::cout); return 0; @@ -90,14 +90,15 @@ int main(int argc, char* argv[]) { Logger& logger = Logger::instance(); configure_logger_from_config(logger, config); + if (config.display.jsonOutput) { + logger.setJsonOutputMode(true); + logger.setConsoleOutput(false); + } - std::unique_ptr display = create_display(config.display); - logger.setDisplay(display.get()); - - std::vector rawArguments; - rawArguments.reserve(static_cast(argc > 1 ? argc - 1 : 0)); - for (int i = 1; i < argc; ++i) { - rawArguments.emplace_back(argv[i]); + std::unique_ptr display; + if (!config.display.jsonOutput) { + display = create_display(config.display); + logger.setDisplay(display.get()); } const int result = dispatch_main_command( diff --git a/src/main/cpp/main_arg_parsing.cpp b/src/main/cpp/main_arg_parsing.cpp index a61038e..42db429 100644 --- a/src/main/cpp/main_arg_parsing.cpp +++ b/src/main/cpp/main_arg_parsing.cpp @@ -137,7 +137,7 @@ bool parse_serve_runtime_options(const std::vector& arguments, for (std::size_t i = 1; i < filtered.size(); ++i) { const std::string& argument = filtered[i]; if (argument == "--stdin") { - options.stdin = true; + options.useStdin = true; continue; } if (argument == "--remote") { @@ -202,7 +202,12 @@ bool parse_serve_runtime_options(const std::vector& arguments, options.inheritedArguments.push_back(argument); } - if (options.stdin == options.remote) { + // --json is also a global output flag and may be stripped before this loop. + if (options.remote && options.remoteProtocol == ServeRemoteProtocol::TEXT && contains_flag(arguments, "--json")) { + options.remoteProtocol = ServeRemoteProtocol::JSON; + } + + if (options.useStdin == options.remote) { error = "serve requires exactly one of --stdin or --remote"; return true; } @@ -214,7 +219,7 @@ bool parse_serve_runtime_options(const std::vector& arguments, error = "--max-connections must be greater than 0"; return true; } - if (options.stdin) { + if (options.useStdin) { if (options.token.has_value() || options.username.has_value() || options.password.has_value() || options.bind != "127.0.0.1" || options.port != 4545 || options.remoteProtocol != ServeRemoteProtocol::TEXT || options.maxConnections != 16) { diff --git a/src/main/cpp/main_dispatch.cpp b/src/main/cpp/main_dispatch.cpp index 38579e2..54369c2 100644 --- a/src/main/cpp/main_dispatch.cpp +++ b/src/main/cpp/main_dispatch.cpp @@ -27,10 +27,10 @@ namespace { int run_host_refresh(Logger& logger) { const std::shared_ptr snapshot = HostInfoService::refreshSnapshot(); - logger.stdout("host refresh: cache updated"); - logger.stdout("host os: " + snapshot->platform.osFamily); - logger.stdout("host arch: " + snapshot->platform.arch); - logger.stdout("host cache: " + default_reqpack_host_info_cache_path().string()); + logger.logStdout("host refresh: cache updated"); + logger.logStdout("host os: " + snapshot->platform.osFamily); + logger.logStdout("host arch: " + snapshot->platform.arch); + logger.logStdout("host cache: " + default_reqpack_host_info_cache_path().string()); return 0; } @@ -85,7 +85,7 @@ int dispatch_main_command(Cli& cli, if (!serveOptions.maxConnectionsExplicit) { serveOptions.maxConnections = config.remote.maxConnections; } - if (serveOptions.stdin) { + if (serveOptions.useStdin) { return run_stdin_serve_loop(cli, config, serveOptions.inheritedArguments); } return run_remote_serve(cli, config, configPath, configOverrides, logger, display, serveOptions); @@ -155,7 +155,7 @@ int dispatch_main_command(Cli& cli, } if (is_version_command(rawArguments)) { - logger.stdout(config.applicationName + " " + reqpack_build_release_id()); + logger.logStdout(config.applicationName + " " + reqpack_build_release_id()); logger.flushSync(); return 0; } diff --git a/src/main/cpp/main_self_update_release.cpp b/src/main/cpp/main_self_update_release.cpp index c70eea3..fe7b396 100644 --- a/src/main/cpp/main_self_update_release.cpp +++ b/src/main/cpp/main_self_update_release.cpp @@ -1,6 +1,7 @@ #include "main_self_update_internal.h" #include "core/common/network_environment.h" +#include "core/common/process_runner.h" #include "core/registry/registry_database.h" #include "output/logger.h" @@ -10,24 +11,29 @@ #include #include +#include #include -#include #include #include #include #include -#include #include -#include #include #include #include -#include #include #include +#if !defined(_WIN32) +#include +#include +#include +#include #include +#include "core/common/pipe_helpers.h" +#endif + namespace { constexpr const char* DEFAULT_SELF_UPDATE_RELEASE_API_BASE_URL = "https://api.github.com"; @@ -354,12 +360,49 @@ bool extract_release_archive(const std::filesystem::path& archivePath, return run_process({"tar", "-xzf", archivePath.string(), "-C", destinationPath.string()}, std::filesystem::current_path()); } +#if defined(_WIN32) + const std::vector arguments{ + "tar", + "-xzvf", + archivePath.string(), + "-C", + destinationPath.string(), + }; + const ReqpackProcessResult captured = reqpack_run_process_capture(arguments, std::filesystem::current_path()); + if (!captured.success()) { + return false; + } + + std::size_t extractedEntries = 0; + std::set extractedEntryNames; + const auto record_lines = [&](const std::string& text) { + std::istringstream stream(text); + std::string line; + while (std::getline(stream, line)) { + if (const std::optional entry = detect_archive_entry_from_tar_output(line, expectedEntries); entry.has_value()) { + if (extractedEntryNames.insert(entry.value()).second) { + extractedEntries = extractedEntryNames.size(); + onEntryExtracted( + std::min(extractedEntries, totalEntries == 0 ? extractedEntries : totalEntries), + totalEntries == 0 ? extractedEntries : totalEntries + ); + } + } + } + }; + record_lines(captured.stdoutText); + record_lines(captured.stderrText); + if (totalEntries > 0 && extractedEntries < totalEntries) { + onEntryExtracted(totalEntries, totalEntries); + } + return captured.success(); +#else int stdoutPipe[2]; int stderrPipe[2]; - if (::pipe(stdoutPipe) != 0) { + if (!create_pipe_cloexec(stdoutPipe)) { return false; } - if (::pipe(stderrPipe) != 0) { + if (!create_pipe_cloexec(stderrPipe)) { (void)::close(stdoutPipe[0]); (void)::close(stdoutPipe[1]); return false; @@ -482,10 +525,15 @@ bool extract_release_archive(const std::filesystem::path& archivePath, {.fd = stderrPipe[0], .events = POLLIN}, }; + constexpr int POLL_TIMEOUT_MS = 30000; bool stdoutOpen = true; bool stderrOpen = true; while (stdoutOpen || stderrOpen) { - const int pollResult = ::poll(fds, 2, -1); + const int pollResult = ::poll(fds, 2, POLL_TIMEOUT_MS); + if (pollResult == 0) { + ::kill(pid, SIGTERM); + break; + } if (pollResult < 0) { if (errno == EINTR) { continue; @@ -534,6 +582,7 @@ bool extract_release_archive(const std::filesystem::path& archivePath, onEntryExtracted(totalEntries, totalEntries); } return WIFEXITED(status) && WEXITSTATUS(status) == 0; +#endif } std::optional locate_extracted_binary(const std::filesystem::path& directory) { diff --git a/src/main/cpp/main_self_update_support.cpp b/src/main/cpp/main_self_update_support.cpp index 081c662..781e7e2 100644 --- a/src/main/cpp/main_self_update_support.cpp +++ b/src/main/cpp/main_self_update_support.cpp @@ -1,20 +1,17 @@ #include "main_self_update_internal.h" -#include "core/common/network_environment.h" +#include "core/common/process_runner.h" +#include "core/common/temp_directory.h" #include #include #include #include #include -#include #include -#include #include #include -#include - namespace self_update_internal { std::string trim_copy(const std::string& value) { @@ -35,56 +32,7 @@ std::optional trim_line(const std::string& value) { } bool run_process(const std::vector& arguments, const std::filesystem::path& workingDirectory) { - if (arguments.empty()) { - return false; - } - - posix_spawn_file_actions_t fileActions; - if (posix_spawn_file_actions_init(&fileActions) != 0) { - return false; - } - - bool ready = true; -#if defined(__linux__) || defined(__APPLE__) - if (!workingDirectory.empty()) { - ready = posix_spawn_file_actions_addchdir_np(&fileActions, workingDirectory.c_str()) == 0; - } -#endif - if (!ready) { - posix_spawn_file_actions_destroy(&fileActions); - return false; - } - - std::vector argv; - argv.reserve(arguments.size() + 1); - for (const std::string& argument : arguments) { - argv.push_back(const_cast(argument.c_str())); - } - argv.push_back(nullptr); - - std::vector environmentStorage = reqpack_sanitized_process_environment(); - std::vector environmentPointers; - environmentPointers.reserve(environmentStorage.size() + 1); - for (std::string& entry : environmentStorage) { - environmentPointers.push_back(entry.data()); - } - environmentPointers.push_back(nullptr); - - pid_t pid = 0; - const int spawnResult = posix_spawnp(&pid, arguments.front().c_str(), &fileActions, nullptr, argv.data(), environmentPointers.data()); - posix_spawn_file_actions_destroy(&fileActions); - if (spawnResult != 0) { - return false; - } - - int status = 0; - while (waitpid(pid, &status, 0) == -1) { - if (errno != EINTR) { - return false; - } - } - - return WIFEXITED(status) && WEXITSTATUS(status) == 0; + return reqpack_run_process(arguments, workingDirectory); } bool ensure_directory(const std::filesystem::path& path) { @@ -224,18 +172,11 @@ std::optional create_self_update_temp_directory() { return std::nullopt; } - for (int attempt = 0; attempt < 100; ++attempt) { - const std::filesystem::path candidate = base / ("session-" + std::to_string(::getpid()) + "-" + std::to_string(std::rand())); - error.clear(); - if (std::filesystem::create_directory(candidate, error)) { - return candidate; - } - if (error && error != std::errc::file_exists) { - return std::nullopt; - } + try { + return reqpack_make_unique_directory(base, "session"); + } catch (...) { + return std::nullopt; } - - return std::nullopt; } bool remove_path_quietly(const std::filesystem::path& path) { diff --git a/src/main/cpp/main_stdin.cpp b/src/main/cpp/main_stdin.cpp index 8e2e544..81ba7a6 100644 --- a/src/main/cpp/main_stdin.cpp +++ b/src/main/cpp/main_stdin.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,12 @@ namespace { +volatile sig_atomic_t g_stdin_shutdown_requested = 0; + +void stdin_shutdown_handler(int) { + g_stdin_shutdown_requested = 1; +} + struct StdinCommand { std::size_t lineNumber{0}; std::string text; @@ -144,6 +151,10 @@ int run_stdin_action_batch(Cli& cli, } ReqPackConfig effectiveConfig = apply_config_overrides(config, inheritedOverrides); + // Config accumulates across stdin lines: each line's overrides layer on top of the + // previous state. This is intentional — the batch runs as a single Orchestrator + // invocation with the final merged config. Scalar fields use last-writer-wins; + // list fields (enabledCategories, ignoreVulnerabilityIds) are replaced, not appended. for (const StdinCommand& command : commands) { const std::vector commandTokens = tokenize_command_line(command.text); if (commandTokens.empty()) { @@ -195,9 +206,21 @@ int run_stdin_serve_loop(Cli& cli, Logger& logger = Logger::instance(); int exitCode = 0; +#if defined(_WIN32) + std::signal(SIGTERM, stdin_shutdown_handler); + std::signal(SIGINT, stdin_shutdown_handler); +#else + struct sigaction sa{}; + sa.sa_handler = stdin_shutdown_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(SIGTERM, &sa, nullptr); + sigaction(SIGINT, &sa, nullptr); +#endif + std::string line; std::size_t lineNumber = 0; - while (std::getline(std::cin, line)) { + while (std::getline(std::cin, line) && !g_stdin_shutdown_requested) { ++lineNumber; const std::string trimmed = trim_copy(line); if (trimmed.empty() || trimmed.rfind("#", 0) == 0) { diff --git a/src/main/cpp/output/logger_emit.cpp b/src/main/cpp/output/logger_emit.cpp index a36d77d..7dfaafe 100644 --- a/src/main/cpp/output/logger_emit.cpp +++ b/src/main/cpp/output/logger_emit.cpp @@ -43,7 +43,7 @@ std::uint64_t Logger::emitDiagnostic(const DiagnosticMessage& diagnostic, bool m }); } -void Logger::stdout(const std::string& message, const std::string& source, const std::string& scope) { +void Logger::logStdout(const std::string& message, const std::string& source, const std::string& scope) { emit(OutputAction::STDOUT, OutputContext{ .level = spdlog::level::info, diff --git a/src/main/cpp/output/logger_processing.cpp b/src/main/cpp/output/logger_processing.cpp index ae8d285..5438d69 100644 --- a/src/main/cpp/output/logger_processing.cpp +++ b/src/main/cpp/output/logger_processing.cpp @@ -230,6 +230,9 @@ void Logger::processEvent(const OutputEvent& event) { } break; case OutputAction::STDOUT: + if (jsonOutputMode.load(std::memory_order_acquire)) { + break; + } if (activeDisplay != nullptr) { routeToDisplay(event); } else { @@ -244,6 +247,9 @@ void Logger::processEvent(const OutputEvent& event) { case OutputAction::PLUGIN_PROGRESS: case OutputAction::PLUGIN_EVENT: case OutputAction::PLUGIN_ARTIFACT: + if (jsonOutputMode.load(std::memory_order_acquire)) { + break; + } if (activeDisplay != nullptr) { routeToDisplay(event); } else { diff --git a/src/main/cpp/output/logger_settings.cpp b/src/main/cpp/output/logger_settings.cpp index f033a15..f225279 100644 --- a/src/main/cpp/output/logger_settings.cpp +++ b/src/main/cpp/output/logger_settings.cpp @@ -1,6 +1,8 @@ #include "output/logger.h" #include "output/logger_core.h" +#include "core/common/time_helpers.h" + #include #include #include @@ -59,7 +61,7 @@ std::string utc_timestamp_now() { const auto now = std::chrono::system_clock::now(); const std::time_t currentTime = std::chrono::system_clock::to_time_t(now); std::tm tm{}; - gmtime_r(¤tTime, &tm); + (void)reqpack_gmtime_utc(&tm, ¤tTime); std::ostringstream stream; stream << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ"); return stream.str(); @@ -151,6 +153,14 @@ bool Logger::isConsoleOutputEnabled() const { return consoleOutputEnabled.load(std::memory_order_acquire); } +void Logger::setJsonOutputMode(bool enable) { + jsonOutputMode.store(enable, std::memory_order_release); +} + +bool Logger::isJsonOutputMode() const { + return jsonOutputMode.load(std::memory_order_acquire); +} + void Logger::setLevel(const std::string& level) { logger->set_level(spdlog::level::from_str(level)); } diff --git a/src/main/cpp/output/plain_display.cpp b/src/main/cpp/output/plain_display.cpp index 004845e..c678f40 100644 --- a/src/main/cpp/output/plain_display.cpp +++ b/src/main/cpp/output/plain_display.cpp @@ -2,6 +2,8 @@ #include "plain_display_internal.h" +#include "core/common/terminal_width.h" + #include #include #include @@ -9,9 +11,6 @@ #include #include -#include -#include - namespace plain_display_internal { std::string join_items(const std::vector& values) { @@ -62,12 +61,7 @@ size_t terminal_width() { } } - winsize size{}; - if (isatty(STDOUT_FILENO) && ioctl(STDOUT_FILENO, TIOCGWINSZ, &size) == 0 && size.ws_col > 0) { - return size.ws_col; - } - - return 100; + return reqpack_terminal_column_count(100); } std::string truncate_middle(const std::string& text, size_t width) { diff --git a/src/main/cpp/output/run_result_json.cpp b/src/main/cpp/output/run_result_json.cpp new file mode 100644 index 0000000..9634cfe --- /dev/null +++ b/src/main/cpp/output/run_result_json.cpp @@ -0,0 +1,98 @@ +#include "output/run_result_json.h" + +#include "output/logger.h" + +#include +#include +#include +#include +#include + +namespace { + +std::string json_escape(const std::string& value) { + std::string escaped; + escaped.reserve(value.size()); + for (const char c : value) { + switch (c) { + case '\\': escaped += "\\\\"; break; + case '"': escaped += "\\\""; break; + case '\b': escaped += "\\b"; break; + case '\f': escaped += "\\f"; break; + case '\n': escaped += "\\n"; break; + case '\r': escaped += "\\r"; break; + case '\t': escaped += "\\t"; break; + default: + if (static_cast(c) < 0x20) { + char buffer[8]; + std::snprintf(buffer, sizeof(buffer), "\\u%04x", static_cast(c)); + escaped += buffer; + } else { + escaped.push_back(c); + } + break; + } + } + return escaped; +} + +} // namespace + +std::string run_result_status_for_action(ActionType action, bool success, bool dryRun) { + if (!success) { + return "failed"; + } + if (dryRun) { + return "planned"; + } + switch (action) { + case ActionType::REMOVE: + return "removed"; + case ActionType::UPDATE: + return "updated"; + case ActionType::ENSURE: + case ActionType::INSTALL: + default: + return "installed"; + } +} + +std::string serialize_run_result_json(const RunResultJsonDocument& document) { + std::map> bySystem; + for (const RunResultJsonItem& item : document.items) { + bySystem[item.system].push_back(&item); + } + + std::ostringstream stream; + stream << "{"; + stream << "\"ok\":" << (document.ok ? "true" : "false"); + stream << ",\"dryRun\":" << (document.dryRun ? "true" : "false"); + + for (const auto& [system, items] : bySystem) { + stream << ",\"" << json_escape(system) << "\":["; + for (std::size_t index = 0; index < items.size(); ++index) { + const RunResultJsonItem& item = *items[index]; + if (index > 0) { + stream << ","; + } + stream << "{"; + stream << "\"name\":\"" << json_escape(item.name) << "\""; + stream << ",\"version\":\"" << json_escape(item.version) << "\""; + stream << ",\"status\":\"" << json_escape(item.status) << "\""; + if (!item.errorMessage.empty()) { + stream << ",\"errorMessage\":\"" << json_escape(item.errorMessage) << "\""; + } + stream << "}"; + } + stream << "]"; + } + + stream << "}"; + return stream.str(); +} + +void emit_run_result_json(const RunResultJsonDocument& document) { + Logger::instance().flushSync(); + std::cout << serialize_run_result_json(document) << '\n'; + std::cout.flush(); +} diff --git a/src/main/cpp/plugins/exec_rules.cpp b/src/main/cpp/plugins/exec_rules.cpp index 7e945cd..fa0ff71 100644 --- a/src/main/cpp/plugins/exec_rules.cpp +++ b/src/main/cpp/plugins/exec_rules.cpp @@ -1,17 +1,29 @@ #include "plugins/exec_rules.h" +#include "core/common/pipe_helpers.h" + #include #include #include #include #include -#include +#include #include -#include #include #include #include +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#include #if defined(__APPLE__) #include #else @@ -20,6 +32,7 @@ #include #include #include +#endif #include "core/common/network_environment.h" #include "plugins/exec_rules_core.h" @@ -85,7 +98,7 @@ OutputContext plugin_output_context(const std::string& sourceId, const std::stri void log_exec_transcript_chunk(Logger& logger, const std::string& pluginId, const std::string& chunk, const bool mirrorToTerminal) { log_plugin_message(logger, spdlog::level::debug, pluginId, std::string("[exec] ") + chunk); if (mirrorToTerminal) { - logger.stdout(chunk, pluginId, "exec"); + logger.logStdout(chunk, pluginId, "exec"); } } @@ -145,6 +158,11 @@ spdlog::level::level_enum parse_log_level(const std::string& value) { } bool write_all(int fd, const std::string& value) { +#if defined(_WIN32) + (void)fd; + (void)value; + return false; +#else std::size_t written = 0; while (written < value.size()) { const ssize_t count = ::write(fd, value.data() + written, value.size() - written); @@ -157,6 +175,7 @@ bool write_all(int fd, const std::string& value) { written += static_cast(count); } return true; +#endif } void emit_log_action(Logger& logger, const std::string& pluginId, spdlog::level::level_enum level, const std::string& message) { @@ -336,6 +355,155 @@ void dispatch_evaluation_result( } } +#if defined(_WIN32) + +std::string resolve_windows_cmd_exe() { + char buffer[MAX_PATH]; + const DWORD length = GetEnvironmentVariableA("ComSpec", buffer, MAX_PATH); + if (length > 0 && length < MAX_PATH) { + return std::string(buffer, length); + } + return "C:\\Windows\\System32\\cmd.exe"; +} + +bool read_windows_pipe_to_result( + HANDLE readPipe, + ExecResult& result, + const std::function& onChunk +) { + std::array buffer{}; + for (;;) { + DWORD count = 0; + if (!ReadFile(readPipe, buffer.data(), static_cast(buffer.size()), &count, nullptr)) { + const DWORD error = GetLastError(); + if (error == ERROR_BROKEN_PIPE) { + return true; + } + result.stderrText = "read failed: Windows error " + std::to_string(error); + return false; + } + if (count == 0) { + return true; + } + const std::string chunk(buffer.data(), static_cast(count)); + result.stdoutText += chunk; + onChunk(chunk); + } +} + +ExecResult run_shell_command( + Logger& logger, + const std::string& pluginScope, + const std::string& command, + const std::function& onChunk, + const bool silent +) { + ExecResult result; + + SECURITY_ATTRIBUTES securityAttributes{}; + securityAttributes.nLength = sizeof(securityAttributes); + securityAttributes.bInheritHandle = TRUE; + + HANDLE readPipe = nullptr; + HANDLE writePipe = nullptr; + if (!CreatePipe(&readPipe, &writePipe, &securityAttributes, 0)) { + result.stderrText = "failed to create pipe: Windows error " + std::to_string(GetLastError()); + return result; + } + if (!SetHandleInformation(readPipe, HANDLE_FLAG_INHERIT, 0)) { + CloseHandle(readPipe); + CloseHandle(writePipe); + result.stderrText = "failed to configure pipe inheritance: Windows error " + std::to_string(GetLastError()); + return result; + } + + HANDLE nulInput = CreateFileA( + "NUL", + GENERIC_READ, + FILE_SHARE_READ, + &securityAttributes, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + nullptr + ); + if (nulInput == INVALID_HANDLE_VALUE) { + CloseHandle(readPipe); + CloseHandle(writePipe); + result.stderrText = "failed to open NUL: Windows error " + std::to_string(GetLastError()); + return result; + } + + STARTUPINFOA startupInfo{}; + startupInfo.cb = sizeof(startupInfo); + startupInfo.dwFlags = STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW; + startupInfo.wShowWindow = SW_HIDE; + startupInfo.hStdInput = nulInput; + startupInfo.hStdOutput = writePipe; + startupInfo.hStdError = writePipe; + + PROCESS_INFORMATION processInfo{}; + const std::string cmdExe = resolve_windows_cmd_exe(); + std::string commandLine = "\"" + cmdExe + "\" /c " + command; + + const BOOL created = CreateProcessA( + cmdExe.c_str(), + commandLine.data(), + nullptr, + nullptr, + TRUE, + CREATE_NO_WINDOW, + nullptr, + nullptr, + &startupInfo, + &processInfo + ); + + CloseHandle(writePipe); + CloseHandle(nulInput); + + if (!created) { + CloseHandle(readPipe); + result.stderrText = "spawn failed: Windows error " + std::to_string(GetLastError()); + return result; + } + + CloseHandle(processInfo.hThread); + + const auto consumeChunk = [&](const std::string& chunk) { + log_exec_transcript_chunk(logger, pluginScope, chunk, !silent && logger.isConsoleOutputEnabled()); + onChunk(chunk); + }; + (void)read_windows_pipe_to_result(readPipe, result, consumeChunk); + CloseHandle(readPipe); + + if (WaitForSingleObject(processInfo.hProcess, INFINITE) != WAIT_OBJECT_0) { + CloseHandle(processInfo.hProcess); + result.stderrText = "wait failed: Windows error " + std::to_string(GetLastError()); + result.exitCode = 1; + result.success = false; + return result; + } + + DWORD exitCode = 1; + if (!GetExitCodeProcess(processInfo.hProcess, &exitCode)) { + CloseHandle(processInfo.hProcess); + result.stderrText = "failed to read exit code: Windows error " + std::to_string(GetLastError()); + result.exitCode = 1; + result.success = false; + return result; + } + CloseHandle(processInfo.hProcess); + + result.exitCode = static_cast(exitCode); + result.success = result.exitCode == 0; + if (!result.success && result.stderrText.empty()) { + result.stderrText = result.stdoutText; + } + return result; +} + +#else + bool read_fd_to_result(Logger& logger, const std::string& pluginId, int fd, ExecResult& result, const std::function& onChunk) { std::array buffer{}; for (;;) { @@ -361,7 +529,7 @@ ExecResult run_shell_command(Logger& logger, const std::string& pluginScope, con ExecResult result; int pipeFds[2] = {-1, -1}; - if (::pipe(pipeFds) != 0) { + if (!create_pipe_cloexec(pipeFds)) { result.stderrText = std::string("failed to create pipe: ") + std::strerror(errno); return result; } @@ -439,6 +607,8 @@ ExecResult run_shell_command(Logger& logger, const std::string& pluginScope, con return result; } +#endif + ExecResult run_plain_command(Logger& logger, const std::string& pluginScope, const std::string& command, const bool silent) { return run_shell_command(logger, pluginScope, command, [](const std::string&) {}, silent); } @@ -461,6 +631,10 @@ ExecResult run_line_command(Logger& logger, const std::string& sourceId, const s } ExecResult run_pty_command(Logger& logger, const std::string& sourceId, const std::string& pluginScope, const std::string& command, const ExecRuleset& ruleset, const bool silent) { +#if defined(_WIN32) + // No ConPTY integration yet: fall back to cmd.exe line capture. + return run_line_command(logger, sourceId, pluginScope, command, ruleset, silent); +#else ExecResult result; int masterFd = -1; @@ -540,6 +714,7 @@ ExecResult run_pty_command(Logger& logger, const std::string& sourceId, const st result.stderrText = result.stdoutText; } return result; +#endif } } // namespace diff --git a/src/main/cpp/plugins/lua_bridge.cpp b/src/main/cpp/plugins/lua_bridge.cpp index e9d0653..f0d815f 100644 --- a/src/main/cpp/plugins/lua_bridge.cpp +++ b/src/main/cpp/plugins/lua_bridge.cpp @@ -41,7 +41,7 @@ LuaBridge::LuaBridge(const std::string& scriptPath, const ReqPackConfig& config) message += LuaBridgeValueMapper::valueToString(argument); } - m_logger.stdout(message, "lua", m_pluginId); + m_logger.logStdout(message, "lua", m_pluginId); }; m_bindings.registerReqpackNamespace(); diff --git a/src/main/cpp/plugins/lua_bridge_bindings.cpp b/src/main/cpp/plugins/lua_bridge_bindings.cpp index ea2eb59..3693783 100644 --- a/src/main/cpp/plugins/lua_bridge_bindings.cpp +++ b/src/main/cpp/plugins/lua_bridge_bindings.cpp @@ -417,7 +417,7 @@ void LuaBridgeBindings::registerContextTypes() { binding->sourceId, command, rules, - m_hostRuntime.hasSilentRuntimeFlag(binding->flags) + m_hostRuntime.shouldUseSilentRuntime(binding->flags) ); } )); diff --git a/src/main/cpp/plugins/lua_bridge_host_runtime.cpp b/src/main/cpp/plugins/lua_bridge_host_runtime.cpp index 8bf29c0..c71e29d 100644 --- a/src/main/cpp/plugins/lua_bridge_host_runtime.cpp +++ b/src/main/cpp/plugins/lua_bridge_host_runtime.cpp @@ -1,13 +1,13 @@ #include "plugins/lua_bridge_host_runtime.h" +#include "core/common/temp_directory.h" + #include #include #include #include #include -#include - namespace { constexpr const char* SILENT_RUNTIME_FLAG = "__reqpack-internal-silent-runtime"; @@ -45,6 +45,13 @@ bool LuaBridgeHostRuntime::hasSilentRuntimeFlag(const std::vector& return std::find(flags.begin(), flags.end(), SILENT_RUNTIME_FLAG) != flags.end(); } +bool LuaBridgeHostRuntime::shouldUseSilentRuntime(const std::vector& flags) const { + if (m_config.logging.consoleOutput) { + return hasSilentRuntimeFlag(flags); + } + return true; +} + void LuaBridgeHostRuntime::setSilentRuntimeOutput(const bool silent) { m_silentRuntimeOutput.store(silent); } @@ -112,17 +119,15 @@ void LuaBridgeHostRuntime::cleanupAfterShutdown() { } std::string LuaBridgeHostRuntime::createTempDirectory(const std::string& pluginId) { - std::filesystem::path tempDir = std::filesystem::temp_directory_path() / ("reqpack-" + pluginId + "-XXXXXX"); - std::string templateString = tempDir.string(); - std::vector buffer(templateString.begin(), templateString.end()); - buffer.push_back('\0'); - char* created = ::mkdtemp(buffer.data()); - if (created == nullptr) { + try { + const std::filesystem::path created = + reqpack_make_unique_directory(std::filesystem::temp_directory_path(), "reqpack-" + pluginId); + m_tempDirectories.emplace_back(created.string()); + if (!m_runtimeWriteRoots.empty()) { + m_runtimeWriteRoots.emplace_back(created.lexically_normal()); + } + return created.string(); + } catch (...) { return {}; } - m_tempDirectories.emplace_back(created); - if (!m_runtimeWriteRoots.empty()) { - m_runtimeWriteRoots.emplace_back(std::filesystem::path(created).lexically_normal()); - } - return created; } diff --git a/src/main/cpp/plugins/lua_bridge_plugin_api.cpp b/src/main/cpp/plugins/lua_bridge_plugin_api.cpp index a8bdc56..a878861 100644 --- a/src/main/cpp/plugins/lua_bridge_plugin_api.cpp +++ b/src/main/cpp/plugins/lua_bridge_plugin_api.cpp @@ -183,7 +183,7 @@ std::vector LuaBridge::list(const PluginCallContext& context) { return {}; } - const bool silentRuntime = m_hostRuntime.hasSilentRuntimeFlag(context.flags); + const bool silentRuntime = m_hostRuntime.shouldUseSilentRuntime(context.flags); m_hostRuntime.setSilentRuntimeOutput(silentRuntime); auto result = func(context); m_hostRuntime.setSilentRuntimeOutput(false); @@ -204,7 +204,10 @@ std::vector LuaBridge::outdated(const PluginCallContext& context) { return {}; } + const bool silentRuntime = m_hostRuntime.shouldUseSilentRuntime(context.flags); + m_hostRuntime.setSilentRuntimeOutput(silentRuntime); auto result = func(context); + m_hostRuntime.setSilentRuntimeOutput(false); if (!result.valid()) { sol::error err = result; log_lua_error(m_logger, m_pluginId, std::string("Lua Error (outdated): ") + err.what()); @@ -222,7 +225,10 @@ std::vector LuaBridge::search(const PluginCallContext& context, con return {}; } + const bool silentRuntime = m_hostRuntime.shouldUseSilentRuntime(context.flags); + m_hostRuntime.setSilentRuntimeOutput(silentRuntime); auto result = func(context, prompt); + m_hostRuntime.setSilentRuntimeOutput(false); if (!result.valid()) { sol::error err = result; log_lua_error(m_logger, m_pluginId, std::string("Lua Error (search): ") + err.what()); @@ -240,7 +246,7 @@ PackageInfo LuaBridge::info(const PluginCallContext& context, const std::string& return {}; } - const bool silentRuntime = m_hostRuntime.hasSilentRuntimeFlag(context.flags); + const bool silentRuntime = m_hostRuntime.shouldUseSilentRuntime(context.flags); m_hostRuntime.setSilentRuntimeOutput(silentRuntime); auto result = func(context, packageName); m_hostRuntime.setSilentRuntimeOutput(false); diff --git a/src/main/cpp/plugins/lua_bridge_runtime.cpp b/src/main/cpp/plugins/lua_bridge_runtime.cpp index d03ee77..cb7bfa7 100644 --- a/src/main/cpp/plugins/lua_bridge_runtime.cpp +++ b/src/main/cpp/plugins/lua_bridge_runtime.cpp @@ -29,7 +29,7 @@ void log_lua_error(Logger& logger, const std::string& scope, const std::string& } LuaBridgeScriptRuntime::LuaBridgeScriptRuntime() { - m_lua.open_libraries(sol::lib::base, sol::lib::table, sol::lib::string, sol::lib::math, sol::lib::io); + m_lua.open_libraries(sol::lib::base, sol::lib::table, sol::lib::string, sol::lib::math, sol::lib::io, sol::lib::os); } sol::state& LuaBridgeScriptRuntime::state() { diff --git a/src/main/cpp/plugins/rqp_plugin_hooks.cpp b/src/main/cpp/plugins/rqp_plugin_hooks.cpp index 6dc9429..22f3373 100644 --- a/src/main/cpp/plugins/rqp_plugin_hooks.cpp +++ b/src/main/cpp/plugins/rqp_plugin_hooks.cpp @@ -2,6 +2,7 @@ #include "core/archive/archive_resolver.h" #include "core/common/network_environment.h" +#include "core/common/temp_directory.h" #include "output/logger.h" #include "plugins/exec_rules.h" @@ -21,8 +22,6 @@ #include #include -#include - namespace { using boost::property_tree::ptree; @@ -148,16 +147,14 @@ class RqpRuntimeHost final : public IPluginRuntimeHost { } std::string createTempDirectory(const std::string& pluginId) override { - std::filesystem::path tempDir = std::filesystem::temp_directory_path() / ("reqpack-" + pluginId + "-XXXXXX"); - std::string templateString = tempDir.string(); - std::vector buffer(templateString.begin(), templateString.end()); - buffer.push_back('\0'); - char* created = ::mkdtemp(buffer.data()); - if (created == nullptr) { + try { + const std::filesystem::path created = + reqpack_make_unique_directory(std::filesystem::temp_directory_path(), "reqpack-" + pluginId); + tempDirectories_.emplace_back(created.string()); + return created.string(); + } catch (...) { return {}; } - tempDirectories_.emplace_back(created); - return created; } DownloadResult download(const std::string& pluginId, const std::string& url, const std::string& destinationPath) override { diff --git a/src/main/include/core/common/pipe_helpers.h b/src/main/include/core/common/pipe_helpers.h new file mode 100644 index 0000000..f529fed --- /dev/null +++ b/src/main/include/core/common/pipe_helpers.h @@ -0,0 +1,3 @@ +#pragma once + +bool create_pipe_cloexec(int pipefd[2]); diff --git a/src/main/include/core/common/process_runner.h b/src/main/include/core/common/process_runner.h new file mode 100644 index 0000000..ff484bf --- /dev/null +++ b/src/main/include/core/common/process_runner.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include + +struct ReqpackProcessResult { + int exitCode{1}; + std::string stdoutText; + std::string stderrText; + + bool success() const { + return exitCode == 0; + } +}; + +bool reqpack_run_process( + const std::vector& arguments, + const std::filesystem::path& workingDirectory = {}); + +ReqpackProcessResult reqpack_run_process_capture( + const std::vector& arguments, + const std::filesystem::path& workingDirectory = {}); diff --git a/src/main/include/core/common/socket_platform.h b/src/main/include/core/common/socket_platform.h new file mode 100644 index 0000000..66acfab --- /dev/null +++ b/src/main/include/core/common/socket_platform.h @@ -0,0 +1,54 @@ +#pragma once + +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include + +using ReqpackSocket = SOCKET; +inline constexpr ReqpackSocket REQPACK_INVALID_SOCKET = INVALID_SOCKET; + +inline void reqpack_ensure_socket_runtime() { + static bool initialized = false; + if (!initialized) { + WSADATA wsa_data{}; + (void)::WSAStartup(MAKEWORD(2, 2), &wsa_data); + initialized = true; + } +} + +inline void reqpack_close_socket(ReqpackSocket socket) { + if (socket != INVALID_SOCKET) { + ::closesocket(socket); + } +} + +inline int reqpack_shutdown_socket(ReqpackSocket socket) { + return ::shutdown(socket, SD_BOTH); +} +#else +#include +#include +#include +#include + +using ReqpackSocket = int; +inline constexpr ReqpackSocket REQPACK_INVALID_SOCKET = -1; + +inline void reqpack_ensure_socket_runtime() {} + +inline void reqpack_close_socket(ReqpackSocket socket) { + if (socket >= 0) { + ::close(socket); + } +} + +inline int reqpack_shutdown_socket(ReqpackSocket socket) { + return ::shutdown(socket, SHUT_RDWR); +} +#endif diff --git a/src/main/include/core/common/temp_directory.h b/src/main/include/core/common/temp_directory.h new file mode 100644 index 0000000..ee88635 --- /dev/null +++ b/src/main/include/core/common/temp_directory.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include +#include + +#if !defined(_WIN32) +#include +#endif + +inline std::filesystem::path reqpack_make_unique_directory( + const std::filesystem::path& parent, + const std::string& prefix +) { + std::error_code error; + std::filesystem::create_directories(parent, error); + if (error) { + throw std::runtime_error("failed to create directory: " + parent.string()); + } + +#if !defined(_WIN32) + const std::filesystem::path pattern = parent / (prefix + "-XXXXXX"); + std::string templateString = pattern.string(); + std::vector buffer(templateString.begin(), templateString.end()); + buffer.push_back('\0'); + if (char* created = ::mkdtemp(buffer.data()); created != nullptr) { + return std::filesystem::path(created); + } +#endif + + for (int attempt = 0; attempt < 100; ++attempt) { + const std::filesystem::path candidate = parent / (prefix + "-" + std::to_string(attempt)); + error.clear(); + if (std::filesystem::create_directory(candidate, error) && !error) { + return candidate; + } + } + + throw std::runtime_error("failed to create unique directory under: " + parent.string()); +} diff --git a/src/main/include/core/common/terminal_width.h b/src/main/include/core/common/terminal_width.h new file mode 100644 index 0000000..e8bfd1e --- /dev/null +++ b/src/main/include/core/common/terminal_width.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include + +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#include +#endif + +inline std::size_t reqpack_terminal_column_count(std::size_t fallback = 100) { + if (const char* columns = std::getenv("COLUMNS")) { + char* end = nullptr; + const unsigned long parsed = std::strtoul(columns, &end, 10); + if (end != columns && end != nullptr && *end == '\0' && parsed > 0) { + return static_cast(parsed); + } + } + +#if defined(_WIN32) + DWORD consoleMode = 0; + const HANDLE stdOut = GetStdHandle(STD_OUTPUT_HANDLE); + if (stdOut != INVALID_HANDLE_VALUE && GetConsoleMode(stdOut, &consoleMode)) { + CONSOLE_SCREEN_BUFFER_INFO info{}; + if (GetConsoleScreenBufferInfo(stdOut, &info)) { + const int width = info.srWindow.Right - info.srWindow.Left + 1; + if (width > 0) { + return static_cast(width); + } + } + } +#else + winsize size{}; + if (isatty(STDOUT_FILENO) && ioctl(STDOUT_FILENO, TIOCGWINSZ, &size) == 0 && size.ws_col > 0) { + return static_cast(size.ws_col); + } +#endif + + return fallback; +} diff --git a/src/main/include/core/common/time_helpers.h b/src/main/include/core/common/time_helpers.h new file mode 100644 index 0000000..cd74111 --- /dev/null +++ b/src/main/include/core/common/time_helpers.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#if defined(_WIN32) +inline bool reqpack_gmtime_utc(std::tm* tm, const std::time_t* timeValue) { + return gmtime_s(tm, timeValue) == 0; +} + +inline bool reqpack_localtime(std::tm* tm, const std::time_t* timeValue) { + return localtime_s(tm, timeValue) == 0; +} +#else +inline bool reqpack_gmtime_utc(std::tm* tm, const std::time_t* timeValue) { + return gmtime_r(timeValue, tm) != nullptr; +} + +inline bool reqpack_localtime(std::tm* tm, const std::time_t* timeValue) { + return localtime_r(timeValue, tm) != nullptr; +} +#endif diff --git a/src/main/include/core/common/tty_helpers.h b/src/main/include/core/common/tty_helpers.h new file mode 100644 index 0000000..0a48972 --- /dev/null +++ b/src/main/include/core/common/tty_helpers.h @@ -0,0 +1,23 @@ +#pragma once + +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#endif + +inline bool reqpack_stdout_is_tty() { +#if defined(_WIN32) + DWORD consoleMode = 0; + const HANDLE stdOut = GetStdHandle(STD_OUTPUT_HANDLE); + return stdOut != INVALID_HANDLE_VALUE && GetConsoleMode(stdOut, &consoleMode); +#else + return isatty(STDOUT_FILENO) != 0; +#endif +} diff --git a/src/main/include/core/common/windows_macro_guards.h b/src/main/include/core/common/windows_macro_guards.h new file mode 100644 index 0000000..aac814c --- /dev/null +++ b/src/main/include/core/common/windows_macro_guards.h @@ -0,0 +1,34 @@ +#pragma once + +#if defined(_WIN32) +#ifdef IGNORE +#undef IGNORE +#endif +#ifdef ERROR +#undef ERROR +#endif +#ifdef FAILED +#undef FAILED +#endif +#ifdef REGISTERED +#undef REGISTERED +#endif +#ifdef ACTIVE +#undef ACTIVE +#endif +#ifdef SUCCESS +#undef SUCCESS +#endif +#ifdef PENDING +#undef PENDING +#endif +#ifdef RUNNING +#undef RUNNING +#endif +#ifdef SHUTDOWN +#undef SHUTDOWN +#endif +#ifdef NOT_FOUND +#undef NOT_FOUND +#endif +#endif diff --git a/src/main/include/core/config/configuration.h b/src/main/include/core/config/configuration.h index 377c462..9d18c36 100644 --- a/src/main/include/core/config/configuration.h +++ b/src/main/include/core/config/configuration.h @@ -9,6 +9,21 @@ #include "core/common/build_info.h" +#if defined(_WIN32) +#ifdef IGNORE +#undef IGNORE +#endif +#ifdef ERROR +#undef ERROR +#endif +#ifdef FAILED +#undef FAILED +#endif +#ifdef stdin +#undef stdin +#endif +#endif + enum class SeverityLevel { UNASSIGNED, LOW, @@ -77,7 +92,7 @@ enum class RepositoryAuthType { enum class RepositoryChecksumPolicy { FAIL, WARN, - IGNORE + SKIP }; struct LoggingConfig { @@ -105,6 +120,14 @@ struct SecurityBackendConfig { OsvRefreshMode refreshMode{OsvRefreshMode::MANUAL}; long refreshIntervalSeconds{24L * 60L * 60L}; std::string overlayPath{}; + std::string apiBaseUrl{}; + std::string apiVersion{}; + std::string tokenEnv{}; + std::string orgId{}; + std::string groupId{}; + std::string dataset{}; + std::vector dbRepositories{}; + std::string helperPath{}; }; struct SecurityConfig { @@ -331,6 +354,7 @@ struct DisplayColorScheme { struct DisplayConfig { DisplayRenderer renderer{DisplayRenderer::PLAIN}; DisplayColorScheme colors{}; + bool jsonOutput{false}; }; struct ReqPackConfig { @@ -395,6 +419,7 @@ struct ReqPackConfigOverrides { std::optional reportOutputPath; std::optional dryRun; + std::optional jsonOutput; std::optional stopOnFirstFailure; std::optional useTransactionDb; std::optional jobs; @@ -609,7 +634,7 @@ inline std::string to_string(RepositoryChecksumPolicy policy) { switch (policy) { case RepositoryChecksumPolicy::FAIL: return "fail"; - case RepositoryChecksumPolicy::IGNORE: + case RepositoryChecksumPolicy::SKIP: return "ignore"; case RepositoryChecksumPolicy::WARN: default: diff --git a/src/main/include/core/execution/executor.h b/src/main/include/core/execution/executor.h index 0b22990..e1a9866 100644 --- a/src/main/include/core/execution/executor.h +++ b/src/main/include/core/execution/executor.h @@ -34,6 +34,8 @@ class Executer { bool usesLocalTarget{false}; IPlugin* plugin{nullptr}; bool pluginLoadFailed{false}; + // Non-nix systems that originally depended on this nix group (Windows soft-skip consumers). + std::vector nixSoftSkipConsumers; }; struct TaskGroupPlan { @@ -81,6 +83,9 @@ class Executer { bool dispatchTaskGroupToPlugin(const TaskGroup& taskGroup) const; std::vector buildSuccessRecords(const TaskGroup& taskGroup) const; std::vector buildFailureRecords(const TaskGroup& taskGroup) const; + std::vector buildAlreadySatisfiedRecords( + const std::vector& allTaskGroups, + const std::vector& executableTaskGroups) const; public: Executer(Registry* registry, const ReqPackConfig& config = default_reqpack_config()); diff --git a/src/main/include/core/planning/planner_platform_policy.h b/src/main/include/core/planning/planner_platform_policy.h new file mode 100644 index 0000000..e3cb883 --- /dev/null +++ b/src/main/include/core/planning/planner_platform_policy.h @@ -0,0 +1,45 @@ +#pragma once + +#include + +namespace planner_platform { + +inline bool isNixInstallSystem(const std::string& system) { + return system == "nix"; +} + +#if defined(_WIN32) +inline constexpr bool reorderNonNixBeforeNix = true; +#else +inline constexpr bool reorderNonNixBeforeNix = false; +#endif + +// On Windows, ignore edges nix -> non-nix for scheduling so Non-nix can run first +// without creating a cycle with the Non-nix-before-nix barrier. +inline bool shouldIgnoreScheduleEdge(const std::string& sourceSystem, const std::string& targetSystem) { +#if defined(_WIN32) + return isNixInstallSystem(sourceSystem) && !isNixInstallSystem(targetSystem); +#else + (void)sourceSystem; + (void)targetSystem; + return false; +#endif +} + +inline bool needsNonNixBeforeNixBarrier() { +#if defined(_WIN32) + return true; +#else + return false; +#endif +} + +inline bool softSkipNixInstalls() { +#if defined(_WIN32) + return true; +#else + return false; +#endif +} + +} // namespace planner_platform diff --git a/src/main/include/core/plugins/plugin_test_runner.h b/src/main/include/core/plugins/plugin_test_runner.h index 9debe70..c2543da 100644 --- a/src/main/include/core/plugins/plugin_test_runner.h +++ b/src/main/include/core/plugins/plugin_test_runner.h @@ -28,8 +28,8 @@ struct PluginTestCaseSummary { std::string message; std::vector commands{}; std::vector events{}; - std::vector stdout{}; - std::vector stderr{}; + std::vector stdoutLines{}; + std::vector stderrLines{}; std::vector artifacts{}; struct EventRecord { std::string name; diff --git a/src/main/include/core/registry/registry.h b/src/main/include/core/registry/registry.h index dd470b8..6794b95 100644 --- a/src/main/include/core/registry/registry.h +++ b/src/main/include/core/registry/registry.h @@ -11,6 +11,8 @@ #include #include "plugins/iplugin.h" +#include "core/common/windows_macro_guards.h" + enum class PluginState { NOT_FOUND, REGISTERED, diff --git a/src/main/include/core/remote/serve_remote.h b/src/main/include/core/remote/serve_remote.h index 095be5c..361878e 100644 --- a/src/main/include/core/remote/serve_remote.h +++ b/src/main/include/core/remote/serve_remote.h @@ -19,7 +19,7 @@ enum class ServeRemoteProtocol { }; struct ServeRuntimeOptions { - bool stdin{false}; + bool useStdin{false}; bool remote{false}; ServeRemoteProtocol remoteProtocol{ServeRemoteProtocol::TEXT}; std::string bind{"127.0.0.1"}; diff --git a/src/main/include/core/security/gh_advisory_vulnerability_sync_service.h b/src/main/include/core/security/gh_advisory_vulnerability_sync_service.h new file mode 100644 index 0000000..3d85416 --- /dev/null +++ b/src/main/include/core/security/gh_advisory_vulnerability_sync_service.h @@ -0,0 +1,21 @@ +#pragma once + +#include "core/config/configuration.h" +#include "core/security/validator_core.h" + +#include +#include +#include + +class GhAdvisoryVulnerabilitySyncService { + ReqPackConfig config; + std::set ecosystems; + +public: + GhAdvisoryVulnerabilitySyncService( + std::set ecosystems, + const ReqPackConfig& config = default_reqpack_config() + ); + + std::vector ensureReady(bool forceRefresh = false) const; +}; diff --git a/src/main/include/core/security/snyk_vulnerability_sync_service.h b/src/main/include/core/security/snyk_vulnerability_sync_service.h new file mode 100644 index 0000000..732a069 --- /dev/null +++ b/src/main/include/core/security/snyk_vulnerability_sync_service.h @@ -0,0 +1,53 @@ +#pragma once + +#include "core/config/configuration.h" +#include "core/security/validator_core.h" +#include "core/security/vulnerability_database.h" + +#include +#include +#include +#include +#include + +struct SnykExportRow { + std::string problemId; + std::string problemTitle; + std::string cve; + std::string packageNameAndVersion; + std::string semverVulnerableRange; + std::string issueSeverity; + double nvdScore{0.0}; + double snykCvssScore{0.0}; + std::string updatedAt; + std::string fixedInVersion; + std::string productName; +}; + +struct ParsedSnykPackage { + std::string name; + std::string version; +}; + +std::vector snyk_parse_csv_line(std::string_view line); +std::optional snyk_parse_export_row( + const std::vector& header, + const std::vector& values +); +std::optional snyk_parse_maven_package_name_and_version(const std::string& value); +std::string snyk_normalize_severity(const std::string& value, double score); + +class SnykVulnerabilitySyncService { + ReqPackConfig config; + VulnerabilityDatabase* database; + std::set ecosystems; + +public: + SnykVulnerabilitySyncService( + VulnerabilityDatabase* database, + std::set ecosystems, + const ReqPackConfig& config = default_reqpack_config() + ); + + std::vector ensureReady(bool forceRefresh = false) const; +}; diff --git a/src/main/include/core/security/trivy_vulnerability_sync_service.h b/src/main/include/core/security/trivy_vulnerability_sync_service.h new file mode 100644 index 0000000..136ce6c --- /dev/null +++ b/src/main/include/core/security/trivy_vulnerability_sync_service.h @@ -0,0 +1,50 @@ +#pragma once + +#include "core/config/configuration.h" +#include "core/security/validator_core.h" +#include "core/security/vulnerability_database.h" + +#include +#include +#include +#include + +struct TrivyHelperRange { + std::string introduced; + std::string fixed; + std::string lastAffected; + std::string limit; +}; + +struct TrivyHelperRecord { + std::string ecosystem; + std::string packageName; + std::string advisoryId; + std::vector aliases; + std::string summary; + std::string description; + std::string severity; + double score{0.0}; + std::vector references; + std::string modified; + std::string published; + std::vector versions; + std::vector ranges; +}; + +std::optional trivy_parse_helper_record(const std::string& line); + +class TrivyVulnerabilitySyncService { + ReqPackConfig config; + VulnerabilityDatabase* database; + std::set ecosystems; + +public: + TrivyVulnerabilitySyncService( + VulnerabilityDatabase* database, + std::set ecosystems, + const ReqPackConfig& config = default_reqpack_config() + ); + + std::vector ensureReady(bool forceRefresh = false) const; +}; diff --git a/src/main/include/output/diagnostic.h b/src/main/include/output/diagnostic.h index e198c09..2936d18 100644 --- a/src/main/include/output/diagnostic.h +++ b/src/main/include/output/diagnostic.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include diff --git a/src/main/include/output/idisplay.h b/src/main/include/output/idisplay.h index e9059d3..58e5760 100644 --- a/src/main/include/output/idisplay.h +++ b/src/main/include/output/idisplay.h @@ -5,6 +5,8 @@ #include "output/progress_metrics.h" +#include "core/common/windows_macro_guards.h" + // ───────────────────────────────────────────────────────────────────────────── // DisplayMode — mirrors ActionType but lives in output layer, no boost dep. // ───────────────────────────────────────────────────────────────────────────── diff --git a/src/main/include/output/logger.h b/src/main/include/output/logger.h index c96e1a5..cf729ae 100644 --- a/src/main/include/output/logger.h +++ b/src/main/include/output/logger.h @@ -3,6 +3,7 @@ #include "output/diagnostic.h" #include "output/idisplay.h" +#include #include #include #include @@ -137,6 +138,7 @@ class Logger { bool stopRequested{false}; std::string pattern{"%^[%T] [%l] %v%$"}; std::atomic consoleOutputEnabled{true}; + std::atomic jsonOutputMode{false}; std::ofstream structuredFileStream; std::string structuredFilePath{}; bool captureDisplayEvents{true}; @@ -175,6 +177,8 @@ class Logger { void setPattern(const std::string& pattern); void setConsoleOutput(bool enable); bool isConsoleOutputEnabled() const; + void setJsonOutputMode(bool enable); + bool isJsonOutputMode() const; void setFileSink(const std::string& filename); void disableFileSink(); void setStructuredFileSink(const std::string& filename); @@ -191,9 +195,9 @@ class Logger { std::uint64_t emit(OutputAction action, const OutputContext& context = {}); std::uint64_t emitDiagnostic(const DiagnosticMessage& diagnostic, bool mirrorToDisplay = true); - void stdout(const std::string& message, - const std::string& source = {}, - const std::string& scope = {}); + void logStdout(const std::string& message, + const std::string& source = {}, + const std::string& scope = {}); void flush(); void flushSync(); diff --git a/src/main/include/output/run_result_json.h b/src/main/include/output/run_result_json.h new file mode 100644 index 0000000..6d65c90 --- /dev/null +++ b/src/main/include/output/run_result_json.h @@ -0,0 +1,24 @@ +#pragma once + +#include "core/common/types.h" + +#include +#include + +struct RunResultJsonItem { + std::string system; + std::string name; + std::string version; + std::string status; + std::string errorMessage; +}; + +struct RunResultJsonDocument { + bool ok{true}; + bool dryRun{false}; + std::vector items; +}; + +std::string run_result_status_for_action(ActionType action, bool success, bool dryRun); +std::string serialize_run_result_json(const RunResultJsonDocument& document); +void emit_run_result_json(const RunResultJsonDocument& document); diff --git a/src/main/include/plugins/lua_bridge_host_runtime.h b/src/main/include/plugins/lua_bridge_host_runtime.h index e1e782d..7655c7b 100644 --- a/src/main/include/plugins/lua_bridge_host_runtime.h +++ b/src/main/include/plugins/lua_bridge_host_runtime.h @@ -37,6 +37,7 @@ class LuaBridgeHostRuntime { const LuaBridgeRuntimeBindingContext* runtimeBindingContext(std::uint64_t contextId) const; bool hasSilentRuntimeFlag(const std::vector& flags) const; + bool shouldUseSilentRuntime(const std::vector& flags) const; void setSilentRuntimeOutput(bool silent); bool silentRuntimeOutput() const; diff --git a/tests/cpp/test_helpers.h b/tests/cpp/test_helpers.h index 6c228fc..895230e 100644 --- a/tests/cpp/test_helpers.h +++ b/tests/cpp/test_helpers.h @@ -1,10 +1,16 @@ #pragma once #include +#include #include +#include #include #include +#if defined(_WIN32) +#include +#endif + #ifndef REQPACK_TEST_REPO_ROOT #error "REQPACK_TEST_REPO_ROOT must be defined for test targets" #endif @@ -52,3 +58,58 @@ inline std::filesystem::path repo_root() { inline std::filesystem::path build_root() { return std::filesystem::path(REQPACK_TEST_BUILD_DIR); } + +inline std::string hermetic_config_cli_arg() { + return " --config " + escape_shell_arg((repo_root() / "tests" / "fixtures" / "hermetic-config.lua").string()); +} + +inline void set_test_environment_value(const std::string& name, const std::optional& value) { +#if defined(_WIN32) + _putenv_s(name.c_str(), value.has_value() ? value->c_str() : ""); +#else + if (value.has_value()) { + ::setenv(name.c_str(), value->c_str(), 1); + } else { + ::unsetenv(name.c_str()); + } +#endif +} + +class ScopedEnvVar { +public: + explicit ScopedEnvVar(std::string name) + : name_(std::move(name)) { + if (const char* existing = std::getenv(name_.c_str())) { + previous_ = std::string(existing); + } + } + + ScopedEnvVar(std::string name, std::string value) + : ScopedEnvVar(std::move(name)) { + set_test_environment_value(name_, value); + } + + ScopedEnvVar(const char* name, const char* value) + : name_(name) { + if (const char* existing = std::getenv(name_.c_str())) { + previous_ = std::string(existing); + } + if (value != nullptr) { + set_test_environment_value(name_, std::string(value)); + } else { + set_test_environment_value(name_, std::nullopt); + } + } + + ~ScopedEnvVar() { + if (previous_.has_value()) { + set_test_environment_value(name_, previous_); + } else { + set_test_environment_value(name_, std::nullopt); + } + } + +private: + std::string name_; + std::optional previous_; +}; diff --git a/tests/fixtures/hermetic-config.lua b/tests/fixtures/hermetic-config.lua new file mode 100644 index 0000000..659b23c --- /dev/null +++ b/tests/fixtures/hermetic-config.lua @@ -0,0 +1,5 @@ +return { + logging = { + fileOutput = false, + }, +} diff --git a/tests/fixtures/plugin-bundles/gh-advisory/metadata.json b/tests/fixtures/plugin-bundles/gh-advisory/metadata.json new file mode 100644 index 0000000..1761d33 --- /dev/null +++ b/tests/fixtures/plugin-bundles/gh-advisory/metadata.json @@ -0,0 +1,8 @@ +{ + "formatVersion": 1, + "name": "gh-advisory", + "version": "0.1.0", + "summary": "ReqPack security-provider wrapper for GitHub Advisory DB sync", + "description": "ReqPack Lua wrapper plugin for importing GitHub Advisory Database advisories into ReqPack shared vulnerability indexes", + "license": "MIT" +} diff --git a/tests/fixtures/plugin-bundles/gh-advisory/reqpack.lua b/tests/fixtures/plugin-bundles/gh-advisory/reqpack.lua new file mode 100644 index 0000000..8d7a119 --- /dev/null +++ b/tests/fixtures/plugin-bundles/gh-advisory/reqpack.lua @@ -0,0 +1,4 @@ +return { + apiVersion = 1, + depends = {} +} diff --git a/tests/fixtures/plugin-bundles/gh-advisory/run.lua b/tests/fixtures/plugin-bundles/gh-advisory/run.lua new file mode 100644 index 0000000..109db61 --- /dev/null +++ b/tests/fixtures/plugin-bundles/gh-advisory/run.lua @@ -0,0 +1,46 @@ +plugin = {} + +function plugin.getName() return "GitHub Advisory" end +function plugin.getVersion() return "0.1.0" end +function plugin.getSecurityMetadata() + return { + role = "security-provider", + capabilities = { "sync" }, + ecosystemScopes = { "all-github-advisory-ecosystems" }, + networkScopes = { + { host = "codeload.github.com", scheme = "https", pathPrefix = "/github/advisory-database/" }, + { host = "github.com", scheme = "https", pathPrefix = "/github/advisory-database/" }, + }, + privilegeLevel = "none", + } +end +function plugin.getRequirements() return {} end +function plugin.getCategories() return { "Security", "Security Provider", "GitHub Advisory" } end +function plugin.getMissingPackages(packages) return packages or {} end +function plugin.install(context, packages) return true end +function plugin.installLocal(context, path) return false end +function plugin.remove(context, packages) return false end +function plugin.update(context, packages) return true end +function plugin.list(context) + return { + { name = "github-advisories", version = "0.1.0", type = "security-database", summary = "Sync all ecosystems available in GitHub Advisory Database" }, + } +end +function plugin.outdated(context) return {} end +function plugin.search(context, prompt) + return { + { name = "github-advisories", version = "0.1.0", type = "security-database", summary = "Sync all ecosystems exposed by GitHub Advisory Database" }, + } +end +function plugin.info(context, name) + return { + name = tostring(name or ""), + version = "0.1.0", + type = "security-database", + summary = "GitHub Advisory Database sync target", + description = "Imports OSV-formatted advisories from github/advisory-database into ReqPack shared vulnerability indexes for requested ecosystems.", + } +end +function plugin.shutdown() return true end + +return plugin diff --git a/tests/fixtures/plugin-bundles/gh-advisory/scripts/install.lua b/tests/fixtures/plugin-bundles/gh-advisory/scripts/install.lua new file mode 100644 index 0000000..b30e187 --- /dev/null +++ b/tests/fixtures/plugin-bundles/gh-advisory/scripts/install.lua @@ -0,0 +1 @@ +return true diff --git a/tests/fixtures/plugin-bundles/gh-advisory/scripts/remove.lua b/tests/fixtures/plugin-bundles/gh-advisory/scripts/remove.lua new file mode 100644 index 0000000..b30e187 --- /dev/null +++ b/tests/fixtures/plugin-bundles/gh-advisory/scripts/remove.lua @@ -0,0 +1 @@ +return true diff --git a/tests/fixtures/plugin-bundles/trivy/metadata.json b/tests/fixtures/plugin-bundles/trivy/metadata.json new file mode 100644 index 0000000..6ce2e72 --- /dev/null +++ b/tests/fixtures/plugin-bundles/trivy/metadata.json @@ -0,0 +1,8 @@ +{ + "formatVersion": 1, + "name": "trivy", + "version": "0.1.0", + "summary": "Trivy plugin", + "description": "fixture trivy security provider", + "license": "MIT" +} diff --git a/tests/fixtures/plugin-bundles/trivy/reqpack.lua b/tests/fixtures/plugin-bundles/trivy/reqpack.lua new file mode 100644 index 0000000..8d7a119 --- /dev/null +++ b/tests/fixtures/plugin-bundles/trivy/reqpack.lua @@ -0,0 +1,4 @@ +return { + apiVersion = 1, + depends = {} +} diff --git a/tests/fixtures/plugin-bundles/trivy/run.lua b/tests/fixtures/plugin-bundles/trivy/run.lua new file mode 100644 index 0000000..27f36bd --- /dev/null +++ b/tests/fixtures/plugin-bundles/trivy/run.lua @@ -0,0 +1,25 @@ +plugin = {} + +function plugin.getName() return "Trivy" end +function plugin.getVersion() return "0.1.0" end +function plugin.getSecurityMetadata() + return { + role = "security-provider", + ecosystemScopes = { "Maven", "crates.io", "Debian", "RPM" }, + privilegeLevel = "none", + } +end +function plugin.getRequirements() return {} end +function plugin.getCategories() return { "security-provider" } end +function plugin.getMissingPackages(packages) return packages end +function plugin.install(context, packages) return true end +function plugin.installLocal(context, path) return true end +function plugin.remove(context, packages) return true end +function plugin.update(context, packages) return true end +function plugin.list(context) return {} end +function plugin.outdated(context) return {} end +function plugin.search(context, prompt) return {} end +function plugin.info(context, package) return { name = package, version = "0.1.0", description = "security provider" } end +function plugin.shutdown() return true end + +return plugin diff --git a/tests/fixtures/plugin-bundles/trivy/scripts/install.lua b/tests/fixtures/plugin-bundles/trivy/scripts/install.lua new file mode 100644 index 0000000..b30e187 --- /dev/null +++ b/tests/fixtures/plugin-bundles/trivy/scripts/install.lua @@ -0,0 +1 @@ +return true diff --git a/tests/fixtures/plugin-bundles/trivy/scripts/remove.lua b/tests/fixtures/plugin-bundles/trivy/scripts/remove.lua new file mode 100644 index 0000000..b30e187 --- /dev/null +++ b/tests/fixtures/plugin-bundles/trivy/scripts/remove.lua @@ -0,0 +1 @@ +return true diff --git a/tests/integration/core/execution/executor_service_tests.cpp b/tests/integration/core/execution/executor_service_tests.cpp index 0f5c597..175fa2d 100644 --- a/tests/integration/core/execution/executor_service_tests.cpp +++ b/tests/integration/core/execution/executor_service_tests.cpp @@ -72,6 +72,7 @@ ReqPackConfig make_executor_test_config(const std::filesystem::path& root) { config.execution.transactionDatabasePath = (root / "transactions").string(); config.execution.checkVirtualFileSystemWrite = false; config.history.historyPath = (root / "history").string(); + config.rqp.statePath = (root / "rqp-state").string(); return config; } diff --git a/tests/integration/core/execution/orchestrator_service_tests.cpp b/tests/integration/core/execution/orchestrator_service_tests.cpp index d45aa98..3136c84 100644 --- a/tests/integration/core/execution/orchestrator_service_tests.cpp +++ b/tests/integration/core/execution/orchestrator_service_tests.cpp @@ -116,6 +116,26 @@ void copy_repo_plugin(const std::filesystem::path& pluginRoot, const std::string } } +void add_minimal_sys_apt_mocks(const std::filesystem::path& fakeBin) { + write_file(fakeBin / "apt-get", + "#!/bin/sh\n" + "exit 0\n"); + write_file(fakeBin / "dpkg-query", + "#!/bin/sh\n" + "exit 1\n"); + REQUIRE(std::system(("chmod +x " + escape_shell_arg((fakeBin / "apt-get").string()) + " " + + escape_shell_arg((fakeBin / "dpkg-query").string())).c_str()) == 0); +} + +std::vector> minimal_sys_apt_environment(const std::filesystem::path& fakeBin) { + return { + {"REQPACK_SYS_BACKEND", "apt"}, + {"REQPACK_SYS_NO_SUDO", "1"}, + {"REQPACK_SYS_APT_BIN", (fakeBin / "apt-get").string()}, + {"REQPACK_SYS_DPKG_QUERY_BIN", (fakeBin / "dpkg-query").string()}, + }; +} + std::filesystem::path write_config( const std::filesystem::path& root, const std::filesystem::path& pluginDirectory, @@ -1368,6 +1388,72 @@ end function plugin.shutdown() return true end )"; +const char* SECURITY_PROVIDER_PLUGIN = R"( +plugin = {} + +function plugin.getName() return REQPACK_PLUGIN_ID end +function plugin.getVersion() return "1.0.0" end +function plugin.getSecurityMetadata() + return { + role = "security-provider", + ecosystemScopes = { "Maven" }, + networkScopes = { "api.snyk.io" }, + privilegeLevel = "none", + } +end +function plugin.getRequirements() return {} end +function plugin.getCategories() return { "security-provider" } end +function plugin.getMissingPackages(packages) return packages end +function plugin.install(context, packages) return true end +function plugin.installLocal(context, path) return true end +function plugin.remove(context, packages) return true end +function plugin.update(context, packages) return true end +function plugin.list(context) return {} end +function plugin.outdated(context) return {} end +function plugin.search(context, prompt) return {} end +function plugin.info(context, package) return { name = package, version = "1.0.0", description = "security provider" } end +function plugin.shutdown() return true end +)"; + +const char* MAVEN_AUDIT_PLUGIN = R"( +plugin = {} + +function plugin.getName() return REQPACK_PLUGIN_ID end +function plugin.getVersion() return "1.0.0" end +function plugin.getSecurityMetadata() + return { + osvEcosystem = "Maven", + purlType = "maven", + versionComparatorProfile = "maven-comparable", + } +end +function plugin.getRequirements() return {} end +function plugin.getCategories() return { "pkg", "orch" } end +function plugin.getMissingPackages(packages) return packages end +function plugin.install(context, packages) return true end +function plugin.installLocal(context, path) return true end +function plugin.remove(context, packages) return true end +function plugin.update(context, packages) return true end +function plugin.list(context) return {} end +function plugin.outdated(context) return {} end +function plugin.search(context, prompt) return {} end +function plugin.info(context, package) + return { + name = package, + version = "unknown", + description = "Maven artifact", + } +end +function plugin.resolvePackage(context, package) + if package.name == "org.apache.logging.log4j:log4j-core" then + package.version = "2.14.1" + return package + end + return nil +end +function plugin.shutdown() return true end +)"; + const char* PACK_PLUGIN = R"( plugin = {} @@ -4134,6 +4220,326 @@ TEST_CASE("orchestrator audit resolves explicit package versions before matching CHECK(output.find("version unavailable") == std::string::npos); } +TEST_CASE("orchestrator install snyk maven imports tenant findings into scoped security index", "[integration][orchestrator][security]") { + TempDir tempDir{"reqpack-orchestrator-install-snyk-maven"}; + const std::filesystem::path pluginDirectory = tempDir.path() / "plugins"; + const std::filesystem::path apiRoot = tempDir.path() / "snyk-api" / "orgs" / "org-1"; + const std::filesystem::path exportFile = tempDir.path() / "snyk-export.csv"; + const std::filesystem::path configPath = tempDir.path() / "config.lua"; + const std::filesystem::path auditOutputPath = tempDir.path() / "audit.json"; + + write_file(apiRoot / "export.json", R"({"export_id":"exp-1"})"); + write_file(apiRoot / "jobs" / "export" / "exp-1.json", R"({"status":"FINISHED"})"); + write_file(apiRoot / "export" / "exp-1.json", std::string{"{"} + "\"download_url\":\"file://" + exportFile.string() + "\"}"); + write_file(exportFile, + "PROBLEM_ID,PROBLEM_TITLE,CVE,PACKAGE_NAME_AND_VERSION,SEMVER_VULNERABLE_RANGE,ISSUE_SEVERITY,NVD_SCORE,SNYK_CVSS_SCORE,UPDATED_AT,FIXED_IN_VERSION,PRODUCT_NAME\n" + "SNYK-JAVA-LOG4J-1,Remote code execution,CVE-2026-1,pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1,\"[,2.15.0)\",critical,9.8,9.8,2026-01-01T00:00:00Z,2.15.0,Snyk Open Source\n" + ); + + write_file(configPath, + "return {\n" + " execution = {\n" + " useTransactionDb = false,\n" + " deleteCommittedTransactions = false,\n" + " checkVirtualFileSystemWrite = false,\n" + " transactionDatabasePath = '" + (tempDir.path() / "transactions").string() + "',\n" + " },\n" + " planner = {\n" + " autoDownloadMissingPlugins = false,\n" + " autoDownloadMissingDependencies = false,\n" + " },\n" + " registry = {\n" + " pluginDirectory = '" + pluginDirectory.string() + "',\n" + " databasePath = '" + (tempDir.path() / "registry-db").string() + "',\n" + " autoLoadPlugins = true,\n" + " shutDownPluginsOnExit = true,\n" + " },\n" + " interaction = {\n" + " interactive = false,\n" + " },\n" + " security = {\n" + " autoFetch = true,\n" + " indexPath = '" + (tempDir.path() / "security-index").string() + "',\n" + " cachePath = '" + (tempDir.path() / "security-cache").string() + "',\n" + " osvDatabasePath = '" + (tempDir.path() / "osv-db").string() + "',\n" + " osvRefreshMode = 'manual',\n" + " backends = {\n" + " snyk = {\n" + " apiBaseUrl = 'file://" + (tempDir.path() / "snyk-api").string() + "',\n" + " apiVersion = '2024-10-15',\n" + " tokenEnv = 'REQPACK_TEST_SNYK_TOKEN',\n" + " orgId = 'org-1',\n" + " dataset = 'issues',\n" + " refreshMode = 'always',\n" + " },\n" + " },\n" + " },\n" + " rqp = {\n" + " statePath = '" + (tempDir.path() / "rqp-state").string() + "',\n" + " },\n" + "}\n"); + + add_plugin_script(pluginDirectory, "snyk", SECURITY_PROVIDER_PLUGIN); + add_plugin_script(pluginDirectory, "maven", MAVEN_AUDIT_PLUGIN); + + int installStatus = 0; + const std::string installOutput = run_reqpack_with_home_env_and_status(tempDir.path(), configPath, tempDir.path(), { + {"REQPACK_TEST_SNYK_TOKEN", "token-1"}, + }, { + "install", + "snyk", + "maven", + }, installStatus); + + CHECK(installStatus == 0); + CHECK(installOutput.find("security gateway action failed") == std::string::npos); + CHECK(std::filesystem::exists(tempDir.path() / "security-index" / "Maven")); + + int auditStatus = 0; + const std::string auditOutput = run_reqpack_with_home_env_and_status(tempDir.path(), configPath, tempDir.path(), { + {"REQPACK_TEST_SNYK_TOKEN", "token-1"}, + }, { + "audit", + "maven", + "org.apache.logging.log4j:log4j-core", + "--format", + "json", + "--output", + auditOutputPath.string(), + }, auditStatus); + + CHECK(auditStatus == 0); + CHECK(auditOutput.find(auditOutputPath.string()) != std::string::npos); + REQUIRE(std::filesystem::exists(auditOutputPath)); + + const std::string report = read_file(auditOutputPath); + CHECK(report.find("\"findingCount\": 1") != std::string::npos); + CHECK(report.find("\"id\": \"SNYK-JAVA-LOG4J-1\"") != std::string::npos); + CHECK(report.find("\"name\": \"org.apache.logging.log4j:log4j-core\"") != std::string::npos); + CHECK(report.find("\"version\": \"2.14.1\"") != std::string::npos); +} + +TEST_CASE("orchestrator install trivy maven imports shared advisories into scoped security index", "[integration][orchestrator][security]") { + TempDir tempDir{"reqpack-orchestrator-install-trivy-maven"}; + const std::filesystem::path pluginDirectory = tempDir.path() / "plugins"; + const std::filesystem::path trivyRoot = tempDir.path() / "trivy-db"; + const std::filesystem::path helperPath = tempDir.path() / "trivy-helper.sh"; + const std::filesystem::path configPath = tempDir.path() / "config.lua"; + const std::filesystem::path auditOutputPath = tempDir.path() / "audit.json"; + + write_file(trivyRoot / "metadata.json", R"({"Version":2})"); + write_file(trivyRoot / "trivy.db", "fixture"); + write_file(helperPath, + "#!/bin/sh\n" + "printf '%s\\n' '{\"ecosystem\":\"Maven\",\"packageName\":\"org.apache.logging.log4j:log4j-core\",\"advisoryId\":\"CVE-2021-44228\",\"aliases\":[\"GHSA-jfh8-c2jp-5v3q\"],\"summary\":\"Remote code execution\",\"description\":\"desc\",\"severity\":\"CRITICAL\",\"score\":10.0,\"references\":[\"https://example.test/CVE-2021-44228\"],\"modified\":\"2021-12-10T00:00:00Z\",\"published\":\"2021-12-10T00:00:00Z\",\"ranges\":[{\"introduced\":\"2.0-beta9\",\"fixed\":\"2.15.0\"}]}'\n" + ); + REQUIRE(std::system((std::string{"chmod +x "} + escape_shell_arg(helperPath.string())).c_str()) == 0); + + write_file(configPath, + "return {\n" + " execution = {\n" + " useTransactionDb = false,\n" + " deleteCommittedTransactions = false,\n" + " checkVirtualFileSystemWrite = false,\n" + " transactionDatabasePath = '" + (tempDir.path() / "transactions").string() + "',\n" + " },\n" + " planner = {\n" + " autoDownloadMissingPlugins = false,\n" + " autoDownloadMissingDependencies = false,\n" + " },\n" + " registry = {\n" + " pluginDirectory = '" + pluginDirectory.string() + "',\n" + " databasePath = '" + (tempDir.path() / "registry-db").string() + "',\n" + " autoLoadPlugins = true,\n" + " shutDownPluginsOnExit = true,\n" + " },\n" + " interaction = {\n" + " interactive = false,\n" + " },\n" + " security = {\n" + " autoFetch = true,\n" + " indexPath = '" + (tempDir.path() / "security-index").string() + "',\n" + " cachePath = '" + (tempDir.path() / "security-cache").string() + "',\n" + " osvDatabasePath = '" + (tempDir.path() / "osv-db").string() + "',\n" + " osvRefreshMode = 'manual',\n" + " backends = {\n" + " trivy = {\n" + " dbRepositories = { 'file://" + trivyRoot.string() + "' },\n" + " helperPath = '" + helperPath.string() + "',\n" + " refreshMode = 'always',\n" + " },\n" + " },\n" + " },\n" + " rqp = {\n" + " statePath = '" + (tempDir.path() / "rqp-state").string() + "',\n" + " },\n" + "}\n"); + + copy_repo_plugin(pluginDirectory, "trivy"); + add_plugin_script(pluginDirectory, "maven", MAVEN_AUDIT_PLUGIN); + + int installStatus = 0; + const std::string installOutput = run_reqpack_with_home_and_status(tempDir.path(), configPath, tempDir.path(), { + "install", + "trivy", + "maven", + }, installStatus); + + CHECK(installStatus == 0); + CHECK(installOutput.find("security gateway action failed") == std::string::npos); + CHECK(std::filesystem::exists(tempDir.path() / "security-index" / "Maven")); + + int auditStatus = 0; + const std::string auditOutput = run_reqpack_with_home_and_status(tempDir.path(), configPath, tempDir.path(), { + "audit", + "maven", + "org.apache.logging.log4j:log4j-core", + "--format", + "json", + "--output", + auditOutputPath.string(), + }, auditStatus); + + CHECK(auditStatus == 0); + CHECK(auditOutput.find(auditOutputPath.string()) != std::string::npos); + REQUIRE(std::filesystem::exists(auditOutputPath)); + + const std::string report = read_file(auditOutputPath); + CHECK(report.find("\"findingCount\": 1") != std::string::npos); + CHECK(report.find("\"id\": \"CVE-2021-44228\"") != std::string::npos); + CHECK(report.find("\"name\": \"org.apache.logging.log4j:log4j-core\"") != std::string::npos); + CHECK(report.find("\"version\": \"2.14.1\"") != std::string::npos); +} + +TEST_CASE("orchestrator install gh-advisory pip imports shared advisories into scoped security index", "[integration][orchestrator][security]") { + TempDir tempDir{"reqpack-orchestrator-install-gh-advisory-pip"}; + const std::filesystem::path pluginDirectory = tempDir.path() / "plugins"; + const std::filesystem::path advisoryRoot = tempDir.path() / "gh-advisory-db" / "advisories" / "github-reviewed" / "2026" / "01"; + const std::filesystem::path configPath = tempDir.path() / "config.lua"; + const std::filesystem::path auditOutputPath = tempDir.path() / "audit.json"; + + write_file(advisoryRoot / "GHSA-demo.json", R"({ + "id": "GHSA-pip-demo", + "modified": "2026-01-01T00:00:00Z", + "summary": "urllib3 issue", + "affected": [{ + "package": {"ecosystem": "pip", "name": "urllib3"}, + "versions": ["1.26.18"] + }] + })"); + + write_file(configPath, + "return {\n" + " execution = {\n" + " useTransactionDb = false,\n" + " deleteCommittedTransactions = false,\n" + " checkVirtualFileSystemWrite = false,\n" + " transactionDatabasePath = '" + (tempDir.path() / "transactions").string() + "',\n" + " },\n" + " planner = {\n" + " autoDownloadMissingPlugins = false,\n" + " autoDownloadMissingDependencies = false,\n" + " },\n" + " registry = {\n" + " pluginDirectory = '" + pluginDirectory.string() + "',\n" + " databasePath = '" + (tempDir.path() / "registry-db").string() + "',\n" + " autoLoadPlugins = true,\n" + " shutDownPluginsOnExit = true,\n" + " },\n" + " interaction = {\n" + " interactive = false,\n" + " },\n" + " security = {\n" + " autoFetch = true,\n" + " indexPath = '" + (tempDir.path() / "security-index").string() + "',\n" + " cachePath = '" + (tempDir.path() / "security-cache").string() + "',\n" + " osvDatabasePath = '" + (tempDir.path() / "osv-db").string() + "',\n" + " osvRefreshMode = 'manual',\n" + " backends = {\n" + " [\"gh-advisory\"] = {\n" + " feedUrl = 'file://" + (tempDir.path() / "gh-advisory-db").string() + "',\n" + " refreshMode = 'always',\n" + " },\n" + " },\n" + " },\n" + " rqp = {\n" + " statePath = '" + (tempDir.path() / "rqp-state").string() + "',\n" + " },\n" + "}\n"); + + copy_repo_plugin(pluginDirectory, "gh-advisory"); + add_plugin_script(pluginDirectory, "pip", R"( +plugin = {} + +function plugin.getName() return REQPACK_PLUGIN_ID end +function plugin.getVersion() return "1.0.0" end +function plugin.getSecurityMetadata() + return { + osvEcosystem = "pip", + purlType = "pypi", + versionComparatorProfile = "pep440", + } +end +function plugin.getRequirements() return {} end +function plugin.getCategories() return { "pkg", "orch" } end +function plugin.getMissingPackages(packages) return packages end +function plugin.install(context, packages) return true end +function plugin.installLocal(context, path) return true end +function plugin.remove(context, packages) return true end +function plugin.update(context, packages) return true end +function plugin.list(context) return {} end +function plugin.outdated(context) return {} end +function plugin.search(context, prompt) return {} end +function plugin.info(context, package) + return { + name = package, + version = "unknown", + description = "pip package", + } +end +function plugin.resolvePackage(context, package) + if package.name == "urllib3" then + package.version = "1.26.18" + return package + end + return nil +end +function plugin.shutdown() return true end +)"); + + int installStatus = 0; + const std::string installOutput = run_reqpack_with_home_and_status(tempDir.path(), configPath, tempDir.path(), { + "install", + "gh-advisory", + "pip", + }, installStatus); + + CHECK(installStatus == 0); + CHECK(installOutput.find("security gateway action failed") == std::string::npos); + CHECK(std::filesystem::exists(tempDir.path() / "security-index" / "pip")); + + int auditStatus = 0; + const std::string auditOutput = run_reqpack_with_home_and_status(tempDir.path(), configPath, tempDir.path(), { + "audit", + "pip", + "urllib3", + "--format", + "json", + "--output", + auditOutputPath.string(), + }, auditStatus); + + CHECK(auditStatus == 0); + CHECK(auditOutput.find(auditOutputPath.string()) != std::string::npos); + REQUIRE(std::filesystem::exists(auditOutputPath)); + + const std::string report = read_file(auditOutputPath); + CHECK(report.find("\"findingCount\": 1") != std::string::npos); + CHECK(report.find("\"id\": \"GHSA-pip-demo\"") != std::string::npos); + CHECK(report.find("\"name\": \"urllib3\"") != std::string::npos); + CHECK(report.find("\"version\": \"1.26.18\"") != std::string::npos); +} + TEST_CASE("reqpack install stdin batches install commands until eof", "[integration][orchestrator][stdin]") { TempDir tempDir{"reqpack-orchestrator-install-stdin"}; const std::filesystem::path pluginDirectory = tempDir.path() / "plugins"; @@ -5246,20 +5652,21 @@ TEST_CASE("orchestrator install maven uses configured repositories and auth sett REQUIRE(std::system(("chmod +x " + escape_shell_arg((fakeBin / "java").string())).c_str()) == 0); REQUIRE(std::system(("chmod +x " + escape_shell_arg((fakeBin / "javac").string())).c_str()) == 0); REQUIRE(std::system(("chmod +x " + escape_shell_arg((fakeBin / "mvn").string())).c_str()) == 0); + add_minimal_sys_apt_mocks(fakeBin); const char* currentPath = std::getenv("PATH"); const std::string pathValue = fakeBin.string() + ":" + (currentPath != nullptr ? currentPath : ""); + std::vector> environment = minimal_sys_apt_environment(fakeBin); + environment.push_back({"PATH", pathValue}); + environment.push_back({"REQPACK_TEST_NEXUS_TOKEN", "secret-token"}); + environment.push_back({"REQPACK_MAVEN_REPO", (tempDir.path() / "custom-m2").string()}); int status = 0; const std::string output = run_reqpack_with_home_env_and_status( tempDir.path(), configPath, tempDir.path(), - { - {"PATH", pathValue}, - {"REQPACK_TEST_NEXUS_TOKEN", "secret-token"}, - {"REQPACK_MAVEN_REPO", (tempDir.path() / "custom-m2").string()}, - }, + environment, {"install", "maven", "org.junit:junit:4.13"}, status ); @@ -5319,25 +5726,26 @@ TEST_CASE("orchestrator install maven fails when configured repositories do not REQUIRE(std::system(("chmod +x " + escape_shell_arg((fakeBin / "java").string())).c_str()) == 0); REQUIRE(std::system(("chmod +x " + escape_shell_arg((fakeBin / "javac").string())).c_str()) == 0); REQUIRE(std::system(("chmod +x " + escape_shell_arg((fakeBin / "mvn").string())).c_str()) == 0); + add_minimal_sys_apt_mocks(fakeBin); const char* currentPath = std::getenv("PATH"); const std::string pathValue = fakeBin.string() + ":" + (currentPath != nullptr ? currentPath : ""); + std::vector> environment = minimal_sys_apt_environment(fakeBin); + environment.push_back({"PATH", pathValue}); int status = 0; const std::string output = run_reqpack_with_home_env_and_status( tempDir.path(), configPath, tempDir.path(), - { - {"PATH", pathValue}, - }, + environment, {"install", "maven", "org.junit:junit:4.13"}, status ); - + CHECK(status != 0); CHECK(output.find("no configured maven repository matched org.junit:junit") != std::string::npos); - CHECK(output.find("INSTALL done: 0 ok, 0 skipped, 1 failed") != std::string::npos); + CHECK(output.find("INSTALL done: 2 ok, 0 skipped, 1 failed") != std::string::npos); CHECK_FALSE(std::filesystem::exists(mvnLog)); } @@ -5473,23 +5881,24 @@ TEST_CASE("orchestrator sys plugin bootstraps nix when nix backend is selected b copy_repo_plugin(pluginDirectory, "sys"); - write_file(bootstrapScript, - "#!/bin/sh\n" - "printf '%s\\n' bootstrap >> " + escape_shell_arg(bootstrapLog.string()) + "\n" - "mkdir -p " + escape_shell_arg(installedNix.parent_path().string()) + "\n" - "cat > " + escape_shell_arg(installedNix.string()) + " <<'EOF'\n" + write_file(fakeBin / "nix-env.stub", "#!/bin/sh\n" "if [ \"$1\" = \"-q\" ]; then\n" " exit 1\n" "fi\n" "printf '%s\\n' \"$*\" >> " + escape_shell_arg(nixLog.string()) + "\n" - "exit 0\n" - "EOF\n" - "chmod +x " + escape_shell_arg(installedNix.string()) + "\n"); - REQUIRE(std::system(("chmod +x " + escape_shell_arg(bootstrapScript.string())).c_str()) == 0); + "exit 0\n"); + write_file(bootstrapScript, + "#!/bin/sh\n" + "printf '%s\\n' bootstrap >> " + escape_shell_arg(bootstrapLog.string()) + "\n" + "mkdir -p \"$HOME/.nix-profile/bin\"\n" + "/bin/cp " + escape_shell_arg((fakeBin / "nix-env.stub").string()) + " \"$HOME/.nix-profile/bin/nix-env\"\n" + "/bin/chmod +x \"$HOME/.nix-profile/bin/nix-env\"\n" + "exit 0\n"); + REQUIRE(std::system(("chmod +x " + escape_shell_arg((fakeBin / "nix-env.stub").string()) + " " + + escape_shell_arg(bootstrapScript.string())).c_str()) == 0); - const char* currentPath = std::getenv("PATH"); - const std::string pathValue = fakeBin.string() + ":" + (currentPath != nullptr ? currentPath : ""); + const std::string pathValue = fakeBin.string(); const std::filesystem::path homePath = tempDir.path() / "home"; std::filesystem::create_directories(homePath); diff --git a/tests/integration/core/plugins/plugin_test_runner_service_tests.cpp b/tests/integration/core/plugins/plugin_test_runner_service_tests.cpp index 9b1a131..e6bc797 100644 --- a/tests/integration/core/plugins/plugin_test_runner_service_tests.cpp +++ b/tests/integration/core/plugins/plugin_test_runner_service_tests.cpp @@ -188,6 +188,14 @@ function plugin.info(context, packageName) end return { name = packageName, version = "4.5.6", description = result.stdout } end + +function plugin.outdated(context) + return {} +end + +function plugin.shutdown() + return true +end )"; const char* PASS_CASE = R"( @@ -359,6 +367,64 @@ return { } )"; +const char* FIXTURE_INFO_PLUGIN = R"( +plugin = {} + +function plugin.getName() return "fixture-demo" end +function plugin.getVersion() return "1.0.0" end +function plugin.getRequirements() return {} end +function plugin.getCategories() return { "test" } end +function plugin.getMissingPackages(packages) return packages or {} end +function plugin.install(context, packages) return true end +function plugin.installLocal(context, path) return true end +function plugin.remove(context, packages) return true end +function plugin.update(context, packages) return true end +function plugin.list(context) return {} end +function plugin.search(context, prompt) return {} end +function plugin.outdated(context) return {} end +function plugin.shutdown() return true end + +function plugin.info(context, packageName) + local root = os.getenv("REQPACK_TEST_FIXTURE_ROOT") or "" + if root == "" then + return {} + end + local path = root .. "/payload/info.txt" + local file = io.open(path, "rb") + if file == nil then + return {} + end + local content = file:read("*a") + file:close() + return { name = packageName, version = "fixture", description = content } +end +)"; + +const char* FIXTURE_INFO_CASE = R"( +return { + name = "fixture-backed info", + request = { + action = "info", + system = "fixture-demo", + prompt = "alpha" + }, + fixtureRoot = "../fixtures-root", + fixtureDirs = { "payload" }, + fixtureFiles = { + { path = "payload/info.txt", content = "from fixture" } + }, + environment = { + REQPACK_TEST_FIXTURE_ROOT = "${fixtureRoot}" + }, + expect = { + success = true, + resultCount = 1, + resultName = "alpha", + resultVersion = "fixture" + } +} +)"; + const char* DNF_PLUGIN = R"( plugin = {} @@ -1006,6 +1072,23 @@ TEST_CASE("plugin test command supports query cases and help", "[integration][pl CHECK(helpOutput.find("--preset ") != std::string::npos); } +TEST_CASE("plugin test command supports filesystem fixtures", "[integration][plugin-test][service]") { + TempDir tempDir{"reqpack-plugin-test-fixtures"}; + const std::filesystem::path plugins = tempDir.path() / "plugins"; + const std::filesystem::path cases = tempDir.path() / "cases"; + write_plugin_bundle(plugins, "fixture-demo", FIXTURE_INFO_PLUGIN); + write_file(cases / "fixture-info.lua", FIXTURE_INFO_CASE); + const std::filesystem::path configPath = write_config(tempDir.path(), plugins); + + const std::string output = run_reqpack(tempDir.path(), configPath, { + "test-plugin", "--plugin", "fixture-demo", "--case", (cases / "fixture-info.lua").string() + }); + + INFO(output); + CHECK(output.find("[PASS] fixture-backed info") != std::string::npos); + CHECK(output.find("Cases: 1, Passed: 1, Failed: 0") != std::string::npos); +} + TEST_CASE("plugin test command validates artifacts and supports presets", "[integration][plugin-test][service]") { TempDir tempDir{"reqpack-plugin-test-preset"}; const std::filesystem::path plugins = tempDir.path() / "plugins"; diff --git a/tests/integration/plugins/exec_rules/smoke_tests.cpp b/tests/integration/plugins/exec_rules/smoke_tests.cpp index f37edb0..fd96b8e 100644 --- a/tests/integration/plugins/exec_rules/smoke_tests.cpp +++ b/tests/integration/plugins/exec_rules/smoke_tests.cpp @@ -13,6 +13,7 @@ std::string run_reqpack_case(const std::string& prompt) { const std::string inner = escape_shell_arg(binary.string()) + + hermetic_config_cli_arg() + " search smoke " + escape_shell_arg(prompt) + " --plugin-dir " + escape_shell_arg(pluginDir.string()) + " 2>&1"; @@ -28,6 +29,7 @@ std::string run_reqpack_case_verbose(const std::string& prompt) { const std::string inner = escape_shell_arg(binary.string()) + + hermetic_config_cli_arg() + " search smoke " + escape_shell_arg(prompt) + " --verbose" + " --plugin-dir " + escape_shell_arg(pluginDir.string()) + diff --git a/tests/unit/core/common/network_environment_tests.cpp b/tests/unit/core/common/network_environment_tests.cpp index c025a2c..a329a32 100644 --- a/tests/unit/core/common/network_environment_tests.cpp +++ b/tests/unit/core/common/network_environment_tests.cpp @@ -11,6 +11,7 @@ #include #include "core/common/network_environment.h" +#include "test_helpers.h" namespace { @@ -35,33 +36,6 @@ class TempDir { std::filesystem::path path_; }; -class ScopedEnvVar { -public: - explicit ScopedEnvVar(std::string name) - : name_(std::move(name)) { - if (const char* existing = std::getenv(name_.c_str())) { - previous_ = std::string(existing); - } - } - - ScopedEnvVar(std::string name, std::string value) - : ScopedEnvVar(std::move(name)) { - ::setenv(name_.c_str(), value.c_str(), 1); - } - - ~ScopedEnvVar() { - if (previous_.has_value()) { - ::setenv(name_.c_str(), previous_->c_str(), 1); - } else { - ::unsetenv(name_.c_str()); - } - } - -private: - std::string name_; - std::optional previous_; -}; - bool contains_prefix(const std::vector& values, const std::string& prefix) { return std::any_of(values.begin(), values.end(), [&prefix](const std::string& value) { return value.rfind(prefix, 0) == 0; diff --git a/tests/unit/core/config/configuration_cli_tests.cpp b/tests/unit/core/config/configuration_cli_tests.cpp index e0c856c..7997c73 100644 --- a/tests/unit/core/config/configuration_cli_tests.cpp +++ b/tests/unit/core/config/configuration_cli_tests.cpp @@ -115,6 +115,7 @@ TEST_CASE("configuration applies CLI overrides and expands path fields", "[unit] overrides.reportFormat = ReportFormat::CYCLONEDX; overrides.reportOutputPath = "~/reports/bom.json"; overrides.dryRun = true; + overrides.jsonOutput = true; overrides.jobs = 4; overrides.jobsMode = ExecutionJobsMode::FIXED; overrides.enableProxyExpansion = false; @@ -157,6 +158,7 @@ TEST_CASE("configuration applies CLI overrides and expands path fields", "[unit] CHECK(config.reports.format == ReportFormat::CYCLONEDX); CHECK(std::filesystem::path(config.reports.outputPath) == home / "reports/bom.json"); CHECK(config.execution.dryRun); + CHECK(config.display.jsonOutput); CHECK(config.execution.jobs == 4); CHECK(config.execution.jobsMode == ExecutionJobsMode::FIXED); CHECK_FALSE(config.planner.enableProxyExpansion); @@ -432,6 +434,7 @@ TEST_CASE("configuration extracts multiple CLI overrides in one pass", "[unit][c std::vector arguments{ "ReqPack", "--dry-run", + "--json", "--jobs", "6", "--registry=/tmp/reqpack-registry", @@ -462,6 +465,8 @@ TEST_CASE("configuration extracts multiple CLI overrides in one pass", "[unit][c REQUIRE(overrides.dryRun.has_value()); CHECK(overrides.dryRun.value()); + REQUIRE(overrides.jsonOutput.has_value()); + CHECK(overrides.jsonOutput.value()); REQUIRE(overrides.jobs.has_value()); CHECK(overrides.jobs.value() == 6); REQUIRE(overrides.jobsMode.has_value()); diff --git a/tests/unit/core/config/configuration_parse_tests.cpp b/tests/unit/core/config/configuration_parse_tests.cpp index 7ed63f0..ed7ed3a 100644 --- a/tests/unit/core/config/configuration_parse_tests.cpp +++ b/tests/unit/core/config/configuration_parse_tests.cpp @@ -9,6 +9,7 @@ #include "core/config/configuration.h" #include "core/remote/remote_profiles.h" +#include "test_helpers.h" namespace { @@ -33,29 +34,6 @@ class TempDir { std::filesystem::path path_; }; -class ScopedEnvVar { -public: - ScopedEnvVar(std::string name, std::string value) - : name_(std::move(name)) { - if (const char* existing = std::getenv(name_.c_str())) { - previous_ = std::string(existing); - } - ::setenv(name_.c_str(), value.c_str(), 1); - } - - ~ScopedEnvVar() { - if (previous_.has_value()) { - ::setenv(name_.c_str(), previous_->c_str(), 1); - } else { - ::unsetenv(name_.c_str()); - } - } - -private: - std::string name_; - std::optional previous_; -}; - void write_file(const std::filesystem::path& path, const std::string& content) { std::filesystem::create_directories(path.parent_path()); std::ofstream output(path); @@ -361,6 +339,27 @@ TEST_CASE("configuration loads lua config, expands paths, and preserves fallback refreshIntervalSeconds = 123, overlayPath = "~/backend-overlay.json", }, + snyk = { + apiBaseUrl = "https://api.eu.snyk.io/rest", + apiVersion = "2024-10-15", + tokenEnv = "REQPACK_SNYK_TOKEN", + orgId = "org-1", + dataset = "issues", + }, + trivy = { + dbRepositories = { + "mirror.gcr.io/aquasec/trivy-db:2", + "ghcr.io/aquasecurity/trivy-db:2", + }, + helperPath = "~/bin/trivydb-exporter", + refreshMode = "periodic", + refreshIntervalSeconds = 321, + }, + ["gh-advisory"] = { + feedUrl = "https://codeload.github.com/github/advisory-database/tar.gz/refs/heads/main", + refreshMode = "always", + refreshIntervalSeconds = 654, + }, }, }, reports = { @@ -467,6 +466,25 @@ TEST_CASE("configuration loads lua config, expands paths, and preserves fallback CHECK(config.security.backends.at("osv").refreshMode == OsvRefreshMode::ALWAYS); CHECK(config.security.backends.at("osv").refreshIntervalSeconds == 123); CHECK(std::filesystem::path(config.security.backends.at("osv").overlayPath) == home / "backend-overlay.json"); + REQUIRE(config.security.backends.contains("snyk")); + CHECK(config.security.backends.at("snyk").apiBaseUrl == "https://api.eu.snyk.io/rest"); + CHECK(config.security.backends.at("snyk").apiVersion == "2024-10-15"); + CHECK(config.security.backends.at("snyk").tokenEnv == "REQPACK_SNYK_TOKEN"); + CHECK(config.security.backends.at("snyk").orgId == "org-1"); + CHECK(config.security.backends.at("snyk").groupId.empty()); + CHECK(config.security.backends.at("snyk").dataset == "issues"); + REQUIRE(config.security.backends.contains("trivy")); + CHECK(config.security.backends.at("trivy").dbRepositories == std::vector{ + "mirror.gcr.io/aquasec/trivy-db:2", + "ghcr.io/aquasecurity/trivy-db:2", + }); + CHECK(std::filesystem::path(config.security.backends.at("trivy").helperPath) == home / "bin/trivydb-exporter"); + CHECK(config.security.backends.at("trivy").refreshMode == OsvRefreshMode::PERIODIC); + CHECK(config.security.backends.at("trivy").refreshIntervalSeconds == 321); + REQUIRE(config.security.backends.contains("gh-advisory")); + CHECK(config.security.backends.at("gh-advisory").feedUrl == "https://codeload.github.com/github/advisory-database/tar.gz/refs/heads/main"); + CHECK(config.security.backends.at("gh-advisory").refreshMode == OsvRefreshMode::ALWAYS); + CHECK(config.security.backends.at("gh-advisory").refreshIntervalSeconds == 654); CHECK(config.reports.enabled); CHECK(config.reports.format == ReportFormat::JSON); CHECK(std::filesystem::path(config.reports.outputPath) == home / "reports/reqpack.json"); @@ -555,6 +573,11 @@ TEST_CASE("configuration defaults build version and user agent from release id", CHECK(config.version == reqpack_build_release_id()); CHECK(config.downloader.userAgent == reqpack_user_agent()); + REQUIRE(config.security.backends.contains("trivy")); + CHECK(config.security.backends.at("trivy").dbRepositories == std::vector{ + "mirror.gcr.io/aquasec/trivy-db:2", + "ghcr.io/aquasecurity/trivy-db:2", + }); } TEST_CASE("configuration resolves execution jobs from fixed and max modes", "[unit][configuration][execution]") { diff --git a/tests/unit/core/download/downloader_tests.cpp b/tests/unit/core/download/downloader_tests.cpp index e7d59f4..b6df553 100644 --- a/tests/unit/core/download/downloader_tests.cpp +++ b/tests/unit/core/download/downloader_tests.cpp @@ -344,6 +344,32 @@ TEST_CASE("downloader copies bundled plugin directories and skips git metadata", CHECK_FALSE(std::filesystem::exists(targetDirectory / ".git")); } +TEST_CASE("downloader copies bundled plugin from repo root letter-grouped layout", "[unit][downloader][service]") { + TempDir tempDir{"reqpack-downloader-grouped-bundle-copy"}; + const std::string script = "return { getName = function() return 'huggingface' end }\n"; + const std::filesystem::path repositoryRoot = tempDir.path() / "repo"; + const std::filesystem::path bundleRoot = write_plugin_bundle(repositoryRoot / "rqp-plugins" / "h", "huggingface", script); + write_file(bundleRoot / "lib" / "helper.txt", "helper\n"); + + ReqPackConfig config = make_downloader_test_config(tempDir.path()); + config.registry.sources["huggingface"] = RegistrySourceEntry{ + .source = repositoryRoot.string(), + .alias = false, + .description = "huggingface bundle", + }; + + RegistryDatabase database(config); + REQUIRE(database.ensureReady()); + + const std::filesystem::path targetDirectory = tempDir.path() / "plugins" / "huggingface"; + Downloader downloader(&database, config); + REQUIRE(downloader.downloadPlugin("huggingface")); + + CHECK(read_file(targetDirectory / "run.lua") == script); + CHECK(read_file(targetDirectory / "metadata.json").find("\"name\": \"huggingface\"") != std::string::npos); + CHECK(read_file(targetDirectory / "lib" / "helper.txt") == "helper\n"); +} + TEST_CASE("downloader materializes run.lua bundles and removes stale bootstrap files", "[unit][downloader][service]") { TempDir tempDir{"reqpack-downloader-run-bundle"}; diff --git a/tests/unit/core/export/audit_exporter_tests.cpp b/tests/unit/core/export/audit_exporter_tests.cpp index 41db3b6..375db94 100644 --- a/tests/unit/core/export/audit_exporter_tests.cpp +++ b/tests/unit/core/export/audit_exporter_tests.cpp @@ -12,6 +12,7 @@ #include "core/export/audit_exporter.h" #include "output/logger.h" +#include "test_helpers.h" namespace { @@ -57,63 +58,6 @@ class StaticMetadataProvider final : public PluginMetadataProvider { } }; -class ScopedColumnsEnv { -public: - explicit ScopedColumnsEnv(const char* value) { - if (const char* existing = std::getenv("COLUMNS")) { - hadValue_ = true; - previousValue_ = existing; - } - - if (value != nullptr) { - ::setenv("COLUMNS", value, 1); - } else { - ::unsetenv("COLUMNS"); - } - } - - ~ScopedColumnsEnv() { - if (hadValue_) { - ::setenv("COLUMNS", previousValue_.c_str(), 1); - } else { - ::unsetenv("COLUMNS"); - } - } - -private: - bool hadValue_ = false; - std::string previousValue_; -}; - -class ScopedEnvVar { -public: - ScopedEnvVar(const char* key, const char* value) : key_(key) { - if (const char* existing = std::getenv(key_)) { - hadValue_ = true; - previousValue_ = existing; - } - - if (value != nullptr) { - ::setenv(key_, value, 1); - } else { - ::unsetenv(key_); - } - } - - ~ScopedEnvVar() { - if (hadValue_) { - ::setenv(key_, previousValue_.c_str(), 1); - } else { - ::unsetenv(key_); - } - } - -private: - const char* key_; - bool hadValue_ = false; - std::string previousValue_; -}; - std::string read_file(const std::filesystem::path& path) { std::ifstream input(path, std::ios::binary); REQUIRE(input.is_open()); @@ -186,7 +130,7 @@ class ScopedLoggerDisplay { } // namespace TEST_CASE("audit exporter renders default table output", "[unit][audit][export]") { - ScopedColumnsEnv columns{"72"}; + ScopedEnvVar columns{"COLUMNS", "72"}; ScopedEnvVar noColor{"NO_COLOR", "1"}; ScopedEnvVar forceColor{"FORCE_COLOR", nullptr}; AuditExporter exporter; @@ -215,7 +159,7 @@ TEST_CASE("audit exporter renders default table output", "[unit][audit][export]" } TEST_CASE("audit exporter colorizes severity in terminal table output", "[unit][audit][export]") { - ScopedColumnsEnv columns{"72"}; + ScopedEnvVar columns{"COLUMNS", "72"}; ScopedEnvVar noColor{"NO_COLOR", nullptr}; ScopedEnvVar forceColor{"FORCE_COLOR", "1"}; AuditExporter exporter; @@ -227,7 +171,7 @@ TEST_CASE("audit exporter colorizes severity in terminal table output", "[unit][ } TEST_CASE("audit exporter supports no-wrap and wide table flags", "[unit][audit][export]") { - ScopedColumnsEnv columns{"72"}; + ScopedEnvVar columns{"COLUMNS", "72"}; ScopedEnvVar noColor{"NO_COLOR", "1"}; ScopedEnvVar forceColor{"FORCE_COLOR", nullptr}; AuditExporter exporter; @@ -255,7 +199,7 @@ TEST_CASE("audit exporter supports no-wrap and wide table flags", "[unit][audit] } TEST_CASE("audit exporter keeps file exports plain even when color forced", "[unit][audit][export]") { - ScopedColumnsEnv columns{"72"}; + ScopedEnvVar columns{"COLUMNS", "72"}; ScopedEnvVar noColor{"NO_COLOR", nullptr}; ScopedEnvVar forceColor{"FORCE_COLOR", "1"}; AuditExporter exporter; diff --git a/tests/unit/core/export/sbom_exporter_tests.cpp b/tests/unit/core/export/sbom_exporter_tests.cpp index 97bb35d..0986a52 100644 --- a/tests/unit/core/export/sbom_exporter_tests.cpp +++ b/tests/unit/core/export/sbom_exporter_tests.cpp @@ -12,6 +12,7 @@ #include "core/export/sbom_exporter.h" #include "output/logger.h" +#include "test_helpers.h" namespace { @@ -57,63 +58,6 @@ class StaticMetadataProvider final : public PluginMetadataProvider { } }; -class ScopedColumnsEnv { -public: - explicit ScopedColumnsEnv(const char* value) { - if (const char* existing = std::getenv("COLUMNS")) { - hadValue_ = true; - previousValue_ = existing; - } - - if (value != nullptr) { - ::setenv("COLUMNS", value, 1); - } else { - ::unsetenv("COLUMNS"); - } - } - - ~ScopedColumnsEnv() { - if (hadValue_) { - ::setenv("COLUMNS", previousValue_.c_str(), 1); - } else { - ::unsetenv("COLUMNS"); - } - } - -private: - bool hadValue_ = false; - std::string previousValue_; -}; - -class ScopedEnvVar { -public: - ScopedEnvVar(const char* key, const char* value) : key_(key) { - if (const char* existing = std::getenv(key_)) { - hadValue_ = true; - previousValue_ = existing; - } - - if (value != nullptr) { - ::setenv(key_, value, 1); - } else { - ::unsetenv(key_); - } - } - - ~ScopedEnvVar() { - if (hadValue_) { - ::setenv(key_, previousValue_.c_str(), 1); - } else { - ::unsetenv(key_); - } - } - -private: - const char* key_; - bool hadValue_ = false; - std::string previousValue_; -}; - std::string read_file(const std::filesystem::path& path) { std::ifstream input(path, std::ios::binary); REQUIRE(input.is_open()); @@ -172,7 +116,7 @@ Graph make_graph() { } // namespace TEST_CASE("sbom exporter renders default table output", "[unit][sbom][export]") { - ScopedColumnsEnv columns{"72"}; + ScopedEnvVar columns{"COLUMNS", "72"}; ScopedEnvVar noColor{"NO_COLOR", "1"}; ScopedEnvVar forceColor{"FORCE_COLOR", nullptr}; SbomExporter exporter; @@ -190,7 +134,7 @@ TEST_CASE("sbom exporter renders default table output", "[unit][sbom][export]") } TEST_CASE("sbom exporter colorizes terminal table output", "[unit][sbom][export]") { - ScopedColumnsEnv columns{"72"}; + ScopedEnvVar columns{"COLUMNS", "72"}; ScopedEnvVar noColor{"NO_COLOR", nullptr}; ScopedEnvVar forceColor{"FORCE_COLOR", "1"}; SbomExporter exporter; @@ -203,7 +147,7 @@ TEST_CASE("sbom exporter colorizes terminal table output", "[unit][sbom][export] } TEST_CASE("sbom exporter supports no-wrap and wide table flags", "[unit][sbom][export]") { - ScopedColumnsEnv columns{"72"}; + ScopedEnvVar columns{"COLUMNS", "72"}; ScopedEnvVar noColor{"NO_COLOR", "1"}; ScopedEnvVar forceColor{"FORCE_COLOR", nullptr}; SbomExporter exporter; diff --git a/tests/unit/core/host/host_info_tests.cpp b/tests/unit/core/host/host_info_tests.cpp index 5587bf4..ec5e201 100644 --- a/tests/unit/core/host/host_info_tests.cpp +++ b/tests/unit/core/host/host_info_tests.cpp @@ -6,6 +6,7 @@ #include #include "core/host/host_info.h" +#include "test_helpers.h" namespace { @@ -30,29 +31,6 @@ class TempDir { std::filesystem::path path_; }; -class ScopedEnvVar { -public: - ScopedEnvVar(std::string name, std::string value) - : name_(std::move(name)) { - if (const char* existing = std::getenv(name_.c_str())) { - previous_ = std::string(existing); - } - ::setenv(name_.c_str(), value.c_str(), 1); - } - - ~ScopedEnvVar() { - if (previous_.has_value()) { - ::setenv(name_.c_str(), previous_->c_str(), 1); - } else { - ::unsetenv(name_.c_str()); - } - } - -private: - std::string name_; - std::optional previous_; -}; - void write_file(const std::filesystem::path& path, const std::string& content) { std::filesystem::create_directories(path.parent_path()); std::ofstream output(path, std::ios::binary); diff --git a/tests/unit/core/planning/planner_core_tests.cpp b/tests/unit/core/planning/planner_core_tests.cpp index dc1d01d..326bc6e 100644 --- a/tests/unit/core/planning/planner_core_tests.cpp +++ b/tests/unit/core/planning/planner_core_tests.cpp @@ -4,6 +4,30 @@ #include #include "core/planning/planner_core.h" +#include "core/planning/planner_platform_policy.h" + +TEST_CASE("planner platform policy recognizes nix install system", "[unit][planner][platform]") { + CHECK(planner_platform::isNixInstallSystem("nix")); + CHECK_FALSE(planner_platform::isNixInstallSystem("npm")); + CHECK_FALSE(planner_platform::isNixInstallSystem("choco")); +} + +TEST_CASE("planner platform schedule edge policy matches compile-time host", "[unit][planner][platform]") { +#if defined(_WIN32) + CHECK(planner_platform::reorderNonNixBeforeNix); + CHECK(planner_platform::needsNonNixBeforeNixBarrier()); + CHECK(planner_platform::softSkipNixInstalls()); + CHECK(planner_platform::shouldIgnoreScheduleEdge("nix", "npm")); + CHECK_FALSE(planner_platform::shouldIgnoreScheduleEdge("npm", "eslint")); + CHECK_FALSE(planner_platform::shouldIgnoreScheduleEdge("choco", "nix")); + CHECK_FALSE(planner_platform::shouldIgnoreScheduleEdge("nix", "nix")); +#else + CHECK_FALSE(planner_platform::reorderNonNixBeforeNix); + CHECK_FALSE(planner_platform::needsNonNixBeforeNixBarrier()); + CHECK_FALSE(planner_platform::softSkipNixInstalls()); + CHECK_FALSE(planner_platform::shouldIgnoreScheduleEdge("nix", "npm")); +#endif +} TEST_CASE("planner expands configured system aliases and preserves unknown systems", "[unit][planner][alias]") { const std::vector requests{ diff --git a/tests/unit/core/security/gh_advisory_vulnerability_sync_service_tests.cpp b/tests/unit/core/security/gh_advisory_vulnerability_sync_service_tests.cpp new file mode 100644 index 0000000..0eafd7c --- /dev/null +++ b/tests/unit/core/security/gh_advisory_vulnerability_sync_service_tests.cpp @@ -0,0 +1,88 @@ +#include + +#include +#include +#include +#include + +#include "core/security/gh_advisory_vulnerability_sync_service.h" +#include "core/security/vulnerability_database.h" + +namespace { + +class TempDir { +public: + explicit TempDir(const std::string& prefix) + : path_(std::filesystem::temp_directory_path() / + (prefix + "-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()))) { + std::filesystem::create_directories(path_); + } + + ~TempDir() { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + const std::filesystem::path& path() const { + return path_; + } + +private: + std::filesystem::path path_; +}; + +void write_file(const std::filesystem::path& path, const std::string& content) { + std::filesystem::create_directories(path.parent_path()); + std::ofstream output(path, std::ios::binary); + REQUIRE(output.is_open()); + output << content; +} + +} // namespace + +TEST_CASE("gh-advisory sync imports OSV advisories from local fixture tree", "[unit][gh-advisory][security]") { + TempDir tempDir{"reqpack-gh-advisory-sync"}; + const std::filesystem::path sourceRoot = tempDir.path() / "advisory-db" / "advisories" / "github-reviewed" / "2026" / "01"; + write_file(sourceRoot / "GHSA-demo.json", R"({ + "id": "GHSA-jfh8-c2jp-5v3q", + "modified": "2026-01-01T00:00:00Z", + "published": "2026-01-01T00:00:00Z", + "summary": "demo advisory", + "affected": [ + { + "package": {"ecosystem": "Maven", "name": "org.apache.logging.log4j:log4j-core"}, + "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "2.0-beta9"}, {"fixed": "2.15.0"}]}] + }, + { + "package": {"ecosystem": "pip", "name": "urllib3"}, + "versions": ["1.26.18"] + } + ] + })"); + + ReqPackConfig config; + config.security.cachePath = (tempDir.path() / "security-cache").string(); + config.security.indexPath = (tempDir.path() / "security-index").string(); + config.security.backends["gh-advisory"].feedUrl = "file://" + (tempDir.path() / "advisory-db").string(); + config.security.backends["gh-advisory"].refreshMode = OsvRefreshMode::ALWAYS; + + GhAdvisoryVulnerabilitySyncService syncService({"maven", "pip"}, config); + const std::vector findings = syncService.ensureReady(); + CHECK(findings.empty()); + + ReqPackConfig mavenDbConfig = config; + mavenDbConfig.security.osvDatabasePath = (tempDir.path() / "security-index" / "Maven").string(); + VulnerabilityDatabase mavenDatabase(mavenDbConfig); + REQUIRE(mavenDatabase.ensureReady()); + const std::vector mavenIds = mavenDatabase.advisoryIdsForPackage("Maven", "org.apache.logging.log4j:log4j-core"); + REQUIRE(mavenIds.size() == 1); + CHECK(mavenIds[0] == "GHSA-jfh8-c2jp-5v3q"); + + ReqPackConfig pipDbConfig = config; + pipDbConfig.security.osvDatabasePath = (tempDir.path() / "security-index" / "pip").string(); + VulnerabilityDatabase pipDatabase(pipDbConfig); + REQUIRE(pipDatabase.ensureReady()); + const std::vector pipIds = pipDatabase.advisoryIdsForPackage("pip", "urllib3"); + REQUIRE(pipIds.size() == 1); + CHECK(pipIds[0] == "GHSA-jfh8-c2jp-5v3q"); +} diff --git a/tests/unit/core/security/security_gateway_service_tests.cpp b/tests/unit/core/security/security_gateway_service_tests.cpp index 491ee74..4e38096 100644 --- a/tests/unit/core/security/security_gateway_service_tests.cpp +++ b/tests/unit/core/security/security_gateway_service_tests.cpp @@ -1,12 +1,17 @@ #include #include +#include #include #include #include +#include +#include #include #include "core/security/security_gateway_service.h" +#include "core/security/vulnerability_database.h" +#include "test_helpers.h" namespace { @@ -124,3 +129,144 @@ TEST_CASE("security gateway merges configured backends with discovered security- return finding.kind == "sync_error" && finding.message == "failed to populate requested security ecosystems"; })); } + +TEST_CASE("security provider plugin name acts as direct gateway system", "[unit][security_gateway_service]") { + TempDir tempDir{"reqpack-security-gateway-direct-provider"}; + ReqPackConfig config = hermetic_security_config(tempDir.path()); + + StaticMetadataProvider metadataProvider; + metadataProvider.metadata["snyk"].role = "security-provider"; + metadataProvider.metadata["snyk"].ecosystemScopes = {"Maven"}; + + SecurityGatewayService gateway(nullptr, &metadataProvider, config); + + CHECK(gateway.isGatewaySystem("snyk")); + CHECK(gateway.configuredGatewayNames().contains("snyk")); +} + +TEST_CASE("security gateway executes snyk backend for maven ecosystem", "[unit][security_gateway_service]") { + TempDir tempDir{"reqpack-security-gateway-snyk"}; + const std::filesystem::path apiRoot = tempDir.path() / "snyk-api" / "orgs" / "org-1"; + const std::filesystem::path exportFile = tempDir.path() / "export.csv"; + write_file(apiRoot / "export.json", R"({"export_id":"exp-1"})"); + write_file(apiRoot / "jobs" / "export" / "exp-1.json", R"({"status":"FINISHED"})"); + write_file(apiRoot / "export" / "exp-1.json", std::string{"{"} + "\"download_url\":\"file://" + exportFile.string() + "\"}"); + write_file(exportFile, + "PROBLEM_ID,PROBLEM_TITLE,CVE,PACKAGE_NAME_AND_VERSION,SEMVER_VULNERABLE_RANGE,ISSUE_SEVERITY,NVD_SCORE,SNYK_CVSS_SCORE,UPDATED_AT,FIXED_IN_VERSION,PRODUCT_NAME\n" + "SNYK-JAVA-LOG4J-1,Remote code execution,CVE-2026-1,pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1,\"[,2.15.0)\",critical,9.8,9.8,2026-01-01T00:00:00Z,2.15.0,Snyk Open Source\n" + ); + + ReqPackConfig config; + config.security.indexPath = (tempDir.path() / "security-index").string(); + config.security.cachePath = (tempDir.path() / "security-cache").string(); + config.security.backends["snyk"].apiBaseUrl = "file://" + (tempDir.path() / "snyk-api").string(); + config.security.backends["snyk"].apiVersion = "2024-10-15"; + config.security.backends["snyk"].tokenEnv = "REQPACK_TEST_SNYK_TOKEN"; + config.security.backends["snyk"].orgId = "org-1"; + config.security.backends["snyk"].dataset = "issues"; + config.security.backends["snyk"].refreshMode = OsvRefreshMode::ALWAYS; + ScopedEnvVar token{"REQPACK_TEST_SNYK_TOKEN", "token-1"}; + + StaticMetadataProvider metadataProvider; + metadataProvider.metadata["snyk"].role = "security-provider"; + metadataProvider.metadata["maven"].osvEcosystem = "Maven"; + + SecurityGatewayService gateway(nullptr, &metadataProvider, config); + const std::vector findings = gateway.executeGatewayRequest( + ActionType::INSTALL, + "snyk", + {Package{.action = ActionType::INSTALL, .system = "snyk", .name = "maven"}} + ); + + CHECK(findings.empty()); + + ReqPackConfig dbConfig = config; + dbConfig.security.osvDatabasePath = (tempDir.path() / "security-index" / "Maven").string(); + VulnerabilityDatabase database(dbConfig); + REQUIRE(database.ensureReady()); + const std::vector ids = database.advisoryIdsForPackage("Maven", "org.apache.logging.log4j:log4j-core"); + REQUIRE(ids.size() == 1); + CHECK(ids[0] == "SNYK-JAVA-LOG4J-1"); +} + +TEST_CASE("security gateway executes trivy backend for maven ecosystem", "[unit][security_gateway_service]") { + TempDir tempDir{"reqpack-security-gateway-trivy"}; + const std::filesystem::path trivyRoot = tempDir.path() / "trivy-db"; + const std::filesystem::path helperPath = tempDir.path() / "trivy-helper.sh"; + write_file(trivyRoot / "metadata.json", R"({"Version":2})"); + write_file(trivyRoot / "trivy.db", "fixture"); + write_file(helperPath, + "#!/bin/sh\n" + "printf '%s\\n' '{\"ecosystem\":\"Maven\",\"packageName\":\"org.apache.logging.log4j:log4j-core\",\"advisoryId\":\"CVE-2021-44228\",\"aliases\":[\"GHSA-jfh8-c2jp-5v3q\"],\"summary\":\"Remote code execution\",\"description\":\"desc\",\"severity\":\"CRITICAL\",\"score\":10.0,\"references\":[\"https://example.test/CVE-2021-44228\"],\"modified\":\"2021-12-10T00:00:00Z\",\"published\":\"2021-12-10T00:00:00Z\",\"ranges\":[{\"introduced\":\"2.0-beta9\",\"fixed\":\"2.15.0\"}]}'\n" + ); + REQUIRE(std::system((std::string{"chmod +x "} + helperPath.string()).c_str()) == 0); + + ReqPackConfig config; + config.security.indexPath = (tempDir.path() / "security-index").string(); + config.security.cachePath = (tempDir.path() / "security-cache").string(); + config.security.backends["trivy"].dbRepositories = {"file://" + trivyRoot.string()}; + config.security.backends["trivy"].helperPath = helperPath.string(); + config.security.backends["trivy"].refreshMode = OsvRefreshMode::ALWAYS; + + StaticMetadataProvider metadataProvider; + metadataProvider.metadata["trivy"].role = "security-provider"; + metadataProvider.metadata["maven"].osvEcosystem = "Maven"; + metadataProvider.metadata["maven"].versionComparator.profile = "maven-comparable"; + + SecurityGatewayService gateway(nullptr, &metadataProvider, config); + const std::vector findings = gateway.executeGatewayRequest( + ActionType::INSTALL, + "trivy", + {Package{.action = ActionType::INSTALL, .system = "trivy", .name = "maven"}} + ); + + CHECK(findings.empty()); + + ReqPackConfig dbConfig = config; + dbConfig.security.osvDatabasePath = (tempDir.path() / "security-index" / "Maven").string(); + VulnerabilityDatabase database(dbConfig); + REQUIRE(database.ensureReady()); + const std::vector ids = database.advisoryIdsForPackage("Maven", "org.apache.logging.log4j:log4j-core"); + REQUIRE(ids.size() == 1); + CHECK(ids[0] == "CVE-2021-44228"); +} + +TEST_CASE("security gateway executes gh-advisory backend for arbitrary available ecosystem", "[unit][security_gateway_service]") { + TempDir tempDir{"reqpack-security-gateway-gh-advisory"}; + const std::filesystem::path advisoryRoot = tempDir.path() / "gh-advisory-db" / "advisories" / "github-reviewed" / "2026" / "01"; + write_file(advisoryRoot / "GHSA-demo.json", R"({ + "id": "GHSA-pip-demo", + "modified": "2026-01-01T00:00:00Z", + "summary": "pip advisory", + "affected": [{ + "package": {"ecosystem": "pip", "name": "urllib3"}, + "versions": ["1.26.18"] + }] + })"); + + ReqPackConfig config; + config.security.indexPath = (tempDir.path() / "security-index").string(); + config.security.cachePath = (tempDir.path() / "security-cache").string(); + config.security.backends["gh-advisory"].feedUrl = "file://" + (tempDir.path() / "gh-advisory-db").string(); + config.security.backends["gh-advisory"].refreshMode = OsvRefreshMode::ALWAYS; + + StaticMetadataProvider metadataProvider; + metadataProvider.metadata["gh-advisory"].role = "security-provider"; + + SecurityGatewayService gateway(nullptr, &metadataProvider, config); + const std::vector findings = gateway.executeGatewayRequest( + ActionType::INSTALL, + "gh-advisory", + {Package{.action = ActionType::INSTALL, .system = "gh-advisory", .name = "pip"}} + ); + + CHECK(findings.empty()); + + ReqPackConfig dbConfig = config; + dbConfig.security.osvDatabasePath = (tempDir.path() / "security-index" / "pip").string(); + VulnerabilityDatabase database(dbConfig); + REQUIRE(database.ensureReady()); + const std::vector ids = database.advisoryIdsForPackage("pip", "urllib3"); + REQUIRE(ids.size() == 1); + CHECK(ids[0] == "GHSA-pip-demo"); +} diff --git a/tests/unit/core/security/snyk_vulnerability_sync_service_tests.cpp b/tests/unit/core/security/snyk_vulnerability_sync_service_tests.cpp new file mode 100644 index 0000000..dca15f0 --- /dev/null +++ b/tests/unit/core/security/snyk_vulnerability_sync_service_tests.cpp @@ -0,0 +1,47 @@ +#include + +#include "core/security/snyk_vulnerability_sync_service.h" + +TEST_CASE("snyk parser reads csv rows with quoted fields", "[unit][snyk][security]") { + const std::vector header = { + "PROBLEM_ID", + "PROBLEM_TITLE", + "CVE", + "PACKAGE_NAME_AND_VERSION", + "SEMVER_VULNERABLE_RANGE", + "ISSUE_SEVERITY", + "NVD_SCORE", + "SNYK_CVSS_SCORE", + "UPDATED_AT", + "FIXED_IN_VERSION", + "PRODUCT_NAME", + }; + const std::vector values = snyk_parse_csv_line( + "SNYK-JAVA-LOG4J-1,\"Remote code execution, critical\",CVE-2026-1,pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1,\"[,2.15.0)\",critical,9.8,9.8,2026-01-01T00:00:00Z,2.15.0,Snyk Open Source" + ); + + const auto row = snyk_parse_export_row(header, values); + REQUIRE(row.has_value()); + CHECK(row->problemId == "SNYK-JAVA-LOG4J-1"); + CHECK(row->problemTitle == "Remote code execution, critical"); + CHECK(row->cve == "CVE-2026-1"); + CHECK(row->packageNameAndVersion == "pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1"); + CHECK(row->fixedInVersion == "2.15.0"); + CHECK(row->snykCvssScore == Approx(9.8)); +} + +TEST_CASE("snyk parser converts maven purl to reqpack package key", "[unit][snyk][security]") { + const auto parsed = snyk_parse_maven_package_name_and_version( + "pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1" + ); + + REQUIRE(parsed.has_value()); + CHECK(parsed->name == "org.apache.logging.log4j:log4j-core"); + CHECK(parsed->version == "2.14.1"); +} + +TEST_CASE("snyk severity normalization maps moderate to medium and score fallback", "[unit][snyk][security]") { + CHECK(snyk_normalize_severity("moderate", 0.0) == "medium"); + CHECK(snyk_normalize_severity("", 9.1) == "critical"); + CHECK(snyk_normalize_severity("unknown", 7.4) == "high"); +} diff --git a/tests/unit/core/security/trivy_vulnerability_sync_service_tests.cpp b/tests/unit/core/security/trivy_vulnerability_sync_service_tests.cpp new file mode 100644 index 0000000..4435805 --- /dev/null +++ b/tests/unit/core/security/trivy_vulnerability_sync_service_tests.cpp @@ -0,0 +1,23 @@ +#include + +#include "core/security/trivy_vulnerability_sync_service.h" + +TEST_CASE("trivy helper parser reads ndjson records with ranges", "[unit][trivy][security]") { + const auto record = trivy_parse_helper_record( + R"({"ecosystem":"Maven","packageName":"org.apache.logging.log4j:log4j-core","advisoryId":"CVE-2021-44228","aliases":["GHSA-jfh8-c2jp-5v3q"],"summary":"Remote code execution in Log4j","description":"desc","severity":"CRITICAL","score":10.0,"references":["https://example.test/CVE-2021-44228"],"modified":"2021-12-10T00:00:00Z","published":"2021-12-10T00:00:00Z","versions":[],"ranges":[{"introduced":"2.0-beta9","fixed":"2.15.0"}]})" + ); + + REQUIRE(record.has_value()); + CHECK(record->ecosystem == "Maven"); + CHECK(record->packageName == "org.apache.logging.log4j:log4j-core"); + CHECK(record->advisoryId == "CVE-2021-44228"); + CHECK(record->severity == "critical"); + CHECK(record->score == Approx(10.0)); + REQUIRE(record->ranges.size() == 1); + CHECK(record->ranges[0].introduced == "2.0-beta9"); + CHECK(record->ranges[0].fixed == "2.15.0"); +} + +TEST_CASE("trivy helper parser rejects incomplete records", "[unit][trivy][security]") { + CHECK_FALSE(trivy_parse_helper_record(R"({"ecosystem":"Maven","packageName":"","advisoryId":"CVE-1"})").has_value()); +} diff --git a/tests/unit/output/logger/format_tests.cpp b/tests/unit/output/logger/format_tests.cpp index 8945d65..45bbc0d 100644 --- a/tests/unit/output/logger/format_tests.cpp +++ b/tests/unit/output/logger/format_tests.cpp @@ -11,6 +11,7 @@ #include "output/logger.h" #include "output/logger_core.h" #include "output/command_output.h" +#include "test_helpers.h" #include #include @@ -69,30 +70,6 @@ class RecordingDisplay final : public IDisplay { std::mutex mutex_; }; -class ScopedEnvVar { -public: - ScopedEnvVar(const char* name, const char* value) : name_(name) { - if (const char* existing = std::getenv(name_)) { - hadPrevious_ = true; - previous_ = existing; - } - ::setenv(name_, value, 1); - } - - ~ScopedEnvVar() { - if (hadPrevious_) { - ::setenv(name_, previous_.c_str(), 1); - } else { - ::unsetenv(name_); - } - } - -private: - const char* name_; - bool hadPrevious_{false}; - std::string previous_; -}; - class TestColorDisplay final : public ColorDisplay { public: using ColorDisplay::ColorDisplay; diff --git a/tests/unit/output/run_result_json_tests.cpp b/tests/unit/output/run_result_json_tests.cpp new file mode 100644 index 0000000..54a30af --- /dev/null +++ b/tests/unit/output/run_result_json_tests.cpp @@ -0,0 +1,35 @@ +#include + +#include "output/run_result_json.h" + +TEST_CASE("run result json groups packages by system with status and errors", "[unit][output][json]") { + RunResultJsonDocument document; + document.ok = false; + document.dryRun = false; + document.items = { + RunResultJsonItem{.system = "npm", .name = "eslint", .version = "9.0.0", .status = "installed"}, + RunResultJsonItem{.system = "npm", .name = "vitest", .version = "3.3.3", .status = "failed", .errorMessage = "plugin action failed"}, + RunResultJsonItem{.system = "apt", .name = "curl", .version = "8.0.0", .status = "installed"}, + }; + + const std::string json = serialize_run_result_json(document); + CHECK(json.find("\"ok\":false") != std::string::npos); + CHECK(json.find("\"dryRun\":false") != std::string::npos); + CHECK(json.find("\"npm\":[") != std::string::npos); + CHECK(json.find("\"name\":\"eslint\"") != std::string::npos); + CHECK(json.find("\"version\":\"9.0.0\"") != std::string::npos); + CHECK(json.find("\"status\":\"installed\"") != std::string::npos); + CHECK(json.find("\"name\":\"vitest\"") != std::string::npos); + CHECK(json.find("\"status\":\"failed\"") != std::string::npos); + CHECK(json.find("\"errorMessage\":\"plugin action failed\"") != std::string::npos); + CHECK(json.find("\"apt\":[") != std::string::npos); + CHECK(json.find("\"name\":\"curl\"") != std::string::npos); +} + +TEST_CASE("run result status mapping covers install remove update and dry-run", "[unit][output][json]") { + CHECK(run_result_status_for_action(ActionType::INSTALL, true, false) == "installed"); + CHECK(run_result_status_for_action(ActionType::REMOVE, true, false) == "removed"); + CHECK(run_result_status_for_action(ActionType::UPDATE, true, false) == "updated"); + CHECK(run_result_status_for_action(ActionType::INSTALL, true, true) == "planned"); + CHECK(run_result_status_for_action(ActionType::INSTALL, false, false) == "failed"); +} diff --git a/tools/trivydb-exporter/go.mod b/tools/trivydb-exporter/go.mod new file mode 100644 index 0000000..8a06ec5 --- /dev/null +++ b/tools/trivydb-exporter/go.mod @@ -0,0 +1,7 @@ +module reqpack/tools/trivydb-exporter + +go 1.23.0 + +require go.etcd.io/bbolt v1.4.3 + +require golang.org/x/sys v0.29.0 // indirect diff --git a/tools/trivydb-exporter/go.sum b/tools/trivydb-exporter/go.sum new file mode 100644 index 0000000..2719b63 --- /dev/null +++ b/tools/trivydb-exporter/go.sum @@ -0,0 +1,4 @@ +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/tools/trivydb-exporter/main.go b/tools/trivydb-exporter/main.go new file mode 100644 index 0000000..3ffe000 --- /dev/null +++ b/tools/trivydb-exporter/main.go @@ -0,0 +1,396 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "sort" + "strings" + + bolt "go.etcd.io/bbolt" +) + +type helperRange struct { + Introduced string `json:"introduced,omitempty"` + Fixed string `json:"fixed,omitempty"` + LastAffected string `json:"lastAffected,omitempty"` + Limit string `json:"limit,omitempty"` +} + +type helperRecord struct { + Ecosystem string `json:"ecosystem"` + PackageName string `json:"packageName"` + AdvisoryID string `json:"advisoryId"` + Aliases []string `json:"aliases,omitempty"` + Summary string `json:"summary,omitempty"` + Description string `json:"description,omitempty"` + Severity string `json:"severity,omitempty"` + Score float64 `json:"score,omitempty"` + References []string `json:"references,omitempty"` + Modified string `json:"modified,omitempty"` + Published string `json:"published,omitempty"` + Versions []string `json:"versions,omitempty"` + Ranges []helperRange `json:"ranges,omitempty"` +} + +type advisoryEntry struct { + VendorIDs []string `json:",omitempty"` + VulnerableVersions []string `json:",omitempty"` + PatchedVersions []string `json:",omitempty"` + AffectedVersion string `json:",omitempty"` + FixedVersion string `json:",omitempty"` + Severity int `json:",omitempty"` +} + +type vulnerabilityDetail struct { + CvssScore float64 `json:",omitempty"` + CvssVector string `json:",omitempty"` + CvssScoreV3 float64 `json:",omitempty"` + CvssVectorV3 string `json:",omitempty"` + CvssScoreV40 float64 `json:",omitempty"` + CvssVectorV40 string `json:",omitempty"` + Severity int `json:",omitempty"` + SeverityV3 int `json:",omitempty"` + SeverityV40 int `json:",omitempty"` + References []string `json:",omitempty"` + Title string `json:",omitempty"` + Description string `json:",omitempty"` + PublishedDate *string `json:",omitempty"` + LastModifiedDate *string `json:",omitempty"` +} + +var severityNames = []string{"unknown", "low", "medium", "high", "critical"} + +type multiFlag []string + +func (m *multiFlag) String() string { + return strings.Join(*m, ",") +} + +func (m *multiFlag) Set(value string) error { + *m = append(*m, value) + return nil +} + +func main() { + var dbPath string + var ecosystems multiFlag + flag.StringVar(&dbPath, "db-path", "", "path to trivy.db") + flag.Var(&ecosystems, "ecosystem", "requested ecosystem") + flag.Parse() + + if dbPath == "" { + fail(errors.New("--db-path required")) + } + if len(ecosystems) == 0 { + fail(errors.New("at least one --ecosystem required")) + } + + requested := map[string]struct{}{} + for _, eco := range ecosystems { + requested[strings.TrimSpace(eco)] = struct{}{} + } + + db, err := bolt.Open(dbPath, 0o644, &bolt.Options{ReadOnly: true}) + if err != nil { + fail(err) + } + defer db.Close() + + if err := db.View(func(tx *bolt.Tx) error { + return exportRecords(tx, requested) + }); err != nil { + fail(err) + } +} + +func fail(err error) { + _, _ = fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) +} + +func exportRecords(tx *bolt.Tx, requested map[string]struct{}) error { + return exportRecordsToWriter(tx, requested, os.Stdout) +} + +func exportRecordsToWriter(tx *bolt.Tx, requested map[string]struct{}, writer io.Writer) error { + advisoryRoot := tx.Bucket([]byte("advisory-detail")) + if advisoryRoot == nil { + return nil + } + vulnRoot := tx.Bucket([]byte("vulnerability-detail")) + + return advisoryRoot.ForEach(func(vulnIDBytes, _ []byte) error { + vulnID := string(vulnIDBytes) + vulnBucket := advisoryRoot.Bucket(vulnIDBytes) + if vulnBucket == nil { + return nil + } + + return vulnBucket.ForEach(func(bucketNameBytes, _ []byte) error { + bucketName := string(bucketNameBytes) + ecosystem, ok := mapBucketEcosystem(bucketName) + if !ok { + return nil + } + if _, wanted := requested[ecosystem]; !wanted { + return nil + } + + packageBucket := vulnBucket.Bucket(bucketNameBytes) + if packageBucket == nil { + return nil + } + details := readVulnerabilityDetail(vulnRoot, vulnID) + return packageBucket.ForEach(func(packageNameBytes, value []byte) error { + if value == nil { + return nil + } + var advisory advisoryEntry + if err := json.Unmarshal(value, &advisory); err != nil { + return err + } + record := buildRecord(ecosystem, string(packageNameBytes), vulnID, advisory, details) + encoded, err := json.Marshal(record) + if err != nil { + return err + } + _, err = fmt.Fprintln(writer, string(encoded)) + return err + }) + }) + }) +} + +func mapBucketEcosystem(bucket string) (string, bool) { + bucket = strings.ToLower(strings.TrimSpace(bucket)) + switch { + case strings.HasPrefix(bucket, "maven::"): + return "Maven", true + case strings.HasPrefix(bucket, "cargo::"): + return "crates.io", true + case strings.HasPrefix(bucket, "debian"): + return "Debian", true + case strings.HasPrefix(bucket, "red hat"), strings.HasPrefix(bucket, "rpm::"): + return "RPM", true + default: + return "", false + } +} + +func readVulnerabilityDetail(root *bolt.Bucket, vulnID string) map[string]vulnerabilityDetail { + result := map[string]vulnerabilityDetail{} + if root == nil { + return result + } + vulnBucket := root.Bucket([]byte(vulnID)) + if vulnBucket == nil { + return result + } + _ = vulnBucket.ForEach(func(sourceID, value []byte) error { + if value == nil { + return nil + } + var detail vulnerabilityDetail + if err := json.Unmarshal(value, &detail); err == nil { + result[string(sourceID)] = detail + } + return nil + }) + return result +} + +func buildRecord(ecosystem, packageName, vulnID string, adv advisoryEntry, details map[string]vulnerabilityDetail) helperRecord { + references := []string{} + title := "" + description := "" + modified := "" + published := "" + severity := severityString(adv.Severity) + score := 0.0 + + for _, source := range sortedKeys(details) { + detail := details[source] + references = append(references, detail.References...) + if title == "" && detail.Title != "" { + title = detail.Title + } + if description == "" && detail.Description != "" { + description = detail.Description + } + if published == "" && detail.PublishedDate != nil { + published = *detail.PublishedDate + } + if modified == "" && detail.LastModifiedDate != nil { + modified = *detail.LastModifiedDate + } + sev, sevScore := bestSeverity(detail) + if severityRank(sev) > severityRank(severity) { + severity = sev + } + if sevScore > score { + score = sevScore + } + } + + versions, ranges := normalizeVersions(adv) + return helperRecord{ + Ecosystem: ecosystem, + PackageName: packageName, + AdvisoryID: vulnID, + Aliases: uniqueStrings(adv.VendorIDs), + Summary: title, + Description: description, + Severity: severity, + Score: score, + References: uniqueStrings(references), + Modified: modified, + Published: published, + Versions: versions, + Ranges: ranges, + } +} + +func normalizeVersions(adv advisoryEntry) ([]string, []helperRange) { + versionsSet := map[string]struct{}{} + ranges := []helperRange{} + for _, expr := range adv.VulnerableVersions { + expr = strings.TrimSpace(expr) + if expr == "" { + continue + } + switch { + case strings.HasPrefix(expr, "="): + versionsSet[strings.TrimSpace(strings.TrimPrefix(expr, "="))] = struct{}{} + case strings.HasPrefix(expr, ">=") && !strings.Contains(expr, ","): + ranges = append(ranges, helperRange{Introduced: strings.TrimSpace(strings.TrimPrefix(expr, ">="))}) + case strings.Contains(expr, ","): + parts := strings.Split(expr, ",") + var r helperRange + for _, part := range parts { + part = strings.TrimSpace(part) + switch { + case strings.HasPrefix(part, ">="): + r.Introduced = strings.TrimSpace(strings.TrimPrefix(part, ">=")) + case strings.HasPrefix(part, "<="): + r.LastAffected = strings.TrimSpace(strings.TrimPrefix(part, "<=")) + case strings.HasPrefix(part, "<"): + r.Fixed = strings.TrimSpace(strings.TrimPrefix(part, "<")) + } + } + if r.Introduced != "" || r.Fixed != "" || r.LastAffected != "" || r.Limit != "" { + ranges = append(ranges, r) + } + default: + versionsSet[expr] = struct{}{} + } + } + if adv.AffectedVersion != "" { + versionsSet[adv.AffectedVersion] = struct{}{} + } + if adv.FixedVersion != "" && len(ranges) == 0 { + ranges = append(ranges, helperRange{Introduced: "0", Fixed: adv.FixedVersion}) + } + if len(ranges) == 0 { + for _, patched := range adv.PatchedVersions { + if patched != "" { + ranges = append(ranges, helperRange{Introduced: "0", Fixed: patched}) + } + } + } + versions := make([]string, 0, len(versionsSet)) + for version := range versionsSet { + versions = append(versions, version) + } + sort.Strings(versions) + return versions, ranges +} + +func sortedKeys[V any](input map[string]V) []string { + keys := make([]string, 0, len(input)) + for key := range input { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func uniqueStrings(values []string) []string { + seen := map[string]struct{}{} + result := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + sort.Strings(result) + return result +} + +func bestSeverity(detail vulnerabilityDetail) (string, float64) { + if detail.SeverityV40 > 0 { + return severityString(detail.SeverityV40), detail.CvssScoreV40 + } + if detail.SeverityV3 > 0 { + return severityString(detail.SeverityV3), detail.CvssScoreV3 + } + if detail.Severity > 0 { + return severityString(detail.Severity), detail.CvssScore + } + if detail.CvssScoreV40 > 0 { + return scoreSeverity(detail.CvssScoreV40), detail.CvssScoreV40 + } + if detail.CvssScoreV3 > 0 { + return scoreSeverity(detail.CvssScoreV3), detail.CvssScoreV3 + } + if detail.CvssScore > 0 { + return scoreSeverity(detail.CvssScore), detail.CvssScore + } + return "unknown", 0 +} + +func severityString(value int) string { + if value >= 0 && value < len(severityNames) { + return severityNames[value] + } + return "unknown" +} + +func scoreSeverity(score float64) string { + switch { + case score >= 9.0: + return "critical" + case score >= 7.0: + return "high" + case score >= 4.0: + return "medium" + case score > 0: + return "low" + default: + return "unknown" + } +} + +func severityRank(value string) int { + switch strings.ToLower(value) { + case "critical": + return 4 + case "high": + return 3 + case "medium": + return 2 + case "low": + return 1 + default: + return 0 + } +} diff --git a/tools/trivydb-exporter/main_test.go b/tools/trivydb-exporter/main_test.go new file mode 100644 index 0000000..21052bb --- /dev/null +++ b/tools/trivydb-exporter/main_test.go @@ -0,0 +1,236 @@ +package main + +import ( + "bytes" + "encoding/json" + "path/filepath" + "strings" + "testing" + + bolt "go.etcd.io/bbolt" +) + +func TestExportRecordsToWriterExportsRequestedEcosystem(t *testing.T) { + db := openTestDB(t) + defer db.Close() + + published := "2021-12-10T00:00:00Z" + modified := "2021-12-11T00:00:00Z" + seedAdvisoryData(t, db, + "CVE-2021-44228", + "maven::github.com/apache/logging-log4j2", + "org.apache.logging.log4j:log4j-core", + advisoryEntry{ + VendorIDs: []string{"GHSA-jfh8-c2jp-5v3q", "GHSA-jfh8-c2jp-5v3q"}, + VulnerableVersions: []string{"=2.14.1", ">=2.0-beta9,<2.15.0"}, + PatchedVersions: []string{"2.15.0"}, + Severity: 1, + }, + map[string]vulnerabilityDetail{ + "nvd": { + CvssScoreV3: 10.0, + SeverityV3: 4, + References: []string{"https://nvd.example/CVE-2021-44228", "https://nvd.example/CVE-2021-44228"}, + Title: "Remote code execution in Log4j", + Description: "Log4Shell", + PublishedDate: &published, + LastModifiedDate: &modified, + }, + }, + ) + seedAdvisoryData(t, db, + "CVE-2022-0001", + "cargo::crates.io/rand", + "rand", + advisoryEntry{VulnerableVersions: []string{"=0.8.5"}}, + nil, + ) + + var output bytes.Buffer + err := db.View(func(tx *bolt.Tx) error { + return exportRecordsToWriter(tx, map[string]struct{}{"Maven": {}}, &output) + }) + if err != nil { + t.Fatalf("exportRecordsToWriter returned error: %v", err) + } + + lines := nonEmptyLines(output.String()) + if len(lines) != 1 { + t.Fatalf("expected 1 output line, got %d: %q", len(lines), output.String()) + } + + var record helperRecord + if err := json.Unmarshal([]byte(lines[0]), &record); err != nil { + t.Fatalf("failed to decode helper record: %v", err) + } + + if record.Ecosystem != "Maven" { + t.Fatalf("unexpected ecosystem: %q", record.Ecosystem) + } + if record.PackageName != "org.apache.logging.log4j:log4j-core" { + t.Fatalf("unexpected package name: %q", record.PackageName) + } + if record.AdvisoryID != "CVE-2021-44228" { + t.Fatalf("unexpected advisory id: %q", record.AdvisoryID) + } + if record.Severity != "critical" { + t.Fatalf("unexpected severity: %q", record.Severity) + } + if record.Score != 10.0 { + t.Fatalf("unexpected score: %v", record.Score) + } + if record.Summary != "Remote code execution in Log4j" { + t.Fatalf("unexpected summary: %q", record.Summary) + } + if record.Description != "Log4Shell" { + t.Fatalf("unexpected description: %q", record.Description) + } + if record.Published != published { + t.Fatalf("unexpected published date: %q", record.Published) + } + if record.Modified != modified { + t.Fatalf("unexpected modified date: %q", record.Modified) + } + if joined := strings.Join(record.Aliases, ","); joined != "GHSA-jfh8-c2jp-5v3q" { + t.Fatalf("unexpected aliases: %q", joined) + } + if joined := strings.Join(record.References, ","); joined != "https://nvd.example/CVE-2021-44228" { + t.Fatalf("unexpected references: %q", joined) + } + if joined := strings.Join(record.Versions, ","); joined != "2.14.1" { + t.Fatalf("unexpected versions: %q", joined) + } + if len(record.Ranges) != 1 { + t.Fatalf("unexpected range count: %d", len(record.Ranges)) + } + if record.Ranges[0].Introduced != "2.0-beta9" || record.Ranges[0].Fixed != "2.15.0" { + t.Fatalf("unexpected range: %+v", record.Ranges[0]) + } +} + +func TestExportRecordsToWriterFallsBackToPatchedVersionRange(t *testing.T) { + db := openTestDB(t) + defer db.Close() + + seedAdvisoryData(t, db, + "CVE-2023-1234", + "debian 11", + "openssl", + advisoryEntry{ + PatchedVersions: []string{"1.1.1n"}, + }, + nil, + ) + + var output bytes.Buffer + err := db.View(func(tx *bolt.Tx) error { + return exportRecordsToWriter(tx, map[string]struct{}{"Debian": {}}, &output) + }) + if err != nil { + t.Fatalf("exportRecordsToWriter returned error: %v", err) + } + + lines := nonEmptyLines(output.String()) + if len(lines) != 1 { + t.Fatalf("expected 1 output line, got %d: %q", len(lines), output.String()) + } + + var record helperRecord + if err := json.Unmarshal([]byte(lines[0]), &record); err != nil { + t.Fatalf("failed to decode helper record: %v", err) + } + + if record.Ecosystem != "Debian" { + t.Fatalf("unexpected ecosystem: %q", record.Ecosystem) + } + if len(record.Ranges) != 1 { + t.Fatalf("unexpected range count: %d", len(record.Ranges)) + } + if record.Ranges[0].Introduced != "0" || record.Ranges[0].Fixed != "1.1.1n" { + t.Fatalf("unexpected range fallback: %+v", record.Ranges[0]) + } + if len(record.Versions) != 0 { + t.Fatalf("expected no explicit versions, got: %+v", record.Versions) + } +} + +func openTestDB(t *testing.T) *bolt.DB { + t.Helper() + path := filepath.Join(t.TempDir(), "trivy.db") + db, err := bolt.Open(path, 0o600, nil) + if err != nil { + t.Fatalf("failed to open test db: %v", err) + } + return db +} + +func seedAdvisoryData( + t *testing.T, + db *bolt.DB, + vulnID string, + bucketName string, + packageName string, + advisory advisoryEntry, + details map[string]vulnerabilityDetail, +) { + t.Helper() + err := db.Update(func(tx *bolt.Tx) error { + advisoryRoot, err := tx.CreateBucketIfNotExists([]byte("advisory-detail")) + if err != nil { + return err + } + vulnRoot, err := tx.CreateBucketIfNotExists([]byte("vulnerability-detail")) + if err != nil { + return err + } + + vulnBucket, err := advisoryRoot.CreateBucketIfNotExists([]byte(vulnID)) + if err != nil { + return err + } + pkgBucket, err := vulnBucket.CreateBucketIfNotExists([]byte(bucketName)) + if err != nil { + return err + } + encodedAdvisory, err := json.Marshal(advisory) + if err != nil { + return err + } + if err := pkgBucket.Put([]byte(packageName), encodedAdvisory); err != nil { + return err + } + + if len(details) == 0 { + return nil + } + detailBucket, err := vulnRoot.CreateBucketIfNotExists([]byte(vulnID)) + if err != nil { + return err + } + for sourceID, detail := range details { + encodedDetail, err := json.Marshal(detail) + if err != nil { + return err + } + if err := detailBucket.Put([]byte(sourceID), encodedDetail); err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatalf("failed to seed advisory data: %v", err) + } +} + +func nonEmptyLines(value string) []string { + parts := strings.Split(value, "\n") + lines := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + lines = append(lines, part) + } + } + return lines +}