diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..c3ef7e78 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +/.github/scripts/apply-nginx-http2-patch.sh text eol=lf +/patches/*.patch text eol=lf diff --git a/.github/scripts/apply-nginx-http2-patch.sh b/.github/scripts/apply-nginx-http2-patch.sh new file mode 100644 index 00000000..64dde04e --- /dev/null +++ b/.github/scripts/apply-nginx-http2-patch.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +# Apply the versioned host correction before configuring/building NGINX. +# The connector itself does not own NGINX's physical HTTP/2 connection events. +set -eu + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 /path/to/nginx/source" >&2 + exit 2 +fi + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +patch_file="$script_dir/../../patches/nginx-http2-close-stream-flush.patch" +nginx_dir=$(CDPATH= cd -- "$1" && pwd) + +if [ ! -f "$nginx_dir/src/http/v2/ngx_http_v2.c" ]; then + echo "NGINX HTTP/2 source is missing in $nginx_dir" >&2 + exit 1 +fi + +if ! command -v patch >/dev/null 2>&1; then + echo "The patch utility is required to apply the NGINX HTTP/2 fix" >&2 + exit 1 +fi + +# Match the source checkout's line endings without rewriting NGINX before the +# dry run. Windows checkouts may use CRLF even when this patch is stored as LF. +patch_input=$(mktemp "${TMPDIR:-/tmp}/modsecurity-nginx-h2-patch.XXXXXX") +trap 'rm -f "$patch_input"' 0 HUP INT TERM +cr=$(printf '\r') +if LC_ALL=C grep -qU "${cr}$" "$nginx_dir/src/http/v2/ngx_http_v2.c"; then + sed "s/${cr}\{0,1\}$/${cr}/" "$patch_file" > "$patch_input" +else + sed "s/${cr}$//" "$patch_file" > "$patch_input" +fi + +if patch --binary --dry-run --batch --forward --fuzz=0 -p1 -d "$nginx_dir" \ + -i "$patch_input" >/dev/null 2>&1; then + patch --binary --batch --forward --fuzz=0 --no-backup-if-mismatch -p1 \ + -d "$nginx_dir" -i "$patch_input" + echo "Applied NGINX HTTP/2 close-stream reset flush fix" +elif patch --binary --dry-run --batch --reverse --fuzz=0 -p1 -d "$nginx_dir" \ + -i "$patch_input" >/dev/null 2>&1; then + echo "NGINX HTTP/2 close-stream reset flush fix is already applied" +else + echo "NGINX HTTP/2 source does not match the versioned fix; review it before building" >&2 + exit 1 +fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0a994adc..40a51f04 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt-get update -y -qq - sudo apt-get install -y make autoconf automake make libyajl-dev libxml2-dev libmaxminddb-dev libcurl4-gnutls-dev $COMPDEPS + sudo apt-get install -y patch make autoconf automake make libyajl-dev libxml2-dev libmaxminddb-dev libcurl4-gnutls-dev $COMPDEPS - name: Get libModSecurity source uses: actions/checkout@v4 with: @@ -50,6 +50,8 @@ jobs: repository: nginx/nginx path: nginx fetch-depth: 1 + - name: Apply NGINX HTTP/2 reset delivery fix + run: sh ModSecurity-nginx/.github/scripts/apply-nginx-http2-patch.sh nginx - name: Get Nginx tests uses: actions/checkout@v4 with: @@ -58,7 +60,7 @@ jobs: fetch-depth: 1 - name: Copy ModSecurity-nginx tests to nginx/test run: | - cp ModSecurity-nginx/tests/* nginx/test + cp ModSecurity-nginx/tests/*.t ModSecurity-nginx/tests/*.pl nginx/test - name: Build nginx with ModSecurity-nginx module working-directory: nginx run: | @@ -234,148 +236,3 @@ jobs: echo "FAIL" exit 1 fi - - - build-windows: - runs-on: windows-2022 - defaults: - run: - shell: msys2 {0} - steps: - - name: Set up MSVC - uses: ilammy/msvc-dev-cmd@v1 - - name: Set up msys - uses: msys2/setup-msys2@v2 - with: - msystem: UCRT64 - path-type: inherit - - name: Get Nginx source - uses: actions/checkout@v4 - with: - repository: nginx/nginx - path: nginx - fetch-depth: 1 - - name: Get Nginx tests - uses: actions/checkout@v4 - with: - repository: nginx/nginx-tests - path: nginx/test - fetch-depth: 1 - - name: Set up third-party libraries - working-directory: nginx - run: | - mkdir objs - mkdir objs/lib - cd objs/lib - # Download to a file before extracting, rather than piping straight - # into tar: a retried download restarts from byte 0, which a pipe - # that tar already consumed can't rewind. The retry loop is at the - # shell level (one wget attempt per iteration, partial file removed - # in between) because wget's own --tries neither retries HTTP 5xx / - # DNS failures nor rewrites a partial -O file: it sends a Range - # request and appends, which corrupts the archive when the server - # ignores Range. -nv keeps wget's errors and one-line summary - # without the dot progress meter. - for url in \ - https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.39/pcre2-10.39.tar.gz \ - https://www.zlib.net/fossils/zlib-1.3.tar.gz \ - https://www.openssl.org/source/openssl-3.0.13.tar.gz \ - ; do - archive="$(basename "$url")" - for attempt in 1 2 3; do - rm -f "$archive" - if wget -nv --tries=1 --timeout=60 -O "$archive" "$url" \ - && tar -xzf "$archive"; then - break - fi - if [ "$attempt" -eq 3 ]; then - echo "::error::failed to fetch $url after $attempt attempts" - exit 1 - fi - echo "fetching $url failed (attempt $attempt), retrying in 5s" - sleep 5 - done - rm -f "$archive" - done - - name: Get libModSecurity source - uses: actions/checkout@v4 - with: - repository: owasp-modsecurity/ModSecurity - submodules: recursive - path: nginx/objs/lib/ModSecurity - fetch-depth: 1 - - name: Setup Conan - shell: cmd - run: | - pip3 install conan --upgrade - conan profile detect - - name: Retry rate-limited Conan downloads - shell: pwsh - run: | - # Conan builds the dependencies without a prebuilt binary from source, and the lmdb - # sources come from git.openldap.org, which rate limits CI runners with HTTP 429. - # By default Conan gives up after 2 retries, 5 seconds apart. - $conf = Join-Path (conan config home) "global.conf" - Add-Content $conf "tools.files.download:retry=10" - Add-Content $conf "tools.files.download:retry_wait=30" - - name: Build libModSecurity - working-directory: nginx/objs/lib/ModSecurity - shell: cmd - run: | - vcbuild.bat - - name: Get ModSecurity-nginx source code - uses: actions/checkout@v4 - with: - path: nginx/objs/lib/ModSecurity-nginx - - name: Copy ModSecurity-nginx tests to nginx/test - working-directory: nginx/test - run: | - cp ../objs/lib/ModSecurity-nginx/tests/* . - - name: Remove /usr/bin/link conflicting with MSVC link.exe - run: | - set -ex - which link - rm /usr/bin/link - - name: Build nginx w/ModSecurity-nginx module - working-directory: nginx - run: | - : # Windows native version of Perl is required by nginx build - export PATH=/c/Strawberry/perl/bin:$PATH - : # Set env variables to point to libModSecurity v3 include & lib directories - export MODSECURITY_INC=objs/lib/ModSecurity/headers - export MODSECURITY_LIB=objs/lib/ModSecurity/build/win32/build/Release - : # Copy libModSecurity.dll to objs dir (to be able to run nginx later) - cp $MODSECURITY_LIB/libModSecurity.dll objs - : # Configure nginx build w/ModSecurity-nginx module - auto/configure \ - --with-cc=cl \ - --with-debug \ - --prefix= \ - --conf-path=conf/nginx.conf \ - --pid-path=logs/nginx.pid \ - --http-log-path=logs/access.log \ - --error-log-path=logs/error.log \ - --sbin-path=nginx.exe \ - --http-client-body-temp-path=temp/client_body_temp \ - --http-proxy-temp-path=temp/proxy_temp \ - --http-fastcgi-temp-path=temp/fastcgi_temp \ - --http-scgi-temp-path=temp/scgi_temp \ - --http-uwsgi-temp-path=temp/uwsgi_temp \ - --with-cc-opt=-DFD_SETSIZE=1024 \ - --with-pcre=objs/lib/pcre2-10.39 \ - --with-zlib=objs/lib/zlib-1.3 \ - --with-openssl=objs/lib/openssl-3.0.13 \ - --with-openssl-opt=no-asm \ - --with-http_ssl_module \ - --with-http_v2_module \ - --with-http_auth_request_module \ - --add-module=objs/lib/ModSecurity-nginx - nmake - - name: Run ModSecurity-nginx tests - working-directory: nginx/test - shell: cmd # tests need to run on a "windows" shell - run: | - md temp - set TEMP=temp - set TEST_NGINX_BINARY=..\objs\nginx.exe - prove modsecurity*.t diff --git a/.github/workflows/test_new.yml b/.github/workflows/test_new.yml index 15f8cd1f..a2fe800f 100644 --- a/.github/workflows/test_new.yml +++ b/.github/workflows/test_new.yml @@ -10,6 +10,22 @@ on: permissions: {} jobs: + phase4-unit-tests: + permissions: + contents: read + runs-on: ubuntu-22.04 + strategy: + matrix: + compiler: [gcc, clang] + env: + CC: "/usr/bin/${{ matrix.compiler }}" + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Run standalone connector helper regressions + run: python3 -m unittest discover -s tests -p 'test_*runtime.py' -v + build-linux: name: linux-${{ matrix.compiler }}-nginx-${{ matrix.nginx_version }} permissions: @@ -18,9 +34,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04] + os: [ubuntu-24.04,ubuntu-26.04] compiler: [gcc, clang] - nginx_version: ["1.30.x", "1.29.x", "1.28.x"] + nginx_version: ["1.31.x", "1.30.x", "1.29.x"] env: NGINX_VERSION: ${{ matrix.nginx_version }} CC: "/usr/bin/${{ matrix.compiler }}" @@ -30,10 +46,11 @@ jobs: - name: Setup Dependencies run: | sudo apt-get update -y -qq - sudo apt-get install -y make autoconf automake pcre2-utils libpcre2-dev libyajl-dev libxml2-dev libmaxminddb-dev libcurl4-gnutls-dev $COMPDEPS + sudo apt-get install -y patch make autoconf automake pcre2-utils libpcre2-dev libyajl-dev libxml2-dev libmaxminddb-dev libcurl4-gnutls-dev $COMPDEPS - name: Get libModSecurity source uses: actions/checkout@v6 with: + persist-credentials: false repository: owasp-modsecurity/ModSecurity path: ModSecurity submodules: recursive @@ -45,8 +62,9 @@ jobs: ./configure --without-lmdb --prefix=/usr make -j $(nproc) sudo make install - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: + persist-credentials: false path: ModSecurity-nginx fetch-depth: 1 - name: Resolve latest Nginx version @@ -55,13 +73,18 @@ jobs: run: bash ModSecurity-nginx/.github/scripts/resolve-nginx-ref.sh - name: Get Nginx source - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: + persist-credentials: false repository: nginx/nginx ref: ${{ steps.nginx_ref.outputs.ref }} path: nginx fetch-depth: 0 + - name: Apply NGINX HTTP/2 reset delivery fix + if: ${{ matrix.os != 'ubuntu-24.04' || matrix.compiler != 'gcc' || matrix.nginx_version != '1.31.x' }} + run: sh ModSecurity-nginx/.github/scripts/apply-nginx-http2-patch.sh nginx + - name: Show Nginx source version working-directory: nginx shell: bash @@ -73,8 +96,9 @@ jobs: grep '#define NGINX_VERSION' src/core/nginx.h || true - name: Get Nginx tests - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: + persist-credentials: false repository: nginx/nginx-tests path: nginx/test fetch-depth: 0 @@ -96,6 +120,19 @@ jobs: make make modules sudo make install + - name: Verify queued reset on unpatched NGINX + if: ${{ matrix.os == 'ubuntu-24.04' && matrix.compiler == 'gcc' && matrix.nginx_version == '1.31.x' }} + working-directory: nginx/test + run: | + TEST_NGINX_BINARY=../objs/nginx TEST_NGINX_H2_RESET_EXPECT_QUEUED=1 prove -v modsecurity-phase4-h2-reset.t + - name: Apply HTTP/2 fix and rebuild after negative control + if: ${{ matrix.os == 'ubuntu-24.04' && matrix.compiler == 'gcc' && matrix.nginx_version == '1.31.x' }} + working-directory: nginx + run: | + sh ../ModSecurity-nginx/.github/scripts/apply-nginx-http2-patch.sh . + make + make modules + sudo make install - name: Run ModSecurity-nginx tests working-directory: nginx/test run: | @@ -271,12 +308,12 @@ jobs: name: windows-nginx-${{ matrix.nginx_version }} # windows-2025 now ships Visual Studio 2026 and CMake 4, with which Conan builds yajl # from source and its configure step fails on CMP0026 (owasp-modsecurity/ModSecurity#3604). - # Stay on windows-2022, like test.yml and libModSecurity's own CI, until that is fixed. + # Stay on windows-2022, like libModSecurity's own CI, until that is fixed. runs-on: windows-2022 strategy: fail-fast: false matrix: - nginx_version: ["1.30.x", "1.29.x", "1.28.x"] + nginx_version: ["1.31.x", "1.30.x", "1.29.x"] env: NGINX_VERSION: ${{ matrix.nginx_version }} defaults: @@ -290,9 +327,11 @@ jobs: with: msystem: UCRT64 path-type: inherit + install: patch - name: Get ModSecurity-nginx workflow scripts - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: + persist-credentials: false path: ModSecurity-nginx fetch-depth: 1 - name: Resolve latest Nginx version @@ -301,13 +340,17 @@ jobs: run: bash ModSecurity-nginx/.github/scripts/resolve-nginx-ref.sh - name: Get Nginx source - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: + persist-credentials: false repository: nginx/nginx ref: ${{ steps.nginx_ref.outputs.ref }} path: nginx fetch-depth: 0 + - name: Apply NGINX HTTP/2 reset delivery fix + run: sh ModSecurity-nginx/.github/scripts/apply-nginx-http2-patch.sh nginx + - name: Show Nginx source version working-directory: nginx shell: bash @@ -319,8 +362,9 @@ jobs: grep '#define NGINX_VERSION' src/core/nginx.h || true - name: Get Nginx tests - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: + persist-credentials: false repository: nginx/nginx-tests path: nginx/test fetch-depth: 0 @@ -349,7 +393,7 @@ jobs: # without the dot progress meter. for url in \ https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.47/pcre2-10.47.tar.gz \ - https://www.zlib.net/fossils/zlib-1.3.2.tar.gz \ + https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz \ https://www.openssl.org/source/openssl-3.6.2.tar.gz \ ; do archive="$(basename "$url")" @@ -369,8 +413,9 @@ jobs: rm -f "$archive" done - name: Get libModSecurity source - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: + persist-credentials: false repository: owasp-modsecurity/ModSecurity submodules: recursive path: nginx/objs/lib/ModSecurity @@ -395,8 +440,9 @@ jobs: run: | vcbuild.bat - name: Get ModSecurity-nginx source code - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: + persist-credentials: false path: nginx/objs/lib/ModSecurity-nginx - name: Copy ModSecurity-nginx tests to nginx/test working-directory: nginx/test diff --git a/.oss-scanner/Dockerfile b/.oss-scanner/Dockerfile index 4e20d3d6..8621c0a6 100644 --- a/.oss-scanner/Dockerfile +++ b/.oss-scanner/Dockerfile @@ -5,7 +5,7 @@ FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates git build-essential clang autoconf automake libtool pkg-config \ + ca-certificates git patch build-essential clang autoconf automake libtool pkg-config \ perl curl gdb valgrind \ libpcre2-dev pcre2-utils zlib1g-dev libyajl-dev libxml2-dev libmaxminddb-dev \ libcurl4-gnutls-dev \ @@ -39,7 +39,8 @@ RUN tag=$(git ls-remote --tags --refs https://github.com/coreruleset/coreruleset && git clone --depth 1 --branch "$tag" https://github.com/coreruleset/coreruleset /opt/coreruleset # nginx with this connector built in, as CI builds it. A failing step fails the build. -RUN cd /opt/nginx \ +RUN sh /src/.github/scripts/apply-nginx-http2-patch.sh /opt/nginx \ + && cd /opt/nginx \ && ./auto/configure --with-debug --with-http_v2_module --with-http_auth_request_module \ --with-cc-opt="$DEBUG_FLAGS" --add-module=/src \ && make -j"$(nproc)" diff --git a/CHANGES b/CHANGES index f6f2c7f7..afa9ca98 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,112 @@ +Unreleased +---------- + + - fix: bundle the NGINX HTTP/2 close-stream correction so a queued + RST_STREAM is flushed while sibling streams remain active. Apply it to + native CI hosts before building NGINX and document the host rebuild + requirement for static and dynamic connector installations. + [PR #377 - @Easton97-Jens] + - test: add a passive HTTP/2 multiplex regression for prompt Strict resets, + independent allowed-stream completion and same-connection reuse. One + Linux job first verifies the queued-reset behavior on an unpatched host, + then applies the core patch and runs the normal suite. + [PR #377 - @Easton97-Jens] + + - feat: add off (default), safe and strict Phase 4 intervention modes. + Safe and strict explicitly enable the additional connector policy; off + leaves ModSecurity response inspection and native interventions active. + [PR #377 - @Easton97-Jens] + - feat: handle late deny and redirect interventions explicitly. After + headers are sent, safe diagnoses optional event-log failures and continues; + strict terminates the response. A late intervention cannot replace an HTTP + status or body already sent. + [PR #377 - @Easton97-Jens] + - feat: add optional JSON-lines events with requested and actual actions, + response and WAF status, mode, reason, content type and rule ID. Redact + engine intervention messages and omit response-body bytes from the event. + [PR #377 - @Easton97-Jens] + - fix: inherit Phase 4 configuration and log files from http/server blocks + into nested locations, with validation of mode and size arguments. + [PR #377 - @Easton97-Jens] + - fix: leave response MIME selection and inspection limits to libModSecurity + through SecResponseBodyAccess, SecResponseBodyMimeType, + SecResponseBodyMimeTypesClear, SecResponseBodyLimit and + SecResponseBodyLimitAction. Earlier revisions of this PR used a connector + MIME-file directive and minimal mode; these are no longer supported. + Retain modsecurity_phase4_body_limit as a deprecated, + ignored compatibility setting with positive-value validation. + [PR #377 - @Easton97-Jens] + - fix: inspect memory and file-backed response buffers, reading file data + in bounded 32 KiB chunks while preserving the outgoing chain. Check byte + ranges, engine results, allocations and file reads. Saturate cumulative + counters without imposing a SIZE_MAX response limit; technical body + processing errors remain failures in every mode, including safe. + [PR #377 - @Easton97-Jens] + - fix: finalize response-body processing once per transaction and respect + main-request/subrequest ownership and end-of-stream markers. Correct + response-header error handling, redirect ownership, cleanup and internal + redirect recovery. + [PR #377 - @Easton97-Jens] + - fix: distinguish redirect and deny actions in events, escape JSON fields, + detect log-write failures, pass the correct NGINX integer type for WAF + status and normalize logged content types with fixed ASCII whitespace. + [PR #377 - @Easton97-Jens] + - fix: validate native intervention and body-append return values as closed + 0/1 contracts, respect nondisruptive results, release owned outputs on + every path and propagate early audit-log failures. + [PR #377 - @Easton97-Jens] + - fix: retain native response-limit Reject in every mode, including safe. + Emit 403 before headers or stop the committed response; never downgrade + the engine limit into a rule-policy event. Collect file interventions + after each chunk while preserving ProcessPartial and the outgoing chain. + [PR #377 - @Easton97-Jens] + - fix: keep response failures terminal across data and NULL-input filter + re-entry, permitting only one synchronous core error response. Accept + valid empty memory/EOS buffers without repeating phase completion. + [PR #377 - @Easton97-Jens] + - fix: preserve request-phase deny/redirect decisions across error_page + routing. Reuse the original transaction for inherited identical policy, + retire it when target policy changes, and audit every transaction owner. + Preserve only the captured local block error page without reinspection; + ordinary internal targets still run their own rules. + [PR #377 - @Easton97-Jens] + - fix: stream file-backed request bodies through bounded 32 KiB reads and + the same native 0/1 append contract as memory bodies. Preserve ProcessPartial + and pending native Reject through P2 dispatch without reopening temp paths. + [PR #377 - @Easton97-Jens] + - fix: identify native response-limit Reject at the actual append boundary, + independent of English log text. Suppress drained replacement data chains + and forward only fresh control markers, with once-only terminal delivery. + [PR #377 - @Easton97-Jens] + - fix: parse complete, unambiguous engine metadata records for rule IDs and + redacted presence summaries. Preserve JSON control characters and valid + UTF-8, escape invalid bytes, and avoid sanitizer buffer truncation. + [PR #377 - @Easton97-Jens] + - fix: attempt native audit logging once and retain failed native results + across re-entry without attempting another client response. + [PR #377 - @Easton97-Jens] + - fix: return NULL for transaction/context creation failures, release + partially created native owners, make cleanup idempotent and terminate + length-delimited custom transaction IDs before passing them to the SDK. + [PR #377 - @Easton97-Jens] + - fix: avoid synthetic Transfer-Encoding inspection headers on HTTP/2 and + HTTP/3, and validate response status before narrowing to the native int. + [PR #377 - @Easton97-Jens] + - test: extend regressions for modes, log inheritance, engine MIME selection, + Reject/ProcessPartial limits, large responses, file buffers, overflow, + subrequests, redirects, processing failures and proxy/HTTP/2 fixtures. + [PR #377 - @Easton97-Jens] + - ci: run the GCC/Clang Phase 4 helper tests in test_new.yml and disable + credential persistence for all ten checkouts in that workflow. Keep the + native Linux/Windows matrix there and remove the duplicate legacy + Windows job from test.yml. Integrate upstream dependency-download retry + fixes without restoring the removed job or credential persistence. + [PR #377 - @Easton97-Jens] + - docs: document Phase 4 timing limits, mode behavior, logging and engine + policy in English, with off/safe/strict and MIME-selection examples. + Remove the German translation and correct the README directive count. + [PR #377 - @Easton97-Jens] + v1.0.4 - 2025-May-21 -------------------- diff --git a/README.md b/README.md index 36b859c2..337f9853 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,15 @@ Apache. As a result, this current version has less dependencies, fewer bugs, and Before compile this software make sure that you have libmodsecurity installed. You can download it from the [ModSecurity git repository](https://github.com/owasp-modsecurity/ModSecurity). For information pertaining to the compilation and installation of libmodsecurity please consult the documentation provided along with it. +For HTTP/2 builds that use Phase 4 enforcement, apply the bundled +[NGINX reset-delivery correction](patches/README.md) to the NGINX sources before +configuration. This is a host-core change; rebuild the NGINX binary when using +a dynamic connector module as well: + +```sh +sh /path/to/ModSecurity-nginx/.github/scripts/apply-nginx-http2-patch.sh /path/to/nginx +``` + With libmodsecurity installed, you can proceed with the installation of the ModSecurity-nginx connector, which follows the nginx third-party module installation procedure. From the nginx source directory: ``` @@ -58,7 +67,7 @@ Further information about building nginx third-party modules is available [here] # Usage ModSecurity for nginx extends your nginx configuration directives. -It adds six new directives and they are: +It adds nine directives and they are: modsecurity ----------- @@ -201,6 +210,73 @@ using the same unique identificator. String can contain variables. + +modsecurity_phase4_mode +------------------------ +**syntax:** *modsecurity_phase4_mode off | safe | strict* + +**context:** *http, server, location* + +**default:** *off* + +Controls the connector's additional phase 4 intervention policy. `safe` logs a +late rule intervention as `log_only` and continues even if optional JSON +logging fails (with an nginx diagnostic); `strict` terminates the affected response once headers +are sent. Before headers are sent, both modes can apply the intervention's HTTP +status. Neither mode guarantees a clean 403 after headers have been sent. + +`off` uses the native intervention path and disables the connector's additional +phase 4 policy. It does not disable ModSecurity response-body +inspection; late native interventions can still fail the response. Configure +inspection with `SecResponseBodyAccess`, `SecResponseBodyMimeType`, and +`SecResponseBodyMimeTypesClear` in ModSecurity rules. + +modsecurity_phase4_body_limit +----------------------------- +**syntax:** *modsecurity_phase4_body_limit <size>* + +**context:** *http, server, location* + +**default:** *1m (1 MiB)* + +Deprecated compatibility setting. Positive values in bytes or nginx size +notation such as `256k` or `2m` remain accepted and inherited; zero remains +invalid. The historical default remains `1m`, but the value is ignored in +`off`, `safe`, and `strict` and does not impose a response-inspection limit. + +Configure the engine's inspection policy with `SecResponseBodyLimit` and +`SecResponseBodyLimitAction`. The connector retains checked byte accounting, +overflow guards, and bounded file reads, without global response buffering. + +modsecurity_phase4_log +---------------------- +**syntax:** *modsecurity_phase4_log <path>* + +**context:** *http, server, location* + +Sets the file used for phase 4 JSON event logging. + +### Phase 4 migration + +The initial phase 4 migration adopted behavior from +[ModSecurity-conector at b0f3bdab429717b5b0311c30c5b4d1153c672ac0](https://github.com/Easton97-Jens/ModSecurity-conector/tree/b0f3bdab429717b5b0311c30c5b4d1153c672ac0/connectors/nginx/src), +adapted for this standalone nginx module. It does not require that repository's +multi-connector runtime. Response-inspection limit ownership now follows +[ModSecurity-conector at 820b6975495bdf0f90aca67eee86e27a3b7d329b](https://github.com/Easton97-Jens/ModSecurity-conector/blob/820b6975495bdf0f90aca67eee86e27a3b7d329b/docs/phase4-mode-budget.md): +libModSecurity owns the inspection limits in every valid phase 4 mode. + +The earlier migration changed configuration compatibility: `minimal` is rejected, +the default changed from `safe` to `off`, and +`modsecurity_phase4_content_types_file` is removed. Set `safe` or `strict` +explicitly when migrating, and move MIME selection into ModSecurity rules. +The latest limit-ownership update retains those modes and their late policies. +Existing `modsecurity_phase4_body_limit` values no longer limit inspection; +migrate that policy to `SecResponseBodyLimit` and `SecResponseBodyLimitAction`. +See the [Phase 4 documentation](docs/phase4-handling.en.md) for the behavior, +limits, logging schema, and examples. The documentation describes the source +implementation; test results from the source repository are not validation of +this migration. + modsecurity_use_error_log ------------------------- **syntax:** *modsecurity_use_error_log on | off* diff --git a/docs/examples/phase4-engine-mime.conf b/docs/examples/phase4-engine-mime.conf new file mode 100644 index 00000000..2b8f7461 --- /dev/null +++ b/docs/examples/phase4-engine-mime.conf @@ -0,0 +1,14 @@ +# ModSecurity rules file: engine response-body MIME additions and limit policy +# Load with modsecurity_rules_file alongside your other rules. +# This is not an nginx include and not a connector MIME allowlist. +# To replace the engine MIME list, load a separate reset BEFORE this file: +# modsecurity_rules 'SecResponseBodyMimeTypesClear'; +# modsecurity_rules_file /etc/modsecurity/phase4-engine-mime.conf; +# Keep the reset in a separate rule load from the MIME additions. +# Adapt this list to the response types your application needs to inspect. + +SecResponseBodyAccess On +# Example engine inspection limit; independent of modsecurity_phase4_mode. +SecResponseBodyLimit 1048576 +SecResponseBodyLimitAction ProcessPartial +SecResponseBodyMimeType text/html text/plain application/json application/problem+json application/xml application/xhtml+xml diff --git a/docs/examples/phase4-off.conf b/docs/examples/phase4-off.conf new file mode 100644 index 00000000..2a803a38 --- /dev/null +++ b/docs/examples/phase4-off.conf @@ -0,0 +1,36 @@ +# nginx example: phase-4 mode "off" (the default) +# Uses native intervention handling. This does not disable ModSecurity or +# response-body inspection; late native interventions can fail the response. +# The connector's additional intervention policy is disabled. + +http { + upstream app_backend { + server 127.0.0.1:8081; + } + + server { + listen 80; + server_name example.local; + + modsecurity on; + modsecurity_phase4_mode off; + + # Illustrative rules; adapt to your application before deployment. + # MIME selection and inspection limits belong to the engine. + # The legacy modsecurity_phase4_body_limit setting is ignored. + # Reset the MIME list in a separate engine rule load. + modsecurity_rules 'SecResponseBodyMimeTypesClear'; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyLimit 1048576 + SecResponseBodyLimitAction ProcessPartial + SecResponseBodyMimeType text/html text/plain application/json + SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940101,phase:4,deny,log,status:403" + '; + + location / { + proxy_pass http://app_backend; + } + } +} diff --git a/docs/examples/phase4-safe.conf b/docs/examples/phase4-safe.conf new file mode 100644 index 00000000..a6eb94ac --- /dev/null +++ b/docs/examples/phase4-safe.conf @@ -0,0 +1,36 @@ +# nginx example: phase-4 mode "safe" (must be selected explicitly) +# Late interventions are logged as log_only. Genuine engine, I/O, and overflow +# failures still stop the affected response. + +http { + upstream app_backend { + server 127.0.0.1:8081; + } + + server { + listen 80; + server_name example.local; + + modsecurity on; + modsecurity_phase4_mode safe; + modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; + + # Illustrative rules; adapt to your application before deployment. + # MIME selection and inspection limits belong to the engine. + # The legacy modsecurity_phase4_body_limit setting is ignored. + # Reset the MIME list in a separate engine rule load. + modsecurity_rules 'SecResponseBodyMimeTypesClear'; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyLimit 1048576 + SecResponseBodyLimitAction ProcessPartial + SecResponseBodyMimeType text/html text/plain application/json + SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940102,phase:4,deny,log,status:403" + '; + + location / { + proxy_pass http://app_backend; + } + } +} diff --git a/docs/examples/phase4-strict.conf b/docs/examples/phase4-strict.conf new file mode 100644 index 00000000..d78598ac --- /dev/null +++ b/docs/examples/phase4-strict.conf @@ -0,0 +1,36 @@ +# nginx example: phase-4 mode "strict" +# Late interventions terminate the response. A clean HTTP 403 after headers +# have been sent is not guaranteed; clients may see a transport failure. + +http { + upstream app_backend { + server 127.0.0.1:8081; + } + + server { + listen 80; + server_name example.local; + + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; + + # Illustrative rules; adapt to your application before deployment. + # MIME selection and inspection limits belong to the engine. + # The legacy modsecurity_phase4_body_limit setting is ignored. + # Reset the MIME list in a separate engine rule load. + modsecurity_rules 'SecResponseBodyMimeTypesClear'; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyLimit 1048576 + SecResponseBodyLimitAction ProcessPartial + SecResponseBodyMimeType text/html text/plain application/json + SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940103,phase:4,deny,log,status:403" + '; + + location / { + proxy_pass http://app_backend; + } + } +} diff --git a/docs/phase4-handling.en.md b/docs/phase4-handling.en.md new file mode 100644 index 00000000..ea6f77b0 --- /dev/null +++ b/docs/phase4-handling.en.md @@ -0,0 +1,253 @@ +# ModSecurity-nginx: Phase 4 Handling (English) + +## 1) Scope and provenance + +Phase 4 inspects the response body. nginx can already have sent headers or body +bytes when a phase 4 rule requests a deny status or redirect. A late +intervention cannot reliably replace the HTTP status already sent to the +client. + +This branch adopts the relevant nginx implementation from +[Easton97-Jens/ModSecurity-conector, commit b0f3bdab429717b5b0311c30c5b4d1153c672ac0](https://github.com/Easton97-Jens/ModSecurity-conector/tree/b0f3bdab429717b5b0311c30c5b4d1153c672ac0/connectors/nginx/src). +The response-limit ownership update follows +[commit 820b6975495bdf0f90aca67eee86e27a3b7d329b](https://github.com/Easton97-Jens/ModSecurity-conector/blob/820b6975495bdf0f90aca67eee86e27a3b7d329b/docs/phase4-mode-budget.md). +Native-result, response completion, and lifecycle corrections follow +[commit f5523da040bd6325b369269ada074be6ba069c4f](https://github.com/Easton97-Jens/ModSecurity-conector/tree/f5523da040bd6325b369269ada074be6ba069c4f/connectors/nginx/src). +The review follow-up follows +[commit d420a12c4155a4bc2b9ea63ee5cef633085705e7](https://github.com/Easton97-Jens/ModSecurity-conector/tree/d420a12c4155a4bc2b9ea63ee5cef633085705e7/connectors/nginx/src). +These changes are adapted to this standalone module and preserve its phase 4 +JSON-lines schema. The source repository's multi-connector runtime and run +evidence are not imported. + +This document describes source behavior. It does not claim that runtime, +protocol, or integration tests have passed for this migration. + +## 2) Configuration and breaking changes + +All three connector directives below are valid in `http`, `server`, and +`location` contexts and inherit from the enclosing context. + +| Directive | Values | Default | +| --- | --- | --- | +| `modsecurity_phase4_mode` | `off`, `safe`, `strict` | `off` | +| `modsecurity_phase4_body_limit` | Deprecated, ignored compatibility value; positive bytes or nginx size, e.g. `256k`, `2m` | `1m` (1 MiB), ignored | +| `modsecurity_phase4_log` | File path for phase 4 JSON-lines events | No dedicated log | + +When migrating older phase 4 configurations, note: + +- `minimal` is no longer valid and is not an alias for `off`. +- The default is `off`; older configurations may have relied on `safe`. + Set `safe` or `strict` explicitly to enable the additional connector policy. +- `modsecurity_phase4_content_types_file` is removed and causes an nginx + configuration error. Move MIME selection into ModSecurity rules. +- `modsecurity_phase4_body_limit` is now deprecated and ignored in all valid + modes. Its positive-value parser, inheritance, and historical 1 MiB default + remain for compatibility; zero is still invalid. Move inspection limits into + `SecResponseBodyLimit` and `SecResponseBodyLimitAction`. + +Validate the migrated configuration with `nginx -t` in the deployment +environment before reloading nginx. + +## 3) Mode behavior and header timing + +`off` disables the connector's additional phase 4 intervention policy. +It **does not disable ModSecurity**, response-body inspection, or phase +4 rules. Native intervention handling remains active. If a native intervention +occurs after headers have been committed, the affected transport is failed +without trying to send another HTTP status; `off` is not a promise to deliver +every response or to reproduce every pre-PR intervention control-flow decision. + +For the additional policy in `safe` and `strict`: + +| Header state | `safe` | `strict` | +| --- | --- | --- | +| Headers not sent | Apply the intervention's status/redirect through the normal path; logged as `deny_status` | Same | +| Headers already sent | Attempt `log_only` logging and continue; diagnose optional event-log failures | Log `connection_abort` and terminate the response | + +`safe` only downgrades late **rule interventions**. An engine API failure, invalid +buffer, failed file read, body-processing allocation failure, or final +processing failure remains a failure. After headers are sent, optional phase 4 event-log allocation and write +failures are diagnosed but do not abort Safe delivery. Before commitment, +event-log failures retain the existing error path. It is not converted into successful +forwarding. Native response-limit Reject is not a rule-policy downgrade. + +Once a response has failed, later data callbacks and NULL-input flushes cannot +resume forwarding. Before headers are sent, nginx may render its synchronous +core error response once; subsequent callbacks retain the failure. Ordinary +successful downstream `NGX_AGAIN` handling remains unchanged. + +`strict` can produce a truncated response or a client/proxy transport error. +It cannot guarantee a clean 403, 401, 301, or 302 after headers have been sent. +Previously forwarded body bytes cannot be recalled. + +## 4) Engine inspection limits and streaming + +libModSecurity owns the WAF response-inspection byte limit through +`SecResponseBodyLimit` and `SecResponseBodyLimitAction` in every phase 4 mode. +The connector does not add a cumulative inspection ceiling for `safe`, +`strict`, or `off`, and does not reject a response merely because it exceeds +`modsecurity_phase4_body_limit`. + +For example, configure this engine policy in your ModSecurity rules: + +```apache +SecResponseBodyLimit 1048576 +SecResponseBodyLimitAction ProcessPartial +``` + +The 1 MiB value is an explicit example setting, not a new engine default. +`ProcessPartial` selects inspection of the portion within the engine limit; +`Reject` is the alternative engine action when rejection is required. Mode +selection does not override that engine policy. Native Reject is recognized +at a valid append boundary before EOS: a rule-free, disruptive native 403 +without a redirect. Recognition does not depend on the diagnostic language. It +produces 403 before headers are sent, or fails the transport after commitment, +in all modes including `safe`. It does not produce a `phase4_intervention` +rule-policy event; a fixed diagnostic is written to the nginx error log. + +The deprecated connector setting remains parseable and inherited, but its +value has no enforcement effect in any valid mode. A successful configuration +load therefore does not establish the old connector inspection limit. Migrate +that policy to the engine settings above. + +Cumulative byte counters saturate at `SIZE_MAX` without limiting delivery. +Large file buffers are streamed without converting their entire length to +`size_t`, including on 32-bit hosts. Genuine processing and file-read failures remain failures; +nonfatal engine `ProcessPartial` ingestion is not a connector failure. + +The connector does not globally buffer responses or reorder nginx body +chains. Memory buffers are inspected directly. File-only buffers are read +through a bounded 32 KiB scratch buffer, without loading the entire file into +memory or replacing the original outgoing chain. Pending interventions are +checked after every file chunk so Reject stops further reads immediately. +Genuine processing or I/O failures stop the affected response. + +The response body is finalized once per transaction: a main request uses +`last_buf`; a subrequest uses `last_in_chain`. Repeated filter calls after +finalization do not run phase 4 again. Empty memory/EOS buffers with both data +pointers NULL are valid; a missing single pointer or inverted range remains an +error. Native byte appends accept only 0/1 (`ProcessPartial` may return 0), +phase evaluation succeeds only with 1, and native intervention collection +accepts only 0/1 and honors the disruptive flag. + +## 5) MIME selection belongs to ModSecurity + +The engine decides response-body inspection using `SecResponseBodyAccess`, +`SecResponseBodyMimeType`, and `SecResponseBodyMimeTypesClear`. There is no +connector content-type allowlist or connector MIME-based downgrade. + +For example, reset the MIME list in one rule load, then add the selected types +in a separate load: + +```nginx +modsecurity_rules 'SecResponseBodyMimeTypesClear'; +modsecurity_rules ' + SecResponseBodyAccess On + SecResponseBodyMimeType text/html text/plain application/json +'; +``` + +The reset is separate because libModSecurity's +[merge implementation](https://raw.githubusercontent.com/owasp-modsecurity/ModSecurity/v3/master/headers/modsecurity/rules_set_properties.h) +can clear MIME values added in the same rule load. + +The standalone [engine MIME example](examples/phase4-engine-mime.conf) contains +MIME additions, response-body access, and an explicit engine limit policy. +It is a ModSecurity rules +file, not an nginx include. To replace the engine's MIME list with this file, +load the reset first, then the file, alongside your other rules: + +```nginx +modsecurity_rules 'SecResponseBodyMimeTypesClear'; +modsecurity_rules_file /etc/modsecurity/phase4-engine-mime.conf; +``` + +The complete nginx examples below configure MIME selection inline using the +same separate-load sequence. + +Engine MIME selection and inspection limits apply in every phase 4 mode; +there is no additional connector inspection budget. + +### HTTP/2 host-core correction + +A Strict intervention on an HTTP/2 response terminates that stream. The +bundled [NGINX core patch](../patches/README.md) ensures that its queued reset +is written promptly when an independent stream on the connection remains +active. The physical connection and allowed sibling streams remain usable. + +Apply the patch before configuring and rebuilding NGINX. This includes the +NGINX host binary when using a dynamic module; replacing only the connector +module cannot change an already installed host's core behavior. The helper +accepts an already-applied patch and rejects source drift without fuzzy +matching. Native CI applies the host correction to its Linux/Windows builds. + +The focused regression releases an auth-gated static Strict response while +an allowed proxied sibling is held open, then waits passively for the reset. +It sends no new +client input that could hide a stalled write. After the reset, the sibling +and a new stream must complete normally. A selected Linux job runs the +controlled queued-reset expectation against an unpatched host before +applying the patch and rebuilding it. + +## 6) Logging format and security boundary + +`modsecurity_phase4_log` preserves the existing JSON-lines intervention schema: + +- `event` (`phase4_intervention`), `uri`, `method`; +- `response_status`, `waf_status`, `content_type`, `header_sent`, `mode`; +- `wanted_action`, `actual_action`, `reason`; +- `intervention`, `rule_id`. + +The `actual_action` values remain `deny_status`, `log_only`, and +`connection_abort`; `mode` uses the current mode names. The intervention +message is redacted rather than copied from the engine, and response-body +payload is not added to this event. nginx's error log may also contain +diagnostics. Rule IDs and presence flags come only from complete, unambiguous +engine metadata records; reflected delimiter-like payloads are not trusted. +JSON preserves control characters through escapes and copies valid UTF-8; +invalid bytes are escaped individually. Redacted summaries have fixed complete +representations rather than variable-sized truncated buffers. + +`off` uses native intervention handling and does not produce these dedicated +policy events. A dedicated intervention event is not a complete inventory of +all engine, buffer, or I/O errors. + +Native audit logging is attempted once, including re-entry, and its return +value is checked. A failed audit attempt is retained as a failure without +trying to generate a new HTTP response. Every request-owned transaction gets +one audit attempt, including superseded owners and cleanup fallback after +NGINX clears the request pool pointer. An internal redirect reuses a transaction +only for identical inherited policy; a changed policy starts a fresh owner and +retires the old response scope. A native request block can render its captured +local `error_page` target without reblocking that same error response. Merely +setting `error_page` does not exempt an unrelated target from inspection. + +File-backed request bodies use their existing NGINX file descriptor and bounded +32 KiB reads, with the same 0/1 append contract as memory bodies. ProcessPartial +remains valid, and collected native request-limit Reject is retained until P2 +dispatch. Replaced response data is consumed without forwarding empty producer +buffers to the writer; terminal control markers are emitted once. + +## 7) Configuration examples and verification + +- [off](examples/phase4-off.conf): native intervention handling with explicit + engine inspection policy. +- [safe](examples/phase4-safe.conf): explicit late `log_only` policy. +- [strict](examples/phase4-strict.conf): late connection termination. + +- [engine MIME selection](examples/phase4-engine-mime.conf): ModSecurity + response-body configuration. + +All three examples configure the same 1 MiB engine limit with `ProcessPartial`; +that limit policy is independent of the connector mode. + +The nginx examples use `location /` and an example upstream at +`127.0.0.1:8081`. Adapt the listen address, upstream, log path, and rules to +your environment. The `sensitive-marker` rule is illustrative. + +Repository test endpoints such as `/phase4` are test fixtures, not required +production paths. Runtime checks should cover late deny/redirect behavior, +HTTP/1.1 and HTTP/2, file-backed responses, subrequests, repeated finalization, +engine-limit boundaries, responses above the ignored legacy setting, and +failure paths. Results from another repository or build +do not establish those behaviors for the deployed module. diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 00000000..8a2c25c5 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,33 @@ +# NGINX host correction for HTTP/2 reset delivery + +`nginx-http2-close-stream-flush.patch` schedules the physical HTTP/2 write +event when a stream closes with queued output while another stream remains +active. This lets a queued reset reach the client without waiting for another +socket read or the sibling stream to finish. The sibling connection remains +usable. + +The patch is copied from +[ModSecurity-test-Framework at 95f228936efe17fda24f21323ad87cd698940b48](https://github.com/Easton97-Jens/ModSecurity-test-Framework/blob/95f228936efe17fda24f21323ad87cd698940b48/ci/provisioning/patches/nginx/h2-close-stream-flush.patch). +It changes NGINX core code, not libModSecurity or the connector's rule policy. + +Before configuring and building NGINX, run from the connector checkout: + +```sh +sh .github/scripts/apply-nginx-http2-patch.sh /path/to/nginx/source +``` + +Then configure NGINX with `--with-http_v2_module` and this connector as usual. +For a dynamic connector, rebuild and deploy the patched NGINX host as well. +Replacing only the module leaves an existing host's HTTP/2 core unchanged. + +The helper requires GNU `patch`, rejects mismatched source +without fuzzy matching, and accepts an already-applied copy. Review this +dependency correction when changing NGINX versions or when an equivalent fix +lands upstream. NGINX binaries supplied by a distribution need the equivalent +host correction from their distributor or a locally rebuilt NGINX binary. + +CI applies the patch to its NGINX sources before native Linux/Windows builds. +A focused native regression checks that a Strict phase-4 reset arrives while +a separate allowed stream is still open, then verifies that the allowed +stream and another request complete on the same HTTP/2 connection. One Linux +job also runs the controlled unpatched baseline before applying the fix. diff --git a/patches/nginx-http2-close-stream-flush.patch b/patches/nginx-http2-close-stream-flush.patch new file mode 100644 index 00000000..d8530213 --- /dev/null +++ b/patches/nginx-http2-close-stream-flush.patch @@ -0,0 +1,15 @@ +--- a/src/http/v2/ngx_http_v2.c ++++ b/src/http/v2/ngx_http_v2.c +@@ -4803,3 +4803,12 @@ + if (h2c->processing || h2c->blocked) { ++ /* ++ * A reset queued while finalizing this stream must not wait for ++ * another stream to finish or for a new socket read event. Let the ++ * existing write handler flush it after this request is destroyed. ++ */ ++ if (h2c->last_out && !h2c->blocked) { ++ ngx_post_event(h2c->connection->write, &ngx_posted_events); ++ } ++ + return; + } diff --git a/src/ngx_http_modsecurity_access.c b/src/ngx_http_modsecurity_access.c index 47175e45..d55c4a2b 100644 --- a/src/ngx_http_modsecurity_access.c +++ b/src/ngx_http_modsecurity_access.c @@ -14,6 +14,7 @@ */ #include +#include #ifndef MODSECURITY_DDEBUG #define MODSECURITY_DDEBUG 0 @@ -22,6 +23,136 @@ #include "ngx_http_modsecurity_common.h" + +static ngx_int_t +ngx_http_modsecurity_request_failure(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) +{ + ctx->intervention_failed = 1; + ctx->intervention_triggered = 1; + if (r->header_sent) { + r->connection->error = 1; + return NGX_ERROR; + } + return NGX_HTTP_INTERNAL_SERVER_ERROR; +} + + +static int +ngx_http_modsecurity_request_intervention(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_int_t early_log) +{ + int result; + + ctx->request_intervention_active = 1; + result = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, + r, early_log); + ctx->request_intervention_active = 0; + return result; +} + + +static ngx_int_t +ngx_http_modsecurity_append_request_chunk(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, const u_char *data, size_t size) +{ + int result; + + result = msc_append_request_body(ctx->modsec_transaction, data, size); + /* Zero is the engine's ProcessPartial result, for memory and file data. */ + if ((result != 0 && result != 1) || + ngx_http_modsecurity_capture_request_body_intervention(r) != NGX_OK) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request body append failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } + return NGX_OK; +} + + +static ngx_int_t +ngx_http_modsecurity_append_request_body(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) +{ + ngx_chain_t *chain = r->request_body != NULL ? r->request_body->bufs : NULL; + ngx_int_t result; + + while (chain != NULL) { + if (chain->buf == NULL) { + return ngx_http_modsecurity_request_failure(r, ctx); + } + if (!ngx_buf_special(chain->buf)) { + if (chain->buf->pos == NULL || chain->buf->last == NULL || + chain->buf->last < chain->buf->pos) { + return ngx_http_modsecurity_request_failure(r, ctx); + } + result = ngx_http_modsecurity_append_request_chunk(r, ctx, + chain->buf->pos, (size_t) (chain->buf->last - chain->buf->pos)); + if (result != NGX_OK) { + return result; + } + } + if (chain->buf->last_buf) { + break; + } + chain = chain->next; + } + return NGX_OK; +} + + +static ngx_int_t +ngx_http_modsecurity_inspect_request_body_file(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) +{ + ngx_file_t *file = &r->request_body->temp_file->file; + ngx_file_info_t info; + off_t size = r->request_body->temp_file->offset; + off_t offset = 0; + size_t amount; + ssize_t read_size; + ngx_int_t result; + + /* Inspect the descriptor NGINX owns. The file-path SDK helper cannot + * distinguish ProcessPartial zero from its file I/O/allocation failures. */ + if (file->fd == NGX_INVALID_FILE || size < 0 || + ngx_fd_info(file->fd, &info) == NGX_FILE_ERROR || + !ngx_is_file(&info) || ngx_file_size(&info) != size) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: invalid request body file descriptor or size"); + return ngx_http_modsecurity_request_failure(r, ctx); + } + if (size > 0 && ctx->request_file_scratch == NULL) { + ctx->request_file_scratch = ngx_pnalloc(r->pool, 32768U); + if (ctx->request_file_scratch == NULL) { + return ngx_http_modsecurity_request_failure(r, ctx); + } + } + while (offset < size) { + amount = (uintmax_t) (size - offset) > 32768U + ? 32768U : (size_t) (size - offset); + read_size = ngx_read_file(file, ctx->request_file_scratch, amount, offset); + if (read_size < 0 || (size_t) read_size != amount) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request body file read is short or failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } + result = ngx_http_modsecurity_append_request_chunk(r, ctx, + ctx->request_file_scratch, amount); + if (result != NGX_OK) { + return result; + } + offset += (off_t) amount; + } + if (ngx_fd_info(file->fd, &info) == NGX_FILE_ERROR || + !ngx_is_file(&info) || ngx_file_size(&info) != size) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request body temp file size changed unexpectedly"); + return ngx_http_modsecurity_request_failure(r, ctx); + } + return NGX_OK; +} + void ngx_http_modsecurity_request_read(ngx_http_request_t *r) { @@ -33,7 +164,7 @@ ngx_http_modsecurity_request_read(ngx_http_request_t *r) r->main->count--; #endif - if (ctx->waiting_more_body) + if (ctx != NULL && ctx->waiting_more_body) { ctx->waiting_more_body = 0; r->write_event_handler = ngx_http_core_run_phases; @@ -51,6 +182,9 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) ngx_http_modsecurity_conf_t *mcf; mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + if (ngx_http_modsecurity_prepare_location(r, mcf) != NGX_OK) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } if (mcf == NULL || mcf->enable != 1) { dd("ModSecurity not enabled... returning"); return NGX_DECLINED; @@ -67,9 +201,17 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) dd("catching a new _access_ phase handler"); + /* Reuse only the same request's equivalent policy, or its own exact + * configured native-block error page. Other targets get a new owner. */ ctx = ngx_http_modsecurity_get_module_ctx(r); + if (ctx != NULL && ctx->r != r) { + ctx = NULL; + } + if (ctx != NULL && ctx->intervention_failed) { + return ngx_http_modsecurity_request_failure(r, ctx); + } - dd("recovering ctx: %p", ctx); + dd("current location ctx: %p", ctx); if (ctx == NULL) { @@ -164,8 +306,10 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) client_addr, client_port, server_addr, server_port); ngx_http_modsecurity_pcre_malloc_done(old_pool); - if (ret != 1){ - dd("Was not able to extract connection information."); + if (ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: connection phase processing failed"); + return ngx_http_modsecurity_request_failure(r, ctx); } /** * @@ -177,7 +321,10 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) * */ dd("Processing intervention with the connection information filled in"); - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 1); + ret = ngx_http_modsecurity_request_intervention(r, ctx, 1); + if (ret < 0) { + return ngx_http_modsecurity_request_failure(r, ctx); + } if (ret > 0) { ctx->intervention_triggered = 1; return ret; @@ -222,11 +369,19 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) return NGX_HTTP_INTERNAL_SERVER_ERROR; } old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_uri(ctx->modsec_transaction, n_uri, n_method, http_version); + ret = msc_process_uri(ctx->modsec_transaction, n_uri, n_method, http_version); ngx_http_modsecurity_pcre_malloc_done(old_pool); + if (ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: URI phase processing failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } dd("Processing intervention with the transaction information filled in (uri, method and version)"); - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 1); + ret = ngx_http_modsecurity_request_intervention(r, ctx, 1); + if (ret < 0) { + return ngx_http_modsecurity_request_failure(r, ctx); + } if (ret > 0) { ctx->intervention_triggered = 1; return ret; @@ -272,13 +427,18 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) */ old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_request_headers(ctx->modsec_transaction); + ret = msc_process_request_headers(ctx->modsec_transaction); ngx_http_modsecurity_pcre_malloc_done(old_pool); + if (ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request headers phase processing failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } dd("Processing intervention with the request headers information filled in"); - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 1); - if (r->error_page) { - return NGX_DECLINED; - } + ret = ngx_http_modsecurity_request_intervention(r, ctx, 1); + if (ret < 0) { + return ngx_http_modsecurity_request_failure(r, ctx); + } if (ret > 0) { ctx->intervention_triggered = 1; return ret; @@ -370,14 +530,11 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) if (ctx->waiting_more_body == 0) { int ret = 0; - int already_inspected = 0; dd("request body is ready to be processed"); r->write_event_handler = ngx_http_core_run_phases; - ngx_chain_t *chain = r->request_body->bufs; - /** * TODO: Speed up the analysis by sending chunk while they arrive. * @@ -387,48 +544,14 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) * function. */ - if (r->request_body->temp_file != NULL) { - ngx_str_t file_path = r->request_body->temp_file->file.name; - const char *file_name = ngx_str_to_char(file_path, r->pool); - if (file_name == (char*)-1) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - /* - * Request body was saved to a file, probably we don't have a - * copy of it in memory. - */ - dd("request body inspection: file -- %s", file_name); - - msc_request_body_from_file(ctx->modsec_transaction, file_name); - - already_inspected = 1; + if (r->request_body != NULL && r->request_body->temp_file != NULL) { + ret = ngx_http_modsecurity_inspect_request_body_file(r, ctx); } else { dd("inspection request body in memory."); + ret = ngx_http_modsecurity_append_request_body(r, ctx); } - - while (chain && !already_inspected) - { - u_char *data = chain->buf->pos; - - msc_append_request_body(ctx->modsec_transaction, data, - chain->buf->last - data); - - if (chain->buf->last_buf) { - break; - } - chain = chain->next; - -/* XXX: chains are processed one-by-one, maybe worth to pass all chains and then call intervention() ? */ - - /** - * ModSecurity may perform stream inspection on this buffer, - * it may ask for a intervention in consequence of that. - * - */ - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); - if (ret > 0) { - return ret; - } + if (ret != NGX_OK) { + return ret; } /** @@ -441,15 +564,21 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) /* XXX: once more -- is body can be modified ? content-length need to be adjusted ? */ old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_request_body(ctx->modsec_transaction); - ctx->request_body_processed = 1; + ret = msc_process_request_body(ctx->modsec_transaction); ngx_http_modsecurity_pcre_malloc_done(old_pool); + if (ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request body phase processing failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } + ctx->request_body_processed = 1; - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); - if (r->error_page) { - return NGX_DECLINED; - } + ret = ngx_http_modsecurity_request_intervention(r, ctx, 0); + if (ret < 0) { + return ngx_http_modsecurity_request_failure(r, ctx); + } if (ret > 0) { + ctx->intervention_triggered = 1; return ret; } } diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index 0c28e3c1..a9752611 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -14,6 +14,7 @@ */ #include +#include #ifndef MODSECURITY_DDEBUG #define MODSECURITY_DDEBUG 0 @@ -21,8 +22,16 @@ #include "ddebug.h" #include "ngx_http_modsecurity_common.h" +#include "ngx_http_modsecurity_log_metadata.h" + +/* Bounded, reusable materialization for file-only response buffers. */ +#define NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK 32768U static ngx_http_output_body_filter_pt ngx_http_next_body_filter; +static ngx_int_t ngx_http_modsecurity_process_response_intervention(ngx_http_request_t *r, ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf); +static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason); +static ngx_int_t ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf); +static ngx_str_t ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in); /* XXX: check behaviour on few body filters installed */ ngx_int_t @@ -34,153 +43,609 @@ ngx_http_modsecurity_body_filter_init(void) return NGX_OK; } -ngx_int_t -ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) +static ngx_int_t +ngx_http_modsecurity_phase4_error_gate(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) { - ngx_chain_t *chain = in; - ngx_http_modsecurity_ctx_t *ctx = NULL; -#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) - ngx_http_modsecurity_conf_t *mcf; - ngx_list_part_t *part = &r->headers_out.headers.part; - ngx_table_elt_t *data = part->elts; - ngx_uint_t i = 0; -#endif + if (!ctx->phase4_failed) { + return NGX_OK; + } + /* Only the synchronous core-generated error body may pass the gate. + * A subsequent flush or data callback cannot resume the failed chain. */ + if (ctx->phase4_error_emitting && r->filter_finalize) { + return NGX_DECLINED; + } + r->connection->error = 1; + return NGX_ERROR; +} - if (in == NULL) { - return ngx_http_next_body_filter(r, in); +static ngx_int_t +ngx_http_modsecurity_response_body_failure(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) +{ + ngx_int_t ret; + + ctx->phase4_append_active = 0; + if (ctx->phase4_failed) { + r->connection->error = 1; + return NGX_ERROR; } + ctx->phase4_failed = 1; + ctx->intervention_triggered = 1; + if (r->header_sent) { + r->connection->error = 1; + return NGX_ERROR; + } + ctx->phase4_error_emitting = 1; + ret = ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, + ctx->intervention_body_limit ? NGX_HTTP_FORBIDDEN : + NGX_HTTP_INTERNAL_SERVER_ERROR); + ctx->phase4_error_emitting = 0; + return ret; +} - ctx = ngx_http_modsecurity_get_module_ctx(r); +/* These are diagnostic counters, not inspection limits. Saturation must not + * interrupt a valid large response, including on a 32-bit size_t build. */ +static ngx_int_t +ngx_http_modsecurity_plan_limited_response_body( + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf, + size_t len, size_t *allowed) +{ + if (allowed == NULL) { + return NGX_ERROR; + } + *allowed = 0U; + if (ctx == NULL || mcf == NULL) { + return NGX_ERROR; + } + if (len == 0U) { + return NGX_OK; + } + ctx->response_body_seen = 1; + if (mcf->phase4_mode != NGX_HTTP_MODSEC_PHASE4_MODE_OFF && + mcf->phase4_mode != NGX_HTTP_MODSEC_PHASE4_MODE_SAFE && + mcf->phase4_mode != NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { + return NGX_ERROR; + } + if (ctx->response_body_bytes_inspected > ctx->response_body_bytes_seen) { + ctx->response_body_truncated = 1; + return NGX_ERROR; + } + if (len > SIZE_MAX - ctx->response_body_bytes_seen) { + ctx->response_body_bytes_seen = SIZE_MAX; + } else { + ctx->response_body_bytes_seen += len; + } + *allowed = len; + return NGX_OK; +} - dd("body filter, recovering ctx: %p", ctx); +static ngx_int_t +ngx_http_modsecurity_append_response_body_chunk( + ngx_http_modsecurity_ctx_t *ctx, u_char *data, size_t bytes) +{ + int ret; - if (ctx == NULL) { - return ngx_http_next_body_filter(r, in); + ctx->phase4_append_active = 0; + if (bytes == 0U) { + return NGX_OK; } + if (data == NULL) { + return NGX_ERROR; + } + ret = msc_append_response_body(ctx->modsec_transaction, data, bytes); + if (ret != 0 && ret != 1) { + return NGX_ERROR; + } + if (bytes > SIZE_MAX - ctx->response_body_bytes_inspected) { + ctx->response_body_bytes_inspected = SIZE_MAX; + } else { + ctx->response_body_bytes_inspected += bytes; + } + /* Keep the accepted append boundary live until its immediate collection. + * The engine evaluates response rules only in process_response_body(). */ + ctx->phase4_append_active = 1; + return NGX_OK; +} - if (ctx->intervention_triggered) { - return ngx_http_next_body_filter(r, in); +static ngx_int_t +ngx_http_modsecurity_append_limited_response_body( + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf, + u_char *data, size_t len) +{ + size_t allowed; + + if (ngx_http_modsecurity_plan_limited_response_body(ctx, mcf, len, + &allowed) != NGX_OK) { + return NGX_ERROR; } + return ngx_http_modsecurity_append_response_body_chunk(ctx, data, allowed); +} -#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) - mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); - if (mcf != NULL && mcf->sanity_checks_enabled != NGX_CONF_UNSET) - { -#if 0 - dd("dumping stored ctx headers"); - for (i = 0; i < ctx->sanity_headers_out->nelts; i++) - { - ngx_http_modsecurity_header_t *vals = ctx->sanity_headers_out->elts; - ngx_str_t *s2 = &vals[i].name, *s3 = &vals[i].value; - dd(" dump[%d]: name = '%.*s', value = '%.*s'", (int)i, - (int)s2->len, (char*)s2->data, - (int)s3->len, (char*)s3->data); +static ngx_int_t +ngx_http_modsecurity_append_file_response_body(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf, + ngx_buf_t *buffer) +{ + uintmax_t file_length; + uintmax_t remaining; + size_t chunk; + off_t file_offset; + ssize_t read_count; + ngx_int_t ret; + + if (buffer == NULL || buffer->file_pos < 0 || + buffer->file_last < buffer->file_pos) { + return NGX_ERROR; + } + file_length = (uintmax_t)buffer->file_last - (uintmax_t)buffer->file_pos; + if (ctx == NULL || mcf == NULL) { + return NGX_ERROR; + } + if (file_length == 0U) { + return NGX_OK; + } + if (buffer->file == NULL) { + return NGX_ERROR; + } + if (ctx->phase4_file_scratch == NULL) { + ctx->phase4_file_scratch = ngx_pnalloc(r->pool, + NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK); + if (ctx->phase4_file_scratch == NULL) { + return NGX_ERROR; } -#endif - /* - * Identify if there is a header that was not inspected by ModSecurity. - */ - int worth_to_fail = 0; - - for (i = 0; ; i++) - { - int found = 0; - ngx_uint_t j = 0; - ngx_table_elt_t *s1; - ngx_http_modsecurity_header_t *vals; - - if (i >= part->nelts) - { - if (part->next == NULL) { - break; - } - - part = part->next; - data = part->elts; - i = 0; - } + } + file_offset = buffer->file_pos; + remaining = file_length; + while (remaining > 0U) { + chunk = remaining > NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK + ? NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK : (size_t)remaining; + read_count = ngx_read_file(buffer->file, ctx->phase4_file_scratch, + chunk, file_offset); + if (read_count < 0 || (size_t)read_count != chunk) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, + read_count < 0 ? ngx_errno : 0, + "ModSecurity: file-backed response body read is short or failed"); + return NGX_ERROR; + } + if (ngx_http_modsecurity_append_limited_response_body(ctx, mcf, + ctx->phase4_file_scratch, chunk) != NGX_OK) { + return NGX_ERROR; + } + /* Native Reject may return a successful append with a pending + * intervention. Collect it before reading the next file chunk. */ + ret = ngx_http_modsecurity_process_response_intervention(r, ctx, mcf); + if (ret != NGX_OK || ctx->intervention_triggered) { + return ret; + } + file_offset += (off_t)chunk; + remaining -= chunk; + } + return NGX_OK; +} + +static ngx_int_t +ngx_http_modsecurity_append_response_body_buffer(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf, + ngx_buf_t *buffer) +{ + if (buffer == NULL) { + return NGX_ERROR; + } + /* Buffers backed by both memory and a file describe the same bytes. */ + if (ngx_buf_in_memory(buffer)) { + if (buffer->pos == NULL && buffer->last == NULL) { + return NGX_OK; + } + if (buffer->pos == NULL || buffer->last == NULL || + buffer->last < buffer->pos) { + return NGX_ERROR; + } + return ngx_http_modsecurity_append_limited_response_body(ctx, mcf, + buffer->pos, (size_t)(buffer->last - buffer->pos)); + } + if (buffer->in_file) { + return ngx_http_modsecurity_append_file_response_body(r, ctx, mcf, + buffer); + } + return NGX_OK; +} + +static ngx_int_t +ngx_http_modsecurity_process_response_intervention(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf) +{ + int ret; + + ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, + 0); + ctx->phase4_append_active = 0; + if (ret == 0) { + return NGX_OK; + } + if (ctx->intervention_failed || (ret < 0 && !ctx->intervention_late)) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_OFF) { + if (r->header_sent) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + ctx->intervention_triggered = 1; + /* Retain the native intervention failure path in off. */ + return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, + ret < 0 ? NGX_HTTP_INTERNAL_SERVER_ERROR : ret); + } + ret = ngx_http_modsecurity_phase4_handle_intervention(r, mcf); + if (ctx->intervention_triggered || ret != NGX_OK) { + return ret; + } + return NGX_OK; +} + +static ngx_int_t +ngx_http_modsecurity_process_final_response_body(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf) +{ + ngx_pool_t *old_pool; + int ret; + + if (ctx->phase4_processed) { + return NGX_OK; + } + ctx->phase4_append_active = 0; + ctx->phase4_processed = 1; + old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); + ret = msc_process_response_body(ctx->modsec_transaction); + ngx_http_modsecurity_pcre_malloc_done(old_pool); + if (ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: response body phase processing failed"); + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + return ngx_http_modsecurity_process_response_intervention(r, ctx, mcf); +} - vals = ctx->sanity_headers_out->elts; - s1 = &data[i]; - - /* - * Headers that were inspected by ModSecurity. - */ - while (j < ctx->sanity_headers_out->nelts) - { - ngx_str_t *s2 = &vals[j].name; - ngx_str_t *s3 = &vals[j].value; - - if (s1->key.len == s2->len && ngx_strncmp(s1->key.data, s2->data, s1->key.len) == 0) - { - if (s1->value.len == s3->len && ngx_strncmp(s1->value.data, s3->data, s1->value.len) == 0) - { - found = 1; - break; - } - } - j++; +#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) +static ngx_int_t +ngx_http_modsecurity_response_header_sanity(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf) +{ + ngx_list_part_t *part = &r->headers_out.headers.part; + ngx_table_elt_t *data = part->elts; + ngx_http_modsecurity_header_t *headers; + ngx_uint_t i = 0; + ngx_uint_t j; + + if (mcf->sanity_checks_enabled == NGX_CONF_UNSET) { + return NGX_OK; + } + if (ctx->sanity_headers_out == NULL) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + headers = ctx->sanity_headers_out->elts; + for (;;) { + while (i >= part->nelts) { + if (part->next == NULL) { + return NGX_OK; } - if (!found) { - dd("header: `%.*s' with value: `%.*s' was not inspected by ModSecurity", - (int) s1->key.len, - (const char *) s1->key.data, - (int) s1->value.len, - (const char *) s1->value.data); - worth_to_fail++; + part = part->next; + data = part->elts; + i = 0; + } + for (j = 0; j < ctx->sanity_headers_out->nelts; j++) { + if (data[i].key.len == headers[j].name.len && + ngx_strncmp(data[i].key.data, headers[j].name.data, + data[i].key.len) == 0 && + data[i].value.len == headers[j].value.len && + ngx_strncmp(data[i].value.data, headers[j].value.data, + data[i].value.len) == 0) { + break; } } - - if (worth_to_fail) - { - dd("%d header(s) were not inspected by ModSecurity, so exiting", worth_to_fail); - return ngx_http_filter_finalize_request(r, - &ngx_http_modsecurity_module, NGX_HTTP_INTERNAL_SERVER_ERROR); + if (j == ctx->sanity_headers_out->nelts) { + return ngx_http_modsecurity_response_body_failure(r, ctx); } + i++; } +} #endif - int is_request_processed = 0; - for (; chain != NULL; chain = chain->next) - { - u_char *data = chain->buf->pos; - int ret; +static ngx_int_t +ngx_http_modsecurity_discard_replaced_response_body(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_chain_t *in) +{ + ngx_chain_t *chain; + ngx_chain_t *out = NULL; + ngx_chain_t **tail = &out; + + for (chain = in; chain != NULL; chain = chain->next) { + ngx_buf_t *source = chain->buf; + ngx_chain_t *control; - msc_append_response_body(ctx->modsec_transaction, data, chain->buf->last - data); - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); - if (ret > 0) { - return ngx_http_filter_finalize_request(r, - &ngx_http_modsecurity_module, ret); + if (source == NULL) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + /* Consume producer-owned bytes without forwarding its now-empty + * memory/file link: NGINX's writer rejects those non-special links. */ + if (ngx_buf_in_memory(source)) { + source->pos = source->last; + } + if (source->in_file) { + source->file_pos = source->file_last; + } + if (ctx->replacement_body_terminal_sent || + !(source->flush || source->sync || source->last_buf || + source->last_in_chain)) { + continue; } + control = ngx_alloc_chain_link(r->pool); + if (control == NULL) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + control->buf = ngx_calloc_buf(r->pool); + if (control->buf == NULL) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + control->next = NULL; + control->buf->flush = source->flush; + control->buf->last_buf = source->last_buf && r == r->main; + control->buf->last_in_chain = source->last_in_chain || + (source->last_buf && r != r->main); + control->buf->sync = source->sync || control->buf->last_in_chain; + *tail = control; + tail = &control->next; + if (control->buf->last_buf || + (r != r->main && control->buf->last_in_chain)) { + /* Downstream can retain this chain on NGX_AGAIN. Claim EOS + * before dispatch so a later callback cannot duplicate it. */ + ctx->replacement_body_terminal_sent = 1; + } + } + if (out == NULL && !r->buffered && !r->connection->buffered) { + return NGX_OK; + } + return ngx_http_next_body_filter(r, out); +} -/* XXX: chain->buf->last_buf || chain->buf->last_in_chain */ - is_request_processed = chain->buf->last_buf; +ngx_int_t +ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + ngx_chain_t *chain; + ngx_http_modsecurity_ctx_t *ctx; + ngx_http_modsecurity_conf_t *mcf; + ngx_int_t ret; - if (is_request_processed) { - ngx_pool_t *old_pool; + ctx = ngx_http_modsecurity_get_module_ctx(r); + if (ctx == NULL) { + return ngx_http_next_body_filter(r, in); + } + ret = ngx_http_modsecurity_phase4_error_gate(r, ctx); + if (ret != NGX_OK) { + return ret == NGX_DECLINED ? ngx_http_next_body_filter(r, in) : ret; + } + if (ctx->response_replaced) { + return ngx_http_modsecurity_discard_replaced_response_body(r, ctx, in); + } + if (in == NULL) { + return ngx_http_next_body_filter(r, in); + } + if (ctx->intervention_triggered || ctx->phase4_processed) { + return ngx_http_next_body_filter(r, in); + } + mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + if (mcf == NULL) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } +#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) + ret = ngx_http_modsecurity_response_header_sanity(r, ctx, mcf); + if (ret != NGX_OK) { + return ret; + } +#endif + ctx->phase4_started = 1; + for (chain = in; chain != NULL; chain = chain->next) { + ret = ngx_http_modsecurity_append_response_body_buffer(r, ctx, mcf, + chain->buf); + if (ctx->intervention_triggered) { + return ret; + } + if (ret != NGX_OK) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: response body inspection append failed"); + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + ret = ngx_http_modsecurity_process_response_intervention(r, ctx, mcf); + if (ret != NGX_OK || ctx->intervention_triggered) { + return ret; + } + /* A main response may contain last_in_chain before the real EOS. + * Subrequests finish with last_in_chain instead of last_buf. */ + if (!(chain->buf->last_buf || + (r != r->main && chain->buf->last_in_chain))) { + continue; + } + ret = ngx_http_modsecurity_process_final_response_body(r, ctx, mcf); + if (ret != NGX_OK || ctx->intervention_triggered) { + return ret; + } + /* Subsequent flush/terminal links still belong to NGINX; never + * append bytes after the transaction has completed. */ + break; + } + return ngx_http_next_body_filter(r, in); +} - old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_response_body(ctx->modsec_transaction); - ngx_http_modsecurity_pcre_malloc_done(old_pool); +static ngx_int_t +ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, + ngx_http_modsecurity_conf_t *mcf) +{ + ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); + const char *wanted = "deny"; + ngx_int_t log_result; + ngx_int_t status; -/* XXX: I don't get how body from modsec being transferred to nginx's buffer. If so - after adjusting of nginx's - XXX: body we can proceed to adjust body size (content-length). see xslt_body_filter() for example */ - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); - if (ret > 0) { - return ret; - } - else if (ret < 0) { - return ngx_http_filter_finalize_request(r, - &ngx_http_modsecurity_module, NGX_HTTP_INTERNAL_SERVER_ERROR); + if (mcf == NULL || ctx == NULL) { + return NGX_ERROR; + } + if (ctx->last_intervention_status >= 300 && + ctx->last_intervention_status < 400) { + wanted = "redirect"; + } + if (ctx->phase4_headers_checked) { + return NGX_OK; + } + ctx->phase4_headers_checked = 1; + if (!r->header_sent) { + status = ctx->last_intervention_status >= 300 + ? ctx->last_intervention_status : NGX_HTTP_FORBIDDEN; + log_result = ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, + "deny_status", + "response_not_committed"); + ctx->intervention_triggered = 1; + if (log_result != NGX_OK) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, + status); + } + if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { + ctx->phase4_strict_abort = 1; + ctx->phase4_failed = 1; + ctx->intervention_triggered = 1; + r->connection->error = 1; + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "modsecurity phase4 intervention after headers sent, action=connection_abort, uri=\"%V\"", &r->uri); + (void)ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, + "connection_abort", "response_committed_strict"); + return NGX_ERROR; + } + log_result = ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, + "log_only", "response_committed_safe"); + if (log_result != NGX_OK) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "modsecurity optional phase4 intervention event could not be written"); + } + return NGX_OK; +} - } +static ngx_int_t +ngx_http_modsecurity_phase4_log_capacity(const size_t *lengths, size_t count, + size_t *capacity) +{ + size_t total = 512U; + size_t i; + + for (i = 0; i < count; i++) { + if (lengths[i] > SIZE_MAX - total) { + return NGX_ERROR; } + total += lengths[i]; + } + *capacity = total; + return NGX_OK; +} + +static ngx_int_t +ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason) +{ + u_char *p; + u_char *dbuf; + size_t lengths[9]; + size_t need; + ssize_t n; + ngx_str_t euri; + ngx_str_t emethod; + ngx_str_t ect; + ngx_str_t elog; + ngx_str_t erule; + ngx_str_t raw_log; + ngx_str_t slog; + ngx_str_t nct; + const char *mode = "off"; + const char *header_sent = r->header_sent ? "true" : "false"; + ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); + if (mcf == NULL || mcf->phase4_log_file == NULL || mcf->phase4_log_file->fd == NGX_INVALID_FILE) return NGX_OK; + ngx_http_modsecurity_json_escape(r->pool, &r->uri, &euri); + ngx_http_modsecurity_json_escape(r->pool, &r->method_name, &emethod); + nct = ngx_http_modsecurity_normalize_content_type(r->pool, r->headers_out.content_type); + ngx_http_modsecurity_json_escape(r->pool, &nct, &ect); + if (ctx) { + raw_log = ctx->last_intervention_log; + ngx_http_modsecurity_extract_rule_id(r->pool, &raw_log, &erule); + slog = ngx_http_modsecurity_sanitize_intervention(r->pool, raw_log); + ngx_http_modsecurity_json_escape(r->pool, &slog, &elog); + } else { + raw_log.len = 0; raw_log.data = (u_char *)""; + elog.len = 0; elog.data=(u_char*)""; + erule.len = 0; erule.data=(u_char*)""; + } + if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_SAFE) mode = "safe"; + else if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) mode = "strict"; + if (euri.data == NULL || emethod.data == NULL || nct.data == NULL || + ect.data == NULL || elog.data == NULL || erule.data == NULL) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "modsecurity phase4 log metadata allocation failed"); + return NGX_ERROR; + } + lengths[0] = euri.len; lengths[1] = emethod.len; + lengths[2] = ect.len; lengths[3] = elog.len; lengths[4] = erule.len; + lengths[5] = ngx_strlen(mode); lengths[6] = ngx_strlen(wanted); + lengths[7] = ngx_strlen(actual); lengths[8] = ngx_strlen(reason); + if (ngx_http_modsecurity_phase4_log_capacity(lengths, + sizeof(lengths) / sizeof(lengths[0]), &need) != NGX_OK) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "modsecurity phase4 log size overflow"); + return NGX_ERROR; } - if (!is_request_processed) - { - dd("buffer was not fully loaded! ctx: %p", ctx); + dbuf = ngx_pnalloc(r->pool, need); + if (dbuf == NULL) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, "modsecurity phase4 log allocation failed"); + return NGX_ERROR; } + p = ngx_snprintf(dbuf, need, + "{\"event\":\"phase4_intervention\",\"uri\":\"%V\",\"method\":\"%V\",\"response_status\":%ui,\"waf_status\":%i,\"content_type\":\"%V\",\"header_sent\":%s,\"mode\":\"%s\",\"wanted_action\":\"%s\",\"actual_action\":\"%s\",\"reason\":\"%s\",\"intervention\":\"%V\",\"rule_id\":\"%V\"}\n", + &euri, &emethod, (ngx_uint_t) r->headers_out.status, + ctx ? ctx->last_intervention_status : (ngx_int_t) 0, + &ect, header_sent, mode, wanted, actual, reason, &elog, &erule); + if (p == dbuf + need || p == dbuf || p[-1] != '\n') { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "modsecurity phase4 log serialization was truncated"); + return NGX_ERROR; + } + n = ngx_write_fd(mcf->phase4_log_file->fd, dbuf, p - dbuf); + if (n < 0 || (size_t) n != (size_t) (p - dbuf)) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, ngx_errno, + "modsecurity phase4 log write failed"); + return NGX_ERROR; + } + return NGX_OK; +} -/* XXX: xflt_filter() -- return NGX_OK here */ - return ngx_http_next_body_filter(r, in); +static ngx_str_t +ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in) +{ + ngx_str_t out; + size_t i; + u_char *semi; + out = in; + if (out.data == NULL || out.len == 0) { + out.data = (u_char *)""; out.len = 0; return out; + } + semi = (u_char *)ngx_strlchr(out.data, out.data + out.len, ';'); + if (semi) out.len = semi - out.data; + while (out.len > 0) { + u_char c = out.data[out.len - 1]; + if (c != ' ' && c != '\t' && c != '\n' && + c != '\r' && c != '\f' && c != '\v') { + break; + } + out.len--; + } + if (out.len == 0) { + out.data = (u_char *)""; return out; + } + out.data = ngx_pnalloc(pool, out.len); + if (out.data == NULL) { out.len = 0; return out; } + for (i = 0; i < out.len; i++) out.data[i] = ngx_tolower(in.data[i]); + return out; } diff --git a/src/ngx_http_modsecurity_common.h b/src/ngx_http_modsecurity_common.h index 9f79b5ba..73c81a54 100644 --- a/src/ngx_http_modsecurity_common.h +++ b/src/ngx_http_modsecurity_common.h @@ -71,15 +71,32 @@ #define MODSECURITY_NGINX_WHOAMI "ModSecurity-nginx v" \ MODSECURITY_NGINX_VERSION +#define NGX_HTTP_MODSEC_PHASE4_MODE_OFF 0 +#define NGX_HTTP_MODSEC_PHASE4_BODY_LIMIT_DEFAULT 1048576U +#define NGX_HTTP_MODSEC_PHASE4_MODE_SAFE 1 +#define NGX_HTTP_MODSEC_PHASE4_MODE_STRICT 2 + typedef struct { ngx_str_t name; ngx_str_t value; } ngx_http_modsecurity_header_t; +typedef struct ngx_http_modsecurity_conf_s ngx_http_modsecurity_conf_t; + typedef struct { ngx_http_request_t *r; + struct { + ngx_pool_t *pool; + const ngx_http_modsecurity_conf_t *conf; + } lifetime; + struct { + ngx_str_t target; + ngx_int_t status; + ngx_int_t response_status; + } error_page; Transaction *modsec_transaction; + ModSecurityIntervention pending_request_intervention; ModSecurityIntervention *delayed_intervention; #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) @@ -98,8 +115,34 @@ typedef struct { unsigned body_requested:1; unsigned processed:1; unsigned logged:1; + unsigned native_logging_failed:1; + unsigned request_policy_superseded:1; + unsigned request_intervention_active:1; + unsigned request_intervention_pending:1; + unsigned native_request_body_limit_rejection:1; unsigned intervention_triggered:1; unsigned request_body_processed:1; + unsigned phase4_headers_checked:1; + unsigned phase4_processed:1; + unsigned phase4_strict_abort:1; + unsigned phase4_started:1; + unsigned phase4_append_active:1; + unsigned phase4_failed:1; + unsigned phase4_error_emitting:1; + unsigned response_body_seen:1; + unsigned response_body_truncated:1; + unsigned intervention_late:1; + unsigned intervention_failed:1; + unsigned intervention_body_limit:1; + unsigned response_replaced:1; + unsigned replacement_body_terminal_sent:1; + unsigned intervention_redirect_location_installed:1; + size_t response_body_bytes_seen; + size_t response_body_bytes_inspected; + u_char *phase4_file_scratch; + u_char *request_file_scratch; + ngx_str_t last_intervention_log; + ngx_int_t last_intervention_status; } ngx_http_modsecurity_ctx_t; @@ -112,10 +155,12 @@ typedef struct { } ngx_http_modsecurity_main_conf_t; -typedef struct { +struct ngx_http_modsecurity_conf_s { void *pool; /* RulesSet or Rules */ void *rules_set; + const void *rules_identity; + unsigned rules_configured:1; ngx_flag_t enable; ngx_flag_t use_error_log; @@ -124,7 +169,12 @@ typedef struct { #endif ngx_http_complex_value_t *transaction_id; -} ngx_http_modsecurity_conf_t; + ngx_uint_t phase4_mode; + /* Deprecated compatibility value; libModSecurity owns inspection limits. */ + size_t phase4_body_limit; + ngx_open_file_t *phase4_log_file; + ngx_str_t phase4_log_path; +}; typedef ngx_int_t (*ngx_http_modsecurity_resolv_header_pt)(ngx_http_request_t *r, ngx_str_t name, off_t offset); @@ -142,6 +192,9 @@ extern ngx_module_t ngx_http_modsecurity_module; int ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_request_t *r, ngx_int_t early_log); ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_create_ctx(ngx_http_request_t *r); ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r); +ngx_int_t ngx_http_modsecurity_prepare_location(ngx_http_request_t *r, const ngx_http_modsecurity_conf_t *mcf); +ngx_int_t ngx_http_modsecurity_log_request_contexts(ngx_http_request_t *r); +ngx_int_t ngx_http_modsecurity_capture_request_body_intervention(ngx_http_request_t *r); char *ngx_str_to_char(ngx_str_t a, ngx_pool_t *p); #if (NGX_PCRE2) #define ngx_http_modsecurity_pcre_malloc_init(x) NULL @@ -165,6 +218,7 @@ int ngx_http_modsecurity_store_ctx_header(ngx_http_request_t *r, ngx_str_t *name /* ngx_http_modsecurity_log.c */ void ngx_http_modsecurity_log(void *log, const void* data); ngx_int_t ngx_http_modsecurity_log_handler(ngx_http_request_t *r); +ngx_int_t ngx_http_modsecurity_log_context(ngx_http_modsecurity_ctx_t *ctx); /* ngx_http_modsecurity_access.c */ ngx_int_t ngx_http_modsecurity_access_handler(ngx_http_request_t *r); diff --git a/src/ngx_http_modsecurity_header_filter.c b/src/ngx_http_modsecurity_header_filter.c index 03b8764d..4b5272e6 100644 --- a/src/ngx_http_modsecurity_header_filter.c +++ b/src/ngx_http_modsecurity_header_filter.c @@ -14,6 +14,7 @@ */ #include +#include #ifndef MODSECURITY_DDEBUG #define MODSECURITY_DDEBUG 0 @@ -350,6 +351,19 @@ ngx_http_modsecurity_resolv_header_transfer_encoding(ngx_http_request_t *r, ngx_ { ngx_http_modsecurity_ctx_t *ctx = NULL; + (void) offset; +#if (NGX_HTTP_V2) + /* Transfer-Encoding is an HTTP/1.x hop-by-hop header. */ + if (r->stream) { + return 1; + } +#endif +#if defined(nginx_version) && nginx_version >= 1025000 + if (r->http_version == NGX_HTTP_VERSION_30) { + return 1; + } +#endif + if (r->chunked) { ngx_str_t value = ngx_string("chunked"); @@ -514,6 +528,16 @@ ngx_http_modsecurity_header_filter(ngx_http_request_t *r) status = r->headers_out.status; } + /* The native API accepts int; validate the wider NGINX status before + * narrowing it, independently of protocol/header representation. */ + if (status > INT_MAX) { + ctx->intervention_triggered = 1; + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: response status exceeds the native integer range"); + return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, + NGX_HTTP_INTERNAL_SERVER_ERROR); + } + /* * NGINX always sends HTTP response with HTTP/1.1, except cases when * HTTP V2 module is enabled, and request has been posted with HTTP/2.0. @@ -526,13 +550,44 @@ ngx_http_modsecurity_header_filter(ngx_http_request_t *r) #endif old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_response_headers(ctx->modsec_transaction, status, http_response_ver); + ret = msc_process_response_headers(ctx->modsec_transaction, (int) status, + http_response_ver); ngx_http_modsecurity_pcre_malloc_done(old_pool); + if (ret != 1) { + ctx->intervention_triggered = 1; + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: response header phase processing failed"); + return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, + NGX_HTTP_INTERNAL_SERVER_ERROR); + } ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); - if (r->error_page) { - return ngx_http_next_header_filter(r); + if (ret < 0) { + ctx->intervention_triggered = 1; + r->connection->error = 1; + return NGX_ERROR; } if (ret > 0) { + /* An error_page response must honor its own response intervention. + * Mark the decision before finalization re-enters the filter. */ + ctx->intervention_triggered = 1; + if (ctx->intervention_redirect_location_installed) { + ctx->response_replaced = 1; + ngx_http_clear_content_length(r); + ngx_http_clear_last_modified(r); + ngx_http_clear_etag(r); + ngx_http_clear_accept_ranges(r); + ngx_str_null(&r->headers_out.content_type); + r->headers_out.content_type_len = 0; + if (r->headers_out.content_encoding != NULL) { + r->headers_out.content_encoding->hash = 0; + r->headers_out.content_encoding = NULL; + } + r->headers_out.status = ret; + ngx_str_null(&r->headers_out.status_line); + r->headers_out.content_length_n = 0; + r->header_only = 1; + return ngx_http_next_header_filter(r); + } return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, ret); } diff --git a/src/ngx_http_modsecurity_log.c b/src/ngx_http_modsecurity_log.c index 094685c5..3e6f4499 100644 --- a/src/ngx_http_modsecurity_log.c +++ b/src/ngx_http_modsecurity_log.c @@ -37,39 +37,48 @@ ngx_http_modsecurity_log(void *log, const void* data) ngx_int_t -ngx_http_modsecurity_log_handler(ngx_http_request_t *r) +ngx_http_modsecurity_log_context(ngx_http_modsecurity_ctx_t *ctx) { ngx_pool_t *old_pool; - ngx_http_modsecurity_ctx_t *ctx; - - dd("catching a new _log_ phase handler"); - - /* - if (r->method != NGX_HTTP_GET && - r->method != NGX_HTTP_POST && r->method != NGX_HTTP_HEAD) { - dd("ModSecurity is not ready to deal with anything different from " \ - "POST, GET or HEAD"); - return NGX_OK; - } - */ - ctx = ngx_http_modsecurity_get_module_ctx(r); + ngx_http_request_t *r; + int result; - dd("recovering ctx: %p", ctx); - - if (ctx == NULL) { - dd("ModSecurity not enabled or error occurred"); - return NGX_OK; + if (ctx == NULL || ctx->r == NULL || ctx->r->connection == NULL || + ctx->lifetime.pool == NULL) { + return NGX_ERROR; } - + r = ctx->r; if (ctx->logged) { - dd("already logged earlier"); - return NGX_OK; + return ctx->native_logging_failed ? NGX_ERROR : NGX_OK; } - dd("calling msc_process_logging for %p", ctx); - old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_logging(ctx->modsec_transaction); + /* Claim the attempt before native callbacks can re-enter this owner. */ + ctx->logged = 1; + ctx->native_logging_failed = 1; + if (ctx->modsec_transaction == NULL) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: native logging transaction is unavailable"); + return NGX_ERROR; + } + /* Pool cleanup can run after NGINX clears r->pool. Retain its real owner. */ + old_pool = ngx_http_modsecurity_pcre_malloc_init(ctx->lifetime.pool); + result = msc_process_logging(ctx->modsec_transaction); ngx_http_modsecurity_pcre_malloc_done(old_pool); - + if (result != 1) { + /* The log phase cannot replace the response or retry the audit sink. */ + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: native logging phase processing failed"); + return NGX_ERROR; + } + ctx->native_logging_failed = 0; return NGX_OK; } + + +ngx_int_t +ngx_http_modsecurity_log_handler(ngx_http_request_t *r) +{ + /* Every transaction owner for this request gets one native audit attempt, + * including prior locations and disabled/different target policies. */ + return ngx_http_modsecurity_log_request_contexts(r); +} diff --git a/src/ngx_http_modsecurity_log_metadata.h b/src/ngx_http_modsecurity_log_metadata.h new file mode 100644 index 00000000..005c7805 --- /dev/null +++ b/src/ngx_http_modsecurity_log_metadata.h @@ -0,0 +1,395 @@ +/* + * ModSecurity-nginx metadata helpers. + * Adapted from common/src/rule_id.c and common/src/json_escape.c in + * Easton97-Jens/ModSecurity-conector at + * d420a12c4155a4bc2b9ea63ee5cef633085705e7. + * Uses bounded NGINX strings and pools without a Common runtime dependency. + */ + +#ifndef NGX_HTTP_MODSECURITY_LOG_METADATA_H +#define NGX_HTTP_MODSECURITY_LOG_METADATA_H + +#include +#include +#include + +typedef struct { + ngx_str_t input; + size_t position; +} ngx_http_modsecurity_metadata_cursor_t; + +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_field_starts(ngx_str_t input, size_t position, + const char *name) +{ + size_t length = strlen(name); + + return position <= input.len && input.len - position >= length + 4 && + input.data[position] == '[' && + memcmp(input.data + position + 1, name, length) == 0 && + input.data[position + length + 1] == ' ' && + input.data[position + length + 2] == '"'; +} + +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_hex_digit(u_char value) +{ + return (value >= '0' && value <= '9') || + (value >= 'a' && value <= 'f') || + (value >= 'A' && value <= 'F'); +} + +/* Engine toHexIfNeeded() uses literal \xHH sequences in the escaped fields. + * Do not decode them while locating delimiters. Raw fields may contain quotes + * without a following ']', but the complete ordered record must still match. */ +static ngx_inline size_t +ngx_http_modsecurity_metadata_field_character_size(ngx_str_t input, + size_t position, ngx_int_t escaped, ngx_int_t decimal) +{ + u_char character = input.data[position]; + size_t remaining = input.len - position; + + if (character < 0x20 || character > 0x7e || + (decimal && (character < '0' || character > '9'))) { + return 0; + } + if (!escaped) { + return 1; + } + if (character == '\\') { + if (remaining < 4 || input.data[position + 1] != 'x' || + !ngx_http_modsecurity_metadata_hex_digit(input.data[position + 2]) || + !ngx_http_modsecurity_metadata_hex_digit(input.data[position + 3])) { + return 0; + } + return 4; + } + return character == '"' ? 0 : 1; +} + +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_parse_field( + ngx_http_modsecurity_metadata_cursor_t *cursor, const char *name, + ngx_int_t escaped, ngx_int_t decimal, ngx_str_t *value) +{ + size_t start; + size_t position; + size_t consumed; + + if (!ngx_http_modsecurity_metadata_field_starts(cursor->input, + cursor->position, name)) { + return 0; + } + start = cursor->position + strlen(name) + 3; + position = start; + while (position < cursor->input.len) { + if (cursor->input.data[position] == '"' && + cursor->input.len - position >= 2 && + cursor->input.data[position + 1] == ']') { + if (decimal && position == start) { + return 0; + } + if (value != NULL) { + value->data = cursor->input.data + start; + value->len = position - start; + } + cursor->position = position + 2; + return 1; + } + consumed = ngx_http_modsecurity_metadata_field_character_size( + cursor->input, position, escaped, decimal); + if (consumed == 0) { + return 0; + } + position += consumed; + } + return 0; +} + +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_next_field( + ngx_http_modsecurity_metadata_cursor_t *cursor, const char *name, + ngx_int_t escaped, ngx_int_t decimal) +{ + if (cursor->position >= cursor->input.len || + cursor->input.data[cursor->position] != ' ') { + return 0; + } + cursor->position++; + return ngx_http_modsecurity_metadata_parse_field(cursor, name, escaped, + decimal, NULL); +} + +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_parse_record( + ngx_http_modsecurity_metadata_cursor_t *cursor, ngx_str_t *rule_id) +{ + if (!ngx_http_modsecurity_metadata_parse_field(cursor, "file", 0, 0, NULL) || + !ngx_http_modsecurity_metadata_next_field(cursor, "line", 0, 1) || + cursor->position >= cursor->input.len || + cursor->input.data[cursor->position] != ' ') { + return 0; + } + cursor->position++; + if (!ngx_http_modsecurity_metadata_parse_field(cursor, "id", 0, 1, rule_id) || + !ngx_http_modsecurity_metadata_next_field(cursor, "rev", 1, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "msg", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "data", 1, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "severity", 0, 1) || + !ngx_http_modsecurity_metadata_next_field(cursor, "ver", 1, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "maturity", 0, 1) || + !ngx_http_modsecurity_metadata_next_field(cursor, "accuracy", 0, 1)) { + return 0; + } + while (cursor->position < cursor->input.len && + cursor->input.data[cursor->position] == ' ' && + ngx_http_modsecurity_metadata_field_starts(cursor->input, + cursor->position + 1, "tag")) { + if (!ngx_http_modsecurity_metadata_next_field(cursor, "tag", 1, 0)) { + return 0; + } + } + if (!ngx_http_modsecurity_metadata_next_field(cursor, "hostname", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "uri", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "unique_id", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "ref", 0, 0)) { + return 0; + } + if (cursor->position < cursor->input.len && + cursor->input.data[cursor->position] == ' ' && + (!ngx_http_modsecurity_metadata_next_field(cursor, "hostname", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "uri", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "unique_id", 0, 0))) { + return 0; + } + return cursor->position == cursor->input.len; +} + +/* Return 1 for one complete, unambiguous native record, 0 for no record, and + * -1 for malformed/ambiguous framing. The successful ID is a bounded view + * into input; input need not be NUL-terminated and no allocation is made. + * Bare ID snippets and the untrusted matched-data prefix are never metadata. */ +static ngx_inline ngx_int_t +ngx_http_modsecurity_parse_rule_metadata(ngx_str_t input, ngx_str_t *rule_id) +{ + static const char marker[] = "[file \""; + ngx_http_modsecurity_metadata_cursor_t cursor; + ngx_str_t candidate; + size_t position; + size_t first; + + if (rule_id == NULL) { + return -1; + } + rule_id->data = (u_char *) ""; + rule_id->len = 0; + if (input.data == NULL) { + return input.len == 0 ? 0 : -1; + } + if (input.len < sizeof(marker) - 1) { + return 0; + } + first = input.len; + for (position = 0; position <= input.len - (sizeof(marker) - 1); + position++) { + if (memcmp(input.data + position, marker, sizeof(marker) - 1) != 0) { + continue; + } + /* Reject extra candidates even if only one suffix would parse: a raw + * URI or reflected prefix can contain a forged complete record. */ + if (first != input.len || + (position != 0 && input.data[position - 1] != ' ')) { + return -1; + } + first = position; + } + if (first == input.len) { + return 0; + } + cursor.input = input; + cursor.position = first; + if (!ngx_http_modsecurity_metadata_parse_record(&cursor, &candidate)) { + return -1; + } + *rule_id = candidate; + return 1; +} + +static ngx_inline void +ngx_http_modsecurity_extract_rule_id(ngx_pool_t *pool, ngx_str_t *intervention, + ngx_str_t *rule_id) +{ + ngx_str_t candidate; + + rule_id->data = (u_char *) ""; + rule_id->len = 0; + if (intervention == NULL || + ngx_http_modsecurity_parse_rule_metadata(*intervention, &candidate) != 1) { + return; + } + rule_id->data = ngx_pnalloc(pool, candidate.len + 1); + if (rule_id->data == NULL) { + return; + } + memcpy(rule_id->data, candidate.data, candidate.len); + rule_id->data[candidate.len] = '\0'; + rule_id->len = candidate.len; +} + +static ngx_inline ngx_str_t +ngx_http_modsecurity_sanitize_intervention(ngx_pool_t *pool, ngx_str_t input) +{ + static ngx_str_t redacted = { + sizeof("redacted") - 1, (u_char *) "redacted" + }; + static ngx_str_t summary = { + sizeof("id:present msg:present op:-") - 1, + (u_char *) "id:present msg:present op:-" + }; + ngx_str_t rule_id; + + (void) pool; + /* A complete record supplies id/msg fields, but no trusted operator + * field. Never derive presence flags from matched data or message prose. + * Static complete strings avoid allocation failure and truncation. */ + return ngx_http_modsecurity_parse_rule_metadata(input, &rule_id) == 1 + ? summary : redacted; +} + +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_utf8_continuation(u_char value) +{ + return value >= 0x80 && value <= 0xbf; +} + +static ngx_inline size_t +ngx_http_modsecurity_metadata_utf8_size(const u_char *input, size_t remaining) +{ + u_char first = input[0]; + + if (first >= 0xc2 && first <= 0xdf) { + return remaining >= 2 && + ngx_http_modsecurity_metadata_utf8_continuation(input[1]) ? 2 : 0; + } + if (first >= 0xe0 && first <= 0xef) { + if (remaining < 3 || + !ngx_http_modsecurity_metadata_utf8_continuation(input[1]) || + !ngx_http_modsecurity_metadata_utf8_continuation(input[2]) || + (first == 0xe0 && input[1] < 0xa0) || + (first == 0xed && input[1] > 0x9f)) { + return 0; + } + return 3; + } + if (first >= 0xf0 && first <= 0xf4) { + if (remaining < 4 || + !ngx_http_modsecurity_metadata_utf8_continuation(input[1]) || + !ngx_http_modsecurity_metadata_utf8_continuation(input[2]) || + !ngx_http_modsecurity_metadata_utf8_continuation(input[3]) || + (first == 0xf0 && input[1] < 0x90) || + (first == 0xf4 && input[1] > 0x8f)) { + return 0; + } + return 4; + } + return 0; +} + +/* Classify one input byte or complete UTF-8 sequence for both JSON passes. */ +static ngx_inline size_t +ngx_http_modsecurity_metadata_json_character_size(const u_char *input, + size_t remaining, size_t *consumed) +{ + u_char character = input[0]; + size_t sequence_size; + + *consumed = 1; + if (character < 0x20) { + return 6; + } + if (character == '"' || character == '\\') { + return 2; + } + if (character < 0x80) { + return 1; + } + sequence_size = ngx_http_modsecurity_metadata_utf8_size(input, remaining); + if (sequence_size == 0) { + return 6; + } + *consumed = sequence_size; + return sequence_size; +} + +static ngx_inline u_char * +ngx_http_modsecurity_metadata_write_json_character(u_char *output, + const u_char *input, size_t consumed, size_t width) +{ + static const u_char hex[] = "0123456789abcdef"; + u_char character = input[0]; + + if (width == 6) { + *output++ = '\\'; + *output++ = 'u'; + *output++ = '0'; + *output++ = '0'; + *output++ = hex[character >> 4]; + *output++ = hex[character & 0x0f]; + } else if (consumed == 1 && width == 2) { + *output++ = '\\'; + *output++ = character; + } else { + memcpy(output, input, consumed); + output += consumed; + } + return output; +} + +/* Preserve valid UTF-8 as-is. JSON controls and invalid UTF-8 bytes are + * represented as \u00xx; each invalid byte retains its unsigned byte value. + * Success is NUL-terminated with an explicit length. Allocation/size failure + * returns data == NULL, len == 0 so the caller can diagnose the optional sink + * failure without confusing it with a valid empty field. */ +static ngx_inline void +ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst) +{ + size_t position; + size_t size; + size_t consumed; + size_t width; + u_char *output; + + dst->data = NULL; + dst->len = 0; + if (src == NULL || src->len == 0) { + dst->data = (u_char *) ""; + return; + } + if (src->data == NULL || src->len == SIZE_MAX) { + return; + } + size = 0; + for (position = 0; position < src->len; position += consumed) { + width = ngx_http_modsecurity_metadata_json_character_size( + src->data + position, src->len - position, &consumed); + if (width > SIZE_MAX - 1 - size) { + return; + } + size += width; + } + dst->data = ngx_pnalloc(pool, size + 1); + if (dst->data == NULL) { + return; + } + output = dst->data; + for (position = 0; position < src->len; position += consumed) { + width = ngx_http_modsecurity_metadata_json_character_size( + src->data + position, src->len - position, &consumed); + output = ngx_http_modsecurity_metadata_write_json_character(output, + src->data + position, consumed, width); + } + *output = '\0'; + dst->len = size; +} + +#endif /* NGX_HTTP_MODSECURITY_LOG_METADATA_H */ diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index 1e55ade0..f77228de 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -21,6 +21,7 @@ #include "ddebug.h" #include "ngx_http_modsecurity_common.h" +#include "ngx_http_modsecurity_log_metadata.h" #include "stdio.h" #include #include @@ -36,7 +37,9 @@ static void *ngx_http_modsecurity_create_conf(ngx_conf_t *cf); static char *ngx_http_modsecurity_merge_conf(ngx_conf_t *cf, void *parent, void *child); static void ngx_http_modsecurity_cleanup_instance(void *data); static void ngx_http_modsecurity_cleanup_rules(void *data); - +static char *ngx_conf_set_phase4_mode(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char *ngx_conf_set_phase4_body_limit(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char *ngx_conf_set_phase4_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); /* * PCRE malloc/free workaround, based on @@ -136,121 +139,343 @@ ngx_inline char *ngx_str_to_char(ngx_str_t a, ngx_pool_t *p) } -int -ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_request_t *r, ngx_int_t early_log) +static ngx_int_t +ngx_http_modsecurity_log_intervention(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf, + const ModSecurityIntervention *intervention) { - char *log = NULL; - ModSecurityIntervention intervention; - intervention.status = 200; - intervention.url = NULL; - intervention.log = NULL; - intervention.disruptive = 0; - ngx_http_modsecurity_ctx_t *ctx = NULL; - ngx_http_modsecurity_conf_t *mcf; + size_t len; + + ngx_str_null(&ctx->last_intervention_log); + if (mcf->phase4_log_file != NULL && intervention->log != NULL) { + len = ngx_strlen(intervention->log); + ctx->last_intervention_log.data = ngx_pnalloc(r->pool, len + 1); + if (ctx->last_intervention_log.data == NULL) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "ModSecurity: optional phase4 metadata allocation failed"); + } else { + ngx_memcpy(ctx->last_intervention_log.data, intervention->log, len + 1); + ctx->last_intervention_log.len = len; + } + } + if (mcf->use_error_log) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "%s", + intervention->log != NULL ? intervention->log : + "(no log message was specified)"); + } + return NGX_OK; +} - dd("processing intervention"); - ctx = ngx_http_modsecurity_get_module_ctx(r); - if (ctx == NULL) - { +static int +ngx_http_modsecurity_process_intervention_redirect(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, + const ModSecurityIntervention *intervention) +{ + ngx_table_elt_t *location; + u_char *url; + size_t len; + size_t i; + + if (r->header_sent) { + ctx->intervention_late = 1; + ctx->intervention_failed = 0; + return NGX_ERROR; + } + len = ngx_strlen(intervention->url); + for (i = 0; i < len; i++) { + if (intervention->url[i] == '\r' || intervention->url[i] == '\n') { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: intervention redirect URL contains CR or LF"); + return NGX_HTTP_BAD_REQUEST; + } + } + url = ngx_pnalloc(r->pool, len); + if (url == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } - - if (msc_intervention(transaction, &intervention) == 0) { - dd("nothing to do"); - return 0; + ngx_memcpy(url, intervention->url, len); + location = ngx_list_push(&r->headers_out.headers); + if (location == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + ngx_http_clear_location(r); + ngx_str_set(&location->key, "Location"); + location->value.data = url; + location->value.len = len; + location->hash = 1; + r->headers_out.location = location; +#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) + if (ngx_http_modsecurity_store_ctx_header(r, &location->key, + &location->value) != NGX_OK) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; } +#endif + ctx->intervention_redirect_location_installed = 1; + ctx->intervention_failed = 0; + return intervention->status; +} - mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); - if (mcf == NULL) { - if (intervention.log != NULL) { - free(intervention.log); + +/* appendResponseBody does not evaluate response rules. Only a rule-free + * disruptive 403 collected immediately after a valid append is native Reject. + * Free-text diagnostics are not part of this native API boundary. */ +static ngx_int_t +ngx_http_modsecurity_is_response_body_limit_rejection( + ngx_http_modsecurity_ctx_t *ctx, + const ModSecurityIntervention *intervention) +{ + ngx_str_t log; + ngx_str_t rule_id; + + log.data = (u_char *) intervention->log; + log.len = intervention->log != NULL ? ngx_strlen(intervention->log) : 0; + return ctx->phase4_append_active && !ctx->phase4_processed && + intervention->disruptive == 1 && + intervention->status == NGX_HTTP_FORBIDDEN && + intervention->url == NULL && + ngx_http_modsecurity_parse_rule_metadata(log, &rule_id) == 0; +} + +/* A block response may traverse only its captured local error-page target. + * The error_page flag alone is insufficient: ordinary redirects still inspect. */ +static void +ngx_http_modsecurity_capture_error_page(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_int_t status) +{ + ngx_http_core_loc_conf_t *clcf; + ngx_http_err_page_t *pages; + ngx_str_t target; + u_char *args; + ngx_uint_t i; + + if (!ctx->request_intervention_active || ctx->native_request_body_limit_rejection || + status < NGX_HTTP_BAD_REQUEST) { + return; + } + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + if (clcf == NULL || clcf->error_pages == NULL) { + return; + } + pages = clcf->error_pages->elts; + for (i = 0; i < clcf->error_pages->nelts; i++) { + if (pages[i].status != status) { + continue; } - if (intervention.url != NULL) { - free(intervention.url); + if (ngx_http_complex_value(r, &pages[i].value, &target) != NGX_OK || + target.data == NULL || target.len == 0 || + (target.data[0] != '/' && target.data[0] != '@')) { + return; } - return NGX_HTTP_INTERNAL_SERVER_ERROR; + args = ngx_strlchr(target.data, target.data + target.len, '?'); + if (args != NULL) { + target.len = (size_t) (args - target.data); + } + ctx->error_page.target = target; + ctx->error_page.status = status; + ctx->error_page.response_status = pages[i].overwrite >= 0 ? + pages[i].overwrite : status; + return; } +} - // logging to nginx error log can be disable by setting `modsecurity_use_error_log` to off - if (mcf->use_error_log) { - log = intervention.log; - if (intervention.log == NULL) { - log = "(no log message was specified)"; - } - ngx_log_error(NGX_LOG_ERR, (ngx_log_t *)r->connection->log, 0, "%s", log); +static ngx_int_t +ngx_http_modsecurity_intervention_failure_status(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) +{ + if (r->header_sent) { + return NGX_ERROR; } + return ctx->intervention_body_limit ? NGX_HTTP_FORBIDDEN : + NGX_HTTP_INTERNAL_SERVER_ERROR; +} - if (intervention.log != NULL) { - free(intervention.log); +static ngx_int_t +ngx_http_modsecurity_collect_native_intervention(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, Transaction *transaction, + ModSecurityIntervention *intervention) +{ + int native_result; + + native_result = msc_intervention(transaction, intervention); + if (native_result != 0 && native_result != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: invalid native intervention result"); + return NGX_ERROR; + } + if (native_result == 0 || !intervention->disruptive) { + ctx->intervention_failed = 0; + return NGX_DECLINED; + } + if (ngx_http_modsecurity_is_response_body_limit_rejection(ctx, + intervention)) { + ctx->intervention_body_limit = 1; + ctx->last_intervention_status = NGX_HTTP_FORBIDDEN; + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: response body limit rejected the response"); + return NGX_ERROR; } + /* Response rules run at EOS. Other disruptive append-time results are + * invalid engine responses and cannot become Safe log-only decisions. */ + if (ctx->phase4_started && !ctx->phase4_processed) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: unexpected intervention during response append"); + return NGX_ERROR; + } + return NGX_OK; +} - if (intervention.url != NULL) - { - dd("intervention -- redirecting to: %s with status code: %d", intervention.url, intervention.status); +static ngx_int_t +ngx_http_modsecurity_defer_late_phase4_intervention(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf) +{ + if (!ctx->phase4_started || !ctx->phase4_processed || !r->header_sent || + mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_OFF) { + return 0; + } + /* Safe/Strict record a WAF decision, not an HTTP status replacement. + * Do not dispatch that late decision through native status/audit hooks. */ + ctx->intervention_late = 1; + ctx->intervention_failed = 0; + return 1; +} - if (r->header_sent) - { - dd("Headers are already sent. Cannot perform the redirection at this point."); - return -1; - } +/* Drain native request append results immediately. Retain an owned limit + * decision until normal P2 dispatch, even if ProcessPartial returns zero. */ +ngx_int_t +ngx_http_modsecurity_capture_request_body_intervention(ngx_http_request_t *r) +{ + ngx_http_modsecurity_ctx_t *ctx; + ModSecurityIntervention intervention; + ngx_int_t result = NGX_OK; + int native_result; - /** - * Not sure if it sane to do this indepent of the phase - * but, here we go... - * - * This code cames from: http/ngx_http_special_response.c - * function: ngx_http_send_error_page - * src/http/ngx_http_core_module.c - * From src/http/ngx_http_core_module.c (line 1910) i learnt - * that location->hash should be set to 1. - * - */ - ngx_http_clear_location(r); - ngx_str_t a = ngx_string(""); - - a.data = (unsigned char *)intervention.url; - a.len = strlen(intervention.url); - - ngx_table_elt_t *location = NULL; - location = ngx_list_push(&r->headers_out.headers); - ngx_str_set(&location->key, "Location"); - location->value = a; - r->headers_out.location = location; - r->headers_out.location->hash = 1; + ctx = ngx_http_modsecurity_get_module_ctx(r); + if (ctx == NULL || ctx->modsec_transaction == NULL) { + return NGX_ERROR; + } + ngx_memzero(&intervention, sizeof(intervention)); + intervention.status = 200; + native_result = msc_intervention(ctx->modsec_transaction, &intervention); + if (native_result != 0 && native_result != 1) { + result = NGX_ERROR; + } else if (native_result == 1 && intervention.disruptive) { + if ((intervention.status != 403 && intervention.status != 413) || + intervention.url != NULL || intervention.log == NULL || + strcmp(intervention.log, + "Request body limit is marked to reject the request") != 0) { + result = NGX_ERROR; + } else if (!ctx->request_intervention_pending) { + ctx->pending_request_intervention = intervention; + ctx->request_intervention_pending = 1; + ngx_memzero(&intervention, sizeof(intervention)); + } + } + if (intervention.log != NULL) free(intervention.log); + if (intervention.url != NULL) free(intervention.url); + return result; +} -#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) - ngx_http_modsecurity_store_ctx_header(r, &location->key, &location->value); -#endif +static ngx_int_t +ngx_http_modsecurity_take_pending_request_intervention( + ngx_http_modsecurity_ctx_t *ctx, ngx_int_t native_result, + ModSecurityIntervention *intervention) +{ + if (native_result == NGX_ERROR || !ctx->request_intervention_active || + !ctx->request_intervention_pending) { + return native_result; + } + if (intervention->log != NULL) free(intervention->log); + if (intervention->url != NULL) free(intervention->url); + *intervention = ctx->pending_request_intervention; + ngx_memzero(&ctx->pending_request_intervention, + sizeof(ctx->pending_request_intervention)); + ctx->request_intervention_pending = 0; + ctx->native_request_body_limit_rejection = 1; + ctx->intervention_failed = 1; + return NGX_OK; +} - return intervention.status; +static void +ngx_http_modsecurity_reset_intervention(ngx_http_modsecurity_ctx_t *ctx) +{ + if (ctx == NULL) { + return; } + ctx->intervention_late = 0; + ctx->intervention_failed = 0; + ctx->intervention_body_limit = 0; + ctx->native_request_body_limit_rejection = 0; + ctx->intervention_redirect_location_installed = 0; + ctx->last_intervention_status = 0; + ngx_str_null(&ctx->last_intervention_log); +} - if (intervention.status != 200) - { - /** - * FIXME: this will bring proper response code to audit log in case - * when e.g. error_page redirect was triggered, but there still won't be another - * required pieces like response headers etc. - * - */ - msc_update_status_code(ctx->modsec_transaction, intervention.status); +int +ngx_http_modsecurity_process_intervention (Transaction *transaction, + ngx_http_request_t *r, ngx_int_t early_log) +{ + ModSecurityIntervention intervention; + ngx_http_modsecurity_ctx_t *ctx; + ngx_http_modsecurity_conf_t *mcf; + int result; + ngx_int_t native_result; - if (early_log) { - dd("intervention -- calling log handler manually with code: %d", intervention.status); - ngx_http_modsecurity_log_handler(r); - ctx->logged = 1; - } + ctx = ngx_http_modsecurity_get_module_ctx(r); + mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + ngx_http_modsecurity_reset_intervention(ctx); + if (ctx == NULL || mcf == NULL || transaction == NULL) { + if (ctx != NULL) ctx->intervention_failed = 1; + return r->header_sent ? NGX_ERROR : NGX_HTTP_INTERNAL_SERVER_ERROR; + } + ngx_memzero(&intervention, sizeof(intervention)); + intervention.status = 200; + ctx->intervention_failed = 1; + native_result = ngx_http_modsecurity_collect_native_intervention(r, ctx, + transaction, &intervention); + native_result = ngx_http_modsecurity_take_pending_request_intervention(ctx, + native_result, &intervention); + result = native_result == NGX_DECLINED ? 0 : + ngx_http_modsecurity_intervention_failure_status(r, ctx); + if (native_result != NGX_OK) { + goto done; + } + ctx->last_intervention_status = intervention.status; + if (ngx_http_modsecurity_log_intervention(r, ctx, mcf, + &intervention) != NGX_OK) { + goto done; + } + if (ngx_http_modsecurity_defer_late_phase4_intervention(r, ctx, mcf)) { + goto done; + } - if (r->header_sent) - { - dd("Headers are already sent. Cannot perform the redirection at this point."); - return -1; + if (intervention.url != NULL) { + result = ngx_http_modsecurity_process_intervention_redirect(r, ctx, + &intervention); + } else if (intervention.status != 200) { + msc_update_status_code(ctx->modsec_transaction, intervention.status); + ngx_http_modsecurity_capture_error_page(r, ctx, intervention.status); + if (early_log && ngx_http_modsecurity_log_handler(r) != NGX_OK) { + goto done; + } + if (r->header_sent) { + ctx->intervention_late = 1; + ctx->intervention_failed = 0; + goto done; } - dd("intervention -- returning code: %d", intervention.status); - return intervention.status; + ctx->intervention_failed = 0; + result = intervention.status; + } else { + ctx->intervention_failed = 0; + result = 0; } - return 0; + +done: + /* libModSecurity owns these allocations until the intervention is read. + * NGINX headers and metadata retain only request-pool copies. */ + if (intervention.log != NULL) free(intervention.log); + if (intervention.url != NULL) free(intervention.url); + return result; } @@ -260,15 +485,35 @@ ngx_http_modsecurity_cleanup(void *data) ngx_http_modsecurity_ctx_t *ctx; ctx = (ngx_http_modsecurity_ctx_t *) data; - - msc_transaction_cleanup(ctx->modsec_transaction); + if (ctx == NULL) { + return; + } + if (ctx->modsec_transaction != NULL) { + /* Redirects may retire this owner before the normal log phase. Its + * retained pool is still valid even after NGINX clears r->pool. */ + (void) ngx_http_modsecurity_log_context(ctx); + if (ctx->pending_request_intervention.log != NULL) { + free(ctx->pending_request_intervention.log); + } + if (ctx->pending_request_intervention.url != NULL) { + free(ctx->pending_request_intervention.url); + } + ngx_memzero(&ctx->pending_request_intervention, + sizeof(ctx->pending_request_intervention)); + ctx->request_intervention_pending = 0; + msc_transaction_cleanup(ctx->modsec_transaction); + ctx->modsec_transaction = NULL; + } #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) /* * Purge stored context headers. Memory allocated for individual stored header * name/value pair will be freed automatically when r->pool is destroyed. */ - ngx_array_destroy(ctx->sanity_headers_out); + if (ctx->sanity_headers_out != NULL) { + ngx_array_destroy(ctx->sanity_headers_out); + ctx->sanity_headers_out = NULL; + } #endif } @@ -281,6 +526,7 @@ ngx_http_modsecurity_create_ctx(ngx_http_request_t *r) ngx_http_modsecurity_ctx_t *ctx; ngx_http_modsecurity_conf_t *mcf; ngx_http_modsecurity_main_conf_t *mmcf; + char *transaction_id; ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_modsecurity_ctx_t)); if (ctx == NULL) @@ -288,65 +534,197 @@ ngx_http_modsecurity_create_ctx(ngx_http_request_t *r) dd("failed to allocate memory for the context."); return NULL; } + ctx->r = r; + ctx->lifetime.pool = r->pool; mmcf = ngx_http_get_module_main_conf(r, ngx_http_modsecurity_module); mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + ctx->lifetime.conf = mcf; + + if (mmcf == NULL || mcf == NULL) { + return NULL; + } dd("creating transaction with the following rules: '%p' -- ms: '%p'", mcf->rules_set, mmcf->modsec); if (mcf->transaction_id) { - if (ngx_http_complex_value(r, mcf->transaction_id, &s) != NGX_OK) { - return NGX_CONF_ERROR; + if (ngx_http_complex_value(r, mcf->transaction_id, &s) != NGX_OK || + s.len == (size_t) -1 || (s.len != 0 && s.data == NULL)) { + return NULL; + } + transaction_id = ngx_pnalloc(r->pool, s.len + 1); + if (transaction_id == NULL) { + return NULL; } - ctx->modsec_transaction = msc_new_transaction_with_id(mmcf->modsec, mcf->rules_set, (char *) s.data, r->connection->log); + if (s.len != 0) { + ngx_memcpy(transaction_id, s.data, s.len); + } + transaction_id[s.len] = '\0'; + ctx->modsec_transaction = msc_new_transaction_with_id(mmcf->modsec, mcf->rules_set, transaction_id, r->connection->log); } else { ctx->modsec_transaction = msc_new_transaction(mmcf->modsec, mcf->rules_set, r->connection->log); } - dd("transaction created"); - - ngx_http_set_ctx(r, ctx, ngx_http_modsecurity_module); + if (ctx->modsec_transaction == NULL) { + return NULL; + } +#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) + ctx->sanity_headers_out = ngx_array_create(r->pool, 12, sizeof(ngx_http_modsecurity_header_t)); + if (ctx->sanity_headers_out == NULL) { + ngx_http_modsecurity_cleanup(ctx); + return NULL; + } +#endif - cln = ngx_pool_cleanup_add(r->pool, sizeof(ngx_http_modsecurity_ctx_t)); + cln = ngx_pool_cleanup_add(r->pool, 0); if (cln == NULL) { dd("failed to create the ModSecurity context cleanup"); - return NGX_CONF_ERROR; + ngx_http_modsecurity_cleanup(ctx); + return NULL; } cln->handler = ngx_http_modsecurity_cleanup; cln->data = ctx; + ngx_http_set_ctx(r, ctx, ngx_http_modsecurity_module); + + return ctx; +} + +static ngx_int_t +ngx_http_modsecurity_same_policy(const ngx_http_modsecurity_conf_t *left, + const ngx_http_modsecurity_conf_t *right) +{ + return left != NULL && right != NULL && + left->rules_identity == right->rules_identity && + left->enable == right->enable && + left->use_error_log == right->use_error_log && + left->transaction_id == right->transaction_id && + left->phase4_mode == right->phase4_mode && + left->phase4_log_file == right->phase4_log_file #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) - ctx->sanity_headers_out = ngx_array_create(r->pool, 12, sizeof(ngx_http_modsecurity_header_t)); - if (ctx->sanity_headers_out == NULL) { - return NGX_CONF_ERROR; - } + && left->sanity_checks_enabled == right->sanity_checks_enabled #endif + ; +} - return ctx; +static ngx_int_t +ngx_http_modsecurity_is_block_error_page(const ngx_http_request_t *r, + const ngx_http_modsecurity_ctx_t *ctx) +{ + ngx_http_core_loc_conf_t *clcf; + ngx_str_t target; + + if (!r->internal || ctx->r != r || !ctx->intervention_triggered || + ctx->intervention_failed || ctx->phase4_failed || + ctx->error_page.target.len == 0 || + ctx->error_page.response_status != (ngx_int_t) r->err_status || + ctx->error_page.status != ctx->last_intervention_status) { + return 0; + } + target = r->uri; + if (ctx->error_page.target.data[0] == '@') { + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + if (clcf == NULL || !clcf->named) { + return 0; + } + target = clcf->name; + } + return target.len == ctx->error_page.target.len && + ngx_strncmp(target.data, ctx->error_page.target.data, target.len) == 0; +} + +/* Retire an allowed source policy when an internal target changes policy or + * disables inspection. Its audit owner remains on the pool cleanup chain. */ +ngx_int_t +ngx_http_modsecurity_prepare_location(ngx_http_request_t *r, + const ngx_http_modsecurity_conf_t *mcf) +{ + ngx_pool_cleanup_t *cln; + ngx_http_modsecurity_ctx_t *ctx; + + if (r == NULL || r->pool == NULL) { + return NGX_ERROR; + } + for (cln = r->pool->cleanup; cln != NULL; cln = cln->next) { + if (cln->handler != ngx_http_modsecurity_cleanup || cln->data == NULL) { + continue; + } + ctx = cln->data; + if (ctx->r == r && ctx->modsec_transaction != NULL && + !ctx->request_policy_superseded && !ctx->intervention_triggered && + !ctx->logged && !ctx->intervention_failed && !ctx->phase4_failed && + !ngx_http_modsecurity_same_policy(ctx->lifetime.conf, mcf)) { + ctx->request_policy_superseded = 1; + } + } + return NGX_OK; } -ngx_inline ngx_http_modsecurity_ctx_t * +ngx_http_modsecurity_ctx_t * ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r) { ngx_http_modsecurity_ctx_t *ctx; + ngx_http_modsecurity_ctx_t *candidate; + ngx_pool_cleanup_t *cln; + ngx_http_modsecurity_conf_t *mcf; + + if (r == NULL || r->pool == NULL) { + return NULL; + } + mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + if (mcf == NULL || mcf->enable != 1) { + return NULL; + } ctx = ngx_http_get_module_ctx(r, ngx_http_modsecurity_module); - if (ctx == NULL) { - /* - * refer /src/http/modules/ngx_http_realip_module.c - * if module context was reset, the original address - * can still be found in the cleanup handler - */ - ngx_pool_cleanup_t *cln; - for (cln = r->pool->cleanup; cln; cln = cln->next) { - if (cln->handler == ngx_http_modsecurity_cleanup) { - ctx = cln->data; - break; - } + if (ctx != NULL && ctx->r == r && ctx->modsec_transaction != NULL && + !ctx->request_policy_superseded && + (ngx_http_modsecurity_same_policy(ctx->lifetime.conf, mcf) || + ngx_http_modsecurity_is_block_error_page(r, ctx))) { + return ctx; + } + /* Internal redirects reset module contexts but retain the same request. + * Subrequests share its pool, so cleanup recovery must match the owner. */ + for (cln = r->pool->cleanup; cln; cln = cln->next) { + if (cln->handler != ngx_http_modsecurity_cleanup) { + continue; + } + candidate = cln->data; + if (candidate != NULL && candidate->r == r && + candidate->modsec_transaction != NULL && + ((!candidate->intervention_triggered && !candidate->logged && + !candidate->request_policy_superseded && + !candidate->intervention_failed && !candidate->phase4_failed && + ngx_http_modsecurity_same_policy(candidate->lifetime.conf, mcf)) || + ngx_http_modsecurity_is_block_error_page(r, candidate))) { + ngx_http_set_ctx(r, candidate, ngx_http_modsecurity_module); + return candidate; } } - return ctx; + return NULL; +} + +ngx_int_t +ngx_http_modsecurity_log_request_contexts(ngx_http_request_t *r) +{ + ngx_pool_cleanup_t *cln; + ngx_http_modsecurity_ctx_t *ctx; + ngx_int_t result = NGX_OK; + + if (r == NULL || r->pool == NULL) { + return NGX_ERROR; + } + for (cln = r->pool->cleanup; cln != NULL; cln = cln->next) { + if (cln->handler != ngx_http_modsecurity_cleanup || cln->data == NULL) { + continue; + } + ctx = cln->data; + if (ctx->r == r && ngx_http_modsecurity_log_context(ctx) != NGX_OK) { + result = NGX_ERROR; + } + } + return result; } char * @@ -378,6 +756,7 @@ ngx_conf_set_rules(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) mmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_modsecurity_module); mmcf->rules_inline += res; + mcf->rules_configured = 1; return NGX_CONF_OK; } @@ -412,6 +791,7 @@ ngx_conf_set_rules_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) mmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_modsecurity_module); mmcf->rules_file += res; + mcf->rules_configured = 1; return NGX_CONF_OK; } @@ -451,6 +831,7 @@ ngx_conf_set_rules_remote(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) mmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_modsecurity_module); mmcf->rules_remote += res; + mcf->rules_configured = 1; return NGX_CONF_OK; } @@ -485,6 +866,52 @@ char *ngx_conf_set_transaction_id(ngx_conf_t *cf, ngx_command_t *cmd, void *conf return NGX_CONF_OK; } +static char * +ngx_conf_set_phase4_mode(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_modsecurity_conf_t *mcf = conf; + ngx_str_t *value = cf->args->elts; + (void) cmd; + if (mcf->phase4_mode != NGX_CONF_UNSET_UINT) return "is duplicate"; + if (ngx_strcmp(value[1].data, "off") == 0) mcf->phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_OFF; + else if (ngx_strcmp(value[1].data, "safe") == 0) mcf->phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_SAFE; + else if (ngx_strcmp(value[1].data, "strict") == 0) mcf->phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_STRICT; + else return "invalid value for modsecurity_phase4_mode (expected off|safe|strict)"; + return NGX_CONF_OK; +} + +/* Keep legacy configurations parseable; use SecResponseBodyLimit in rules. */ +static char * +ngx_conf_set_phase4_body_limit(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_modsecurity_conf_t *mcf = conf; + ngx_str_t *value = cf->args->elts; + ssize_t limit; + + (void) cmd; + if (mcf->phase4_body_limit != NGX_CONF_UNSET_SIZE) { + return "is duplicate"; + } + limit = ngx_parse_size(&value[1]); + if (limit <= 0) { + return "invalid value for modsecurity_phase4_body_limit"; + } + mcf->phase4_body_limit = (size_t) limit; + return NGX_CONF_OK; +} + +static char * +ngx_conf_set_phase4_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_modsecurity_conf_t *mcf = conf; + ngx_str_t *value = cf->args->elts; + mcf->phase4_log_path = value[1]; + mcf->phase4_log_file = ngx_conf_open_file(cf->cycle, &mcf->phase4_log_path); + if (mcf->phase4_log_file == NULL) { + return NGX_CONF_ERROR; + } + return NGX_CONF_OK; +} static ngx_command_t ngx_http_modsecurity_commands[] = { { @@ -527,6 +954,30 @@ static ngx_command_t ngx_http_modsecurity_commands[] = { 0, NULL }, + { + ngx_string("modsecurity_phase4_mode"), + NGX_HTTP_LOC_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_phase4_mode, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL + }, + { + ngx_string("modsecurity_phase4_body_limit"), + NGX_HTTP_LOC_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_phase4_body_limit, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL + }, + { + ngx_string("modsecurity_phase4_log"), + NGX_HTTP_LOC_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_phase4_log, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL + }, { ngx_string("modsecurity_use_error_log"), NGX_HTTP_LOC_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_MAIN_CONF|NGX_CONF_FLAG, @@ -727,9 +1178,15 @@ ngx_http_modsecurity_create_conf(ngx_conf_t *cf) conf->enable = NGX_CONF_UNSET; conf->rules_set = msc_create_rules_set(); + conf->rules_identity = conf->rules_set; conf->pool = cf->pool; conf->transaction_id = NGX_CONF_UNSET_PTR; conf->use_error_log = NGX_CONF_UNSET; + conf->phase4_mode = NGX_CONF_UNSET_UINT; + conf->phase4_body_limit = NGX_CONF_UNSET_SIZE; + conf->phase4_log_file = NGX_CONF_UNSET_PTR; + conf->phase4_log_path.len = 0; + conf->phase4_log_path.data = NULL; #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) conf->sanity_checks_enabled = NGX_CONF_UNSET; #endif @@ -770,6 +1227,10 @@ ngx_http_modsecurity_merge_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(c->enable, p->enable, 0); ngx_conf_merge_ptr_value(c->transaction_id, p->transaction_id, NULL); ngx_conf_merge_value(c->use_error_log, p->use_error_log, 1); + ngx_conf_merge_uint_value(c->phase4_mode, p->phase4_mode, NGX_HTTP_MODSEC_PHASE4_MODE_OFF); + ngx_conf_merge_size_value(c->phase4_body_limit, p->phase4_body_limit, + NGX_HTTP_MODSEC_PHASE4_BODY_LIMIT_DEFAULT); + ngx_conf_merge_ptr_value(c->phase4_log_file, p->phase4_log_file, NULL); #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) ngx_conf_merge_value(c->sanity_checks_enabled, p->sanity_checks_enabled, 0); #endif @@ -786,6 +1247,10 @@ ngx_http_modsecurity_merge_conf(ngx_conf_t *cf, void *parent, void *child) return strdup(error); } + if (!c->rules_configured) { + c->rules_identity = p->rules_identity; + } + #if defined(MODSECURITY_DDEBUG) && (MODSECURITY_DDEBUG) dd("NEW CHILD RULES"); msc_rules_dump(c->rules_set); diff --git a/tests/README.md b/tests/README.md index 3c9bdb18..737e40cd 100644 --- a/tests/README.md +++ b/tests/README.md @@ -7,4 +7,80 @@ For more information about those tests, read the subsection "Testing your patch" on the project's README file. For more about nginx tests, check their repository: -http://hg.nginx.org/nginx-tests/ \ No newline at end of file +https://github.com/nginx/nginx-tests + +## Standalone Phase4 helper regressions + +Run from the repository root with Python 3 and GCC or Clang: + +```sh +CC=gcc python3 -m unittest discover -s tests -p 'test_*runtime.py' -v +CC=clang python3 -m unittest discover -s tests -p 'test_*runtime.py' -v +``` + +The suites compile the production body-filter functions, request-context +creation/lookup/cleanup, intervention helpers, and request/audit handlers with +small NGINX and libModSecurity doubles. +They check forwarding above the deprecated connector limit in every mode, +checked accounting, overflow, memory/file buffers, bounded file reads and allocation/read +errors, finalization at EOS, native failures, late interventions, downstream +`NGX_AGAIN`, request ownership, and internal redirect recovery. They need no +Common connector library. The intervention cases run with sanity checks enabled +and disabled and cover redirect ownership, status values, and cleanup. These +are helper behavior tests; they do not replace +the native HTTP integration tests. + +Further regressions exercise invalid native result values, nondisruptive +outputs, exact native response-limit rejection, valid empty EOS markers, +failed filter re-entry (including NULL-input flushes), a synchronous core +error response, and early rejection after a single file chunk. Request tests +exercise internal target-location isolation, error-page P1/P2 continuation, +once-only audit logging, failed context creation and custom-ID termination. +Review follow-ups cover same-policy redirect reuse, changed-policy audit owners, +exact block-page routing, file-backed request ProcessPartial, saturated counters, +replacement control chains, language-independent append-limit classification, +complete metadata parsing and valid JSON for control/UTF-8/invalid-byte inputs. + +## Native Phase4 integration tests + +Copy `tests/*.t` and `tests/*.pl` into an nginx-tests checkout after building +NGINX with this connector and libModSecurity. From that checkout, run: + +```sh +TEST_NGINX_BINARY=/absolute/path/to/nginx prove modsecurity-phase4-*.t +TEST_NGINX_BINARY=/absolute/path/to/nginx prove modsecurity*.t +``` + +The Phase4 suites cover `off` (the default), `safe`, and `strict`, JSON event +logging, engine-owned `SecResponseBodyMimeType` selection, complete response +delivery above the deprecated connector limit in every mode, and engine-owned +`SecResponseBodyLimit` behavior with `Reject` and `ProcessPartial`. They also +reject the removed `minimal` mode and connector +MIME directive. Late deny/redirect assertions check transport interruption, +because headers have already been committed; they do not promise a clean 403. +Native engine Reject is an error in every mode, including safe; it does not +emit a rule-policy JSON event or attempt a second response header. The error +page suite also verifies target-location P1/P2 blocking after internal routing. + +A separate GitHub workflow job runs the standalone suite immediately with +both Linux compilers, independently of the libModSecurity build. The native +Perl suites run in the Linux and Windows build jobs. + +`modsecurity-phase4-subrequest.t` checks that a disabled, header-only +`auth_request` response cannot consume the main transaction's response headers +or MIME selection. The main response must still run its Phase3 header rule and +Phase4 body rule. Direct subrequest EOS ownership is exercised separately by +the compiled helper suite; the auth fixture does not claim to emit a body EOS. + +## HTTP/2 reset delivery regression + +`modsecurity-phase4-h2-reset.t` uses Test::Nginx::HTTP2 and controlled proxy +backends. It requires a prompt Strict reset while an allowed sibling remains +active, then checks sibling completion and connection reuse. Normal CI runs +it with the bundled NGINX host-core correction applied. + +Only the controlled unpatched negative run sets +`TEST_NGINX_H2_RESET_EXPECT_QUEUED=1`. That run requires the reset to remain +queued during the passive wait and to arrive after the held sibling is +released. The selected Linux job subsequently applies the patch, rebuilds +NGINX, and runs the normal tests without that variable. diff --git a/tests/modsecurity-config-audit-redirect.t b/tests/modsecurity-config-audit-redirect.t new file mode 100644 index 00000000..d2cf89f6 --- /dev/null +++ b/tests/modsecurity-config-audit-redirect.t @@ -0,0 +1,99 @@ +#!/usr/bin/perl + +# Preserve native audit owners across same-policy, changed-policy and disabled +# internal targets. These checks require a real NGINX/libModSecurity build. + +use warnings; +use strict; +use Test::More; +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http/)->plan(8); + +$t->write_file_expand('nginx.conf', <<'EOF'); +%%TEST_GLOBALS%% +daemon off; +events {} +http { + %%TEST_GLOBALS_HTTP%% + server { + listen 127.0.0.1:8080; + server_name localhost; + modsecurity on; + modsecurity_rules ' + SecRuleEngine On + '; + location /same/ { + modsecurity_rules ' + SecAuditEngine On + SecAuditLogType Serial + SecAuditLogParts ABZ + SecAuditLog %%TESTDIR%%/audit-source.log + '; + try_files $uri /same/index.html; + } + location = /changed-start { + modsecurity_rules ' + SecAuditEngine On + SecAuditLogType Serial + SecAuditLogParts ABZ + SecAuditLog %%TESTDIR%%/audit-source.log + '; + try_files /nonexistent-audit-source /changed-final; + } + location = /changed-final { + modsecurity_rules ' + SecAuditEngine On + SecAuditLogType Serial + SecAuditLogParts ABZ + SecAuditLog %%TESTDIR%%/audit-target.log + '; + } + location = /disabled-start { + modsecurity_rules ' + SecAuditEngine On + SecAuditLogType Serial + SecAuditLogParts ABZ + SecAuditLog %%TESTDIR%%/audit-source.log + '; + try_files /nonexistent-audit-source /disabled-final; + } + location = /disabled-final { + modsecurity off; + } + } +} +EOF + +mkdir($t->testdir() . '/same'); +$t->write_file('same/index.html', 'same-policy-response'); +$t->write_file('changed-final', 'changed-policy-response'); +$t->write_file('disabled-final', 'disabled-policy-response'); +$t->run(); + +my $same = http_get('/same/missing'); +like($same, qr/200 OK/, 'same-policy target succeeds'); +like($same, qr/same-policy-response/, 'same-policy response preserved'); +my $changed = http_get('/changed-start'); +like($changed, qr/200 OK/, 'changed-policy target succeeds'); +like($changed, qr/changed-policy-response/, 'changed-policy response preserved'); +my $disabled = http_get('/disabled-start'); +like($disabled, qr/200 OK/, 'disabled target succeeds'); +like($disabled, qr/disabled-policy-response/, 'disabled response preserved'); + +# Each location owns its audit settings: the engine's rules merge otherwise +# lets an explicitly configured parent audit path override a child path. +# Stop first so the native serial audit epilogues are fully written. +$t->stop(); +my $source = $t->read_file('audit-source.log'); +my $target = $t->read_file('audit-target.log'); +my $source_records = () = $source =~ /^GET /mg; +my $target_records = () = $target =~ /^GET /mg; +is($source_records, 3, + 'one source audit per request, including changed and disabled targets'); +is($target_records, 1, 'only the changed policy creates a second audit owner'); diff --git a/tests/modsecurity-config-custom-error-page.t b/tests/modsecurity-config-custom-error-page.t index a8f58622..be73b528 100644 --- a/tests/modsecurity-config-custom-error-page.t +++ b/tests/modsecurity-config-custom-error-page.t @@ -103,6 +103,81 @@ http { location / { } } + + server { + listen 127.0.0.1:8080; + server_name s3; + root %%TESTDIR%%/http; + + error_page 404 = /protected-error.txt; + + location = /trigger { + modsecurity on; + modsecurity_rules ' + SecRuleEngine On + SecAction "id:910501,phase:1,pass,nolog" + '; + try_files /missing-error-source =404; + } + + location = /protected-error.txt { + internal; + modsecurity on; + modsecurity_rules ' + SecRuleEngine On + SecRule REQUEST_HEADERS:X-Phase1-Deny "@streq yes" "id:910502,phase:1,deny,status:451,nolog" + SecRule REQUEST_HEADERS:X-Phase2-Deny "@streq yes" "id:910503,phase:2,deny,status:403,nolog" + '; + } + } + + server { + listen 127.0.0.1:8080; + server_name s4; + root %%TESTDIR%%/http; + modsecurity on; + modsecurity_rules ' + SecRuleEngine On + SecRule REQUEST_HEADERS:X-Scanner "@streq yes" "id:910510,phase:1,deny,status:403,log" + SecRule REQUEST_HEADERS:X-Body-Deny "@streq yes" "id:910511,phase:2,deny,status:403,log" + SecAuditEngine On + SecAuditLogParts ABHZ + SecAuditLog %%TESTDIR%%/auditlog-redirect-all.txt + SecAuditLogType Serial + '; + error_page 403 /block.html; + + location = /attack.html { } + location = /block.html { internal; } + location = /same-origin { try_files /missing-same-origin /same-target.html; } + location = /same-target.html { } + location = /changed-origin { try_files /missing-changed-origin /changed-target.html; } + location = /changed-target.html { + modsecurity_rules 'SecAction "id:910514,phase:1,pass,nolog"'; + } + } + + server { + listen 127.0.0.1:8080; + server_name s5; + root %%TESTDIR%%/http; + client_body_buffer_size 64k; + modsecurity on; + modsecurity_rules ' + SecRuleEngine On + SecRequestBodyAccess On + SecRequestBodyLimit 11 + SecRequestBodyLimitAction ProcessPartial + SecRule REQUEST_BODY "@beginsWith partial-hit" "id:910520,phase:2,deny,status:409,nolog" + SecRule REQUEST_BODY "@contains beyond-limit" "id:910521,phase:2,deny,status:418,nolog" + SecAuditEngine Off + '; + error_page 405 =200 /partial-accepted.txt; + location = /file-partial { client_body_in_file_only clean; } + location = /memory-partial { } + location = /partial-accepted.txt { internal; modsecurity off; } + } + } EOF @@ -112,9 +187,17 @@ my $custom_txt = "This is a custom error page."; $t->write_file("/index.html", $index_txt); mkdir($t->testdir() . '/http'); $t->write_file("/http/403.html", $custom_txt); +my $protected_txt = "This is a protected error target."; +$t->write_file("/http/protected-error.txt", $protected_txt); +my $block_txt = "Configured WAF block page."; +$t->write_file('/http/block.html', $block_txt); +for my $name (qw/attack.html same-target.html changed-target.html file-partial memory-partial/) { + $t->write_file('/http/' . $name, 'allowed static response'); +} +$t->write_file('/http/partial-accepted.txt', 'accepted partial request'); $t->run(); -$t->plan(10); +$t->plan(29); ############################################################################### @@ -160,13 +243,81 @@ unlike($global, qr/what=other/, 'ModSecurity at server / other not present in au like($local, qr/Access denied with code 403/, 'ModSecurity at location / 403 in auditlog'); like($global, qr/Access denied with code 403/, 'ModSecurity at server / 403 in auditlog'); +# The origin runs access processing before try_files redirects to a different +# protected location. The target must get its own transaction and run P1/P2. +my $target_allow = http_get_host('s3', '/trigger'); +my $target_p1 = http_get_host('s3', '/trigger', 'X-Phase1-Deny: yes'); +my $target_p2 = http_get_host('s3', '/trigger', 'X-Phase2-Deny: yes'); +like($target_allow, qr/^HTTP\/1\.[01] 200/, 'error-page target allows legitimate request'); +like($target_allow, qr/\Q$protected_txt\E/, 'error-page target delivers legitimate content'); +like($target_p1, qr/^HTTP\/1\.[01] 451/, 'error-page target preserves P1 deny'); +like($target_p2, qr/^HTTP\/1\.[01] 403/, 'error-page target continues from P1 allow to P2 deny'); + +# A configured block page must not run the original scanner/header rules again. +my $blocked_p1 = http_get_host('s4', '/attack.html?marker=p1-scan', 'X-Scanner: yes'); +my $blocked_p2 = http_get_host('s4', '/attack.html?marker=p2-scan', 'X-Body-Deny: yes'); +like($blocked_p1, qr/^HTTP\/1\.[01] 403/, 'P1 denial keeps its status through the configured block page'); +like($blocked_p1, qr/\Q$block_txt\E/, 'P1 denial serves the configured block page'); +like($blocked_p2, qr/^HTTP\/1\.[01] 403/, 'P2 denial keeps its status through the configured block page'); +like($blocked_p2, qr/\Q$block_txt\E/, 'P2 denial serves the configured block page'); + +my $same_policy = http_get_host('s4', '/same-origin?marker=same-policy'); +my $changed_policy = http_get_host('s4', '/changed-origin?marker=changed-policy'); +like($same_policy, qr/^HTTP\/1\.[01] 200/, 'equivalent inherited policy permits the internal redirect'); +like($changed_policy, qr/^HTTP\/1\.[01] 200/, 'changed target policy permits its legitimate control'); +my $redirect_audit = do { + local $/ = undef; + open my $fh, '<', "$d/auditlog-redirect-all.txt" or die "could not open: $!"; + <$fh>; +}; +# Part B has one exact HTTP request line per serial native audit transaction. +for my $observation ( + ['/attack.html?marker=p1-scan', 1, 'P1 block is audited once'], + ['/attack.html?marker=p2-scan', 1, 'original P2 block is audited once'], + ['/same-origin?marker=same-policy', 1, 'equivalent inherited policy does not duplicate audits'], + ['/changed-origin?marker=changed-policy', 2, 'both changed-policy transaction owners are audited'], +) { + my ($uri, $expected, $name) = @$observation; + my $count = () = $redirect_audit =~ /^GET \Q$uri\E HTTP\/1\.[01]\r?$/mg; + is($count, $expected, $name); +} + +# The retained prefix is inspected identically in memory and a forced temp file. +# The allowed control's forbidden tail must stay beyond the engine-owned limit. +my $partial_hit = 'partial-hit-beyond-limit'; +my $partial_allow = 'allowed-xxxbeyond-limit'; +my $file_hit = http_post_host('s5', '/file-partial', $partial_hit); +my $memory_hit = http_post_host('s5', '/memory-partial', $partial_hit); +my $file_allow = http_post_host('s5', '/file-partial', $partial_allow); +my $memory_allow = http_post_host('s5', '/memory-partial', $partial_allow); +like($file_hit, qr/^HTTP\/1\.[01] 409/, 'file-backed ProcessPartial runs P2 on the retained prefix'); +like($memory_hit, qr/^HTTP\/1\.[01] 409/, 'memory ProcessPartial runs P2 on the retained prefix'); +like($file_allow, qr/^HTTP\/1\.[01] 200/, 'file-backed ProcessPartial permits its retained-prefix control'); +like($memory_allow, qr/^HTTP\/1\.[01] 200/, 'memory ProcessPartial permits its retained-prefix control'); +like($file_allow, qr/accepted partial request/, 'file-backed partial control delivers the expected response'); + ############################################################################### +sub http_post_host { + my ($host, $url, $body) = @_; + my $length = length($body); + return http(<new()->has(qw/http/); +$t->write_file_expand('nginx.conf', <<'EOC'); +%%TEST_GLOBALS%% +daemon off; +events {} +http { + %%TEST_GLOBALS_HTTP%% + server { + listen 127.0.0.1:8080; + server_name localhost; + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log %%TESTDIR%%/phase4-content-types.log; + # Separate loads exercise the engine's sequential clear/add merge. + modsecurity_rules 'SecResponseBodyMimeType text/plain'; + modsecurity_rules 'SecResponseBodyMimeTypesClear'; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyMimeType application/json + SecRule RESPONSE_BODY "@rx HIT" "id:920001,phase:4,deny,log,status:403" + '; + location /json { default_type application/json; } + location /unknown { default_type image/png; } + location /plain { default_type text/plain; } + } +} +EOC + +$t->write_file('/json', 'HIT JSON'); +$t->write_file('/unknown', 'HIT PNG'); +$t->write_file('/plain', 'HIT PLAIN'); +$t->run(); +$t->plan(8); + +is(http_get('/json'), '', 'engine-selected JSON triggers strict late abort'); +like(http_get('/unknown'), qr/HIT PNG/, 'engine-excluded image body continues'); +like(http_get('/plain'), qr/HIT PLAIN/, 'engine MIME clear/add override excludes inherited text/plain'); +my $log = $t->read_file('phase4-content-types.log'); +my @events; +my $valid_json = eval { + @events = map { decode_json($_) } grep { length $_ } split /\n/, $log; + 1; +}; +ok($valid_json, 'MIME test event is valid JSON'); +diag($@) unless $valid_json; +is(scalar @events, 1, 'only engine-inspected matching body causes an event'); +is($events[0]->{content_type}, 'application/json', 'engine-selected content type logged'); +is($events[0]->{actual_action}, 'connection_abort', 'JSON intervention abort logged'); +unlike($log, qr/HIT JSON|HIT PNG|HIT PLAIN/, 'no response body is copied to Phase4 log'); diff --git a/tests/modsecurity-phase4-engine-limits.t b/tests/modsecurity-phase4-engine-limits.t new file mode 100644 index 00000000..6cc36a62 --- /dev/null +++ b/tests/modsecurity-phase4-engine-limits.t @@ -0,0 +1,94 @@ +#!/usr/bin/perl +use warnings; use strict; +use Test::More; +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +# Exercise libModSecurity's real append result and queued intervention. The +# static file exceeds the connector's file-reading chunk size; its legacy +# limit is deliberately smaller than the engine's inspection limit. +my $t = Test::Nginx->new()->has(qw/http/); +my $prefix = 'ENGINE_PREFIX_' . ('A' x 50); # exactly 64 bytes +my $body = $prefix . ('B' x 70000) . 'ENGINE_TAIL_MARKER'; +my $locations = ''; +my $rule_id = 950000; + +for my $action (qw/partial reject/) { + my $engine_action = $action eq 'partial' ? 'ProcessPartial' : 'Reject'; + for my $mode (qw/off safe strict/) { + my $prefix_id = ++$rule_id; + my $tail_id = ++$rule_id; + $locations .= <write_file('/' . $action . '-' . $mode, $body); + } +} + +$t->write_file_expand('nginx.conf', <run(); +$t->plan(18); + +for my $mode (qw/off safe strict/) { + my $response = http_get('/partial-' . $mode); + like($response, qr/^HTTP\/1\.1 200 OK/, "$mode accepts engine ProcessPartial"); + my ($headers, $payload) = split /\x0d\x0a\x0d\x0a/, $response, 2; + is($payload, $body, "$mode ProcessPartial forwards the entire static file"); +} + +# Static response headers have already passed the header filter when the +# engine queues its body-limit 403. Reject is an engine failure in every mode, +# not a rule intervention that Safe may downgrade. No body reaches downstream. +for my $mode (qw/off safe strict/) { + is(http_get('/reject-' . $mode), '', "$mode stops native engine Reject"); +} + +$t->stop(); +my $log = $t->read_file('phase4-engine-limits.log'); +is($log, '', 'native Reject and ProcessPartial never emit rule-policy events'); + +my $error_log = $t->read_file('error.log'); +for my $mode (qw/off safe strict/) { + is(scalar(() = $error_log =~ /engine-prefix-exact64-partial-\Q$mode\E/g), 1, + "$mode engine RESPONSE_BODY is exactly the first 64 bytes"); +} +unlike($error_log, qr/engine-tail-seen-/, 'engine inspection excludes the tail in every mode'); +unlike($error_log, qr/engine-prefix-exact64-reject-/, + 'engine Reject does not expose a partial inspection prefix'); +unlike($error_log, qr/response body inspection append failed/, + 'the engine Reject append result is not mistaken for an API failure'); + +is(scalar(() = $error_log =~ /response body limit rejected the response/g), 3, + 'every rejected response retains its native-limit diagnostic'); +unlike($error_log, qr/\[alert\]/, 'native Reject never attempts a second response header'); diff --git a/tests/modsecurity-phase4-h2-reset.t b/tests/modsecurity-phase4-h2-reset.t new file mode 100644 index 00000000..c8bcceef --- /dev/null +++ b/tests/modsecurity-phase4-h2-reset.t @@ -0,0 +1,239 @@ +#!/usr/bin/perl + +# Exercise Strict phase-4 stream finalization while another HTTP/2 stream is +# active. The auth backend resumes static content outside the HTTP/2 read +# handler; its small, unbuffered final body reaches the filter as last_buf. + +use warnings; +use strict; + +use Test::More; +use IO::Socket::INET; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; +use Test::Nginx::HTTP2; + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $owner = $$; +my $t = Test::Nginx->new()->has(qw/http http_v2 proxy auth_request/)->plan(21); +my $s; +my $prefix = 'allowed-prefix-'; +my $tail = 'allowed-tail'; +my $forbidden = 'H2-STRICT-FORBIDDEN-TAIL'; +my $expect_queued = $ENV{TEST_NGINX_H2_RESET_EXPECT_QUEUED} || 0; + +# Release blocked backends on failed assertions or an early bailout as well. +# Daemon children inherit END blocks and must not release the parent's gates. +END { + if ($$ == $owner && defined $t) { + eval { + $t->write_file('hold-release', "\n"); + $t->write_file('strict-release', "\n"); + }; + undef $s; + undef $t; + } +} + +$t->write_file_expand('nginx.conf', <<'EOF'); +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + default_type text/plain; + sendfile off; + postpone_output 0; + + server { + listen 127.0.0.1:8080 http2; + server_name localhost; + root %%TESTDIR%%; + + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log %%TESTDIR%%/strict-h2.log; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyMimeType text/plain + SecRule RESPONSE_BODY "@contains H2-STRICT-FORBIDDEN-TAIL" "id:942377,phase:4,deny,status:403,log" + '; + + location = /hold { + proxy_pass http://127.0.0.1:8081; + proxy_buffering off; + proxy_read_timeout 30s; + } + + location = /strict { + auth_request /authorize-strict; + } + + location = /authorize-strict { + internal; + modsecurity off; + proxy_pass http://127.0.0.1:8082; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_read_timeout 30s; + } + + location = /allow { + } + } +} +EOF + +$t->write_file('strict', $forbidden); +$t->write_file('allow', 'allowed-after-reset'); +$t->run_daemon(\&backend, 'hold', port(8081), $t->testdir()); +$t->run_daemon(\&backend, 'strict', port(8082), $t->testdir()); +$t->run(); +$t->waitforsocket('127.0.0.1:' . port(8081)) or BAIL_OUT('hold backend unavailable'); +$t->waitforsocket('127.0.0.1:' . port(8082)) or BAIL_OUT('auth backend unavailable'); + +$s = Test::Nginx::HTTP2->new(); +my $held = $s->new_stream({ path => '/hold' }); +my $frames = $s->read(all => [{ sid => $held, length => length($prefix) }]); +is(stream_status($frames, $held), 200, 'held stream has response headers'); +is(stream_body($frames, $held), $prefix, 'held stream starts delivering allowed data'); +ok(!stream_finished($frames, $held), 'held stream remains active'); + +my $strict = $s->new_stream({ path => '/strict' }); +my $ready = $t->waitforfile($t->testdir() . '/strict-ready'); +ok($ready, 'strict request reached asynchronous auth gate'); +BAIL_OUT('strict auth request did not reach its gate') unless $ready; + +# Settle all earlier input, including the constructor's SETTINGS ACK, while +# auth is still blocked. Reading frames never generates acknowledgements. +$s->h2_ping('gatewait'); +my $barrier = $s->read(all => [{ type => 'PING' }], wait => 3); +push @$frames, @$barrier; +my $settled = grep { $_->{type} eq 'PING' && ($_->{flags} & 1) + && $_->{value} eq 'gatewait' } @$barrier; +ok($settled, 'PING acknowledgement settles client input before auth release'); +BAIL_OUT('HTTP/2 input did not settle before auth release') unless $settled; +$t->write_file('strict-release', "\n"); + +# No client frame is sent between releasing auth and observing RST_STREAM. +# A real phase-4 event is required: a broken backend is never baseline proof. +my $decision = wait_for_decision(); +ok($decision, 'native Strict policy reached connection_abort'); +BAIL_OUT('native Strict phase-4 decision was not recorded') unless $decision; +my $prompt = $s->read(all => [{ type => 'RST_STREAM' }], wait => 1); +push @$frames, @$prompt; +is(!!stream_reset($prompt, $strict), !$expect_queued, + $expect_queued ? 'unpatched core queues reset while another stream is active' + : 'Strict reset arrives promptly while another stream is active'); +ok(!stream_finished($frames, $held) && !stream_reset($frames, $held), + 'held stream is still active during reset observation'); + +$t->write_file('hold-release', "\n"); +my @remaining = ({ sid => $held, fin => 1 }); +push @remaining, { type => 'RST_STREAM' } unless stream_reset($frames, $strict); +my $released = $s->read(all => \@remaining, wait => 5); +push @$frames, @$released; +ok(stream_reset($frames, $strict), 'strict stream eventually receives a definite reset'); +is(stream_status($frames, $strict), 200, 'strict response started before its late denial'); +unlike(stream_body($frames, $strict), qr/\Q$forbidden\E/, + 'Strict final body is withheld'); +ok(!stream_finished($frames, $strict), 'strict stream has no successful end of response'); +is(stream_body($frames, $held), $prefix . $tail, 'unrelated stream delivers its complete body'); +ok(stream_finished($frames, $held), 'unrelated stream ends normally'); +ok(!stream_reset($frames, $held), 'unrelated stream is not reset'); +ok(!grep({ $_->{type} eq 'GOAWAY' } @$frames), 'multiplexed connection is not closed'); + +my $allowed = $s->new_stream({ path => '/allow' }); +my $after = $s->read(all => [{ sid => $allowed, fin => 1 }], wait => 5); +is(stream_status($after, $allowed), 200, 'same connection accepts another request'); +is(stream_body($after, $allowed), 'allowed-after-reset', 'later allowed response is complete'); +ok(stream_finished($after, $allowed), 'later allowed response ends normally'); +ok(!grep({ $_->{type} eq 'GOAWAY' } @$after), 'reused connection has no GOAWAY'); +ok(!stream_reset($after, $allowed), 'later allowed stream is not reset'); + +undef $s; +# Run Test::Nginx's alert/sanitizer assertions before Test::More finalizes TAP. +undef $t; + +sub stream_body { + my ($list, $sid) = @_; + return join '', map { $_->{data} } + grep { $_->{type} eq 'DATA' && $_->{sid} == $sid } @$list; +} + +sub stream_status { + my ($list, $sid) = @_; + my ($header) = grep { $_->{type} eq 'HEADERS' && $_->{sid} == $sid } @$list; + return $header ? $header->{headers}->{':status'} : undef; +} + +sub stream_finished { + my ($list, $sid) = @_; + return scalar grep { $_->{sid} == $sid && ($_->{flags} & 1) + && ($_->{type} eq 'HEADERS' || $_->{type} eq 'DATA') } @$list; +} + +sub stream_reset { + my ($list, $sid) = @_; + return scalar grep { $_->{type} eq 'RST_STREAM' && $_->{sid} == $sid } @$list; +} + +sub wait_for_decision { + for (1 .. 100) { + my $log = $t->read_file('strict-h2.log'); + return 1 if defined $log && $log =~ /"uri":"\/strict"[^\n]*"actual_action":"connection_abort"/; + select undef, undef, undef, 0.05; + } + return 0; +} + +sub backend { + my ($kind, $listen, $dir) = @_; + my $server = IO::Socket::INET->new(Proto => 'tcp', + LocalHost => '127.0.0.1', LocalPort => $listen, + Listen => 5, Reuse => 1) or die "listen: $!"; + local $SIG{PIPE} = 'IGNORE'; + while (my $client = $server->accept()) { + $client->autoflush(1); + binmode $client; + my $headers = ''; + while (my $line = <$client>) { + $headers .= $line; + last if $line =~ /^\r?\n$/; + } + unless (length $headers) { + close $client; + next; + } + open my $ready, '>', "$dir/$kind-ready" or die "ready: $!"; + close $ready; + if ($kind eq 'hold') { + print $client "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n" + . 'Content-Length: ' . length($prefix . $tail) + . "\r\nConnection: close\r\n\r\n$prefix"; + } + for (1 .. 1500) { + last if -e "$dir/$kind-release"; + select undef, undef, undef, 0.02; + } + die "gate timed out: $kind" unless -e "$dir/$kind-release"; + if ($kind eq 'hold') { + print $client $tail; + } else { + print $client "HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n" + . "Connection: close\r\n\r\n"; + } + close $client; + } +} diff --git a/tests/modsecurity-phase4-invalid-config.t b/tests/modsecurity-phase4-invalid-config.t new file mode 100644 index 00000000..1c2c7969 --- /dev/null +++ b/tests/modsecurity-phase4-invalid-config.t @@ -0,0 +1,38 @@ +#!/usr/bin/perl +use warnings; use strict; +use Test::More; +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +my $t = Test::Nginx->new()->has(qw/http/)->plan(3); +# Test::Nginx always reads this file in its two cleanup assertions, even +# when nginx -t exits during configuration parsing before opening its log. +$t->write_file('error.log', ''); +my $binary = $ENV{TEST_NGINX_BINARY} || ($t->testdir() . '/../nginx'); + +for my $case ( + ['modsecurity_phase4_mode minimal;', qr/invalid value.*minimal|invalid.*phase4.*mode/, 'removed minimal mode is rejected'], + ['modsecurity_phase4_content_types_file removed.conf;', qr/unknown directive.*modsecurity_phase4_content_types_file/, 'removed connector MIME directive is rejected'], + ['modsecurity_phase4_body_limit 0;', qr/(?:must be|invalid|greater than).*0|invalid.*body.*limit|body.*limit.*(?:positive|zero)/, 'zero connector body limit is rejected'], +) { + my ($directive, $expected, $label) = @$case; + $t->write_file_expand('nginx.conf', <<"EOF"); +%%TEST_GLOBALS%% +events {} +http { + %%TEST_GLOBALS_HTTP%% + server { + listen 127.0.0.1:19849; + location / { + modsecurity on; + $directive + return 200 "ok\\n"; + } + } +} +EOF + my $prefix = $t->testdir(); + my $out = `"$binary" -p "$prefix/" -c nginx.conf -e error.log -t 2>&1`; + like($out, $expected, $label); +} diff --git a/tests/modsecurity-phase4-modes.t b/tests/modsecurity-phase4-modes.t new file mode 100644 index 00000000..7c2032f9 --- /dev/null +++ b/tests/modsecurity-phase4-modes.t @@ -0,0 +1,78 @@ +#!/usr/bin/perl +use warnings; use strict; +use Test::More; +use JSON::PP qw(decode_json); +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +# A committed response must fail without attempting another HTTP status. +my $t = Test::Nginx->new()->has(qw/http/); +$t->write_file_expand('nginx.conf', <<'EOC'); +%%TEST_GLOBALS%% +daemon off; +events {} +http { + %%TEST_GLOBALS_HTTP%% + server { + listen 127.0.0.1:8080; + server_name localhost; + default_type text/plain; + modsecurity on; + modsecurity_phase4_log %%TESTDIR%%/phase4.log; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyMimeType text/plain + SecRule RESPONSE_BODY "@rx Hello" "id:910001,phase:4,deny,log,status:403" + '; + location /off { modsecurity_phase4_mode off; } + location /default { } + location /safe { modsecurity_phase4_mode safe; } + location /strict { modsecurity_phase4_mode strict; } + } +} +EOC + +for my $mode (qw/off default safe strict/) { + $t->write_file('/' . $mode, 'Hello ' . $mode); +} +$t->run(); +$t->plan(19); + +is(http_get('/off'), '', 'off preserves native late-deny failure'); +is(http_get('/default'), '', 'off is the default'); +like(http_get('/safe'), qr/Hello safe/, 'safe continues after a late rule intervention'); +is(http_get('/strict'), '', 'strict aborts after headers are committed'); + +my $log = $t->read_file('phase4.log'); +my @events; +my $valid_json = eval { + @events = map { decode_json($_) } grep { length $_ } split /\n/, $log; + 1; +}; +ok($valid_json, 'every Phase4 event is valid JSON'); +diag($@) unless $valid_json; +is(scalar @events, 2, 'off and default off emit no dedicated Phase4 event'); +my %event = map { $_->{uri} => $_ } @events; +is($event{'/safe'}->{actual_action}, 'log_only', 'safe action logged'); +is($event{'/safe'}->{reason}, 'response_committed_safe', 'safe reason logged'); +is($event{'/strict'}->{actual_action}, 'connection_abort', 'strict action logged'); +is($event{'/strict'}->{reason}, 'response_committed_strict', 'strict reason logged'); +is($event{'/safe'}->{mode}, 'safe', 'safe mode logged'); +is($event{'/strict'}->{mode}, 'strict', 'strict mode logged'); +ok(@events && !grep({ !$_->{header_sent} } @events), 'header_sent is true for late interventions'); +ok(@events && !grep({ $_->{event} ne 'phase4_intervention' } @events), 'event kind is stable'); +unlike($log, qr/Hello off|Hello default|Hello safe|Hello strict/, 'response body is absent from Phase4 log'); + +# Stop first so the explicit alert audit includes all worker output. +$t->stop(); +my $error_log = $t->read_file('error.log'); +my @alerts = $error_log =~ /^.*\[alert\].*$/gm; +is(scalar @alerts, 0, 'no nginx alerts after late intervention'); +unlike($error_log, qr/header already sent\b.*request: "GET \/off HTTP\/1\.[01]"/, + 'explicit off never attempts a second response header'); +unlike($error_log, qr/header already sent\b.*request: "GET \/default HTTP\/1\.[01]"/, + 'default off never attempts a second response header'); +is(scalar(() = $error_log =~ /modsecurity phase4 intervention after headers sent, action=connection_abort, uri="\/strict"/g), + 1, 'strict abort is diagnosed once'); diff --git a/tests/modsecurity-phase4-regression.t b/tests/modsecurity-phase4-regression.t new file mode 100644 index 00000000..8e8b62e3 --- /dev/null +++ b/tests/modsecurity-phase4-regression.t @@ -0,0 +1,54 @@ +#!/usr/bin/perl +use warnings; use strict; +use Test::More; +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +my $t = Test::Nginx->new()->has(qw/http/); +my $big = ('A' x 1100000) . 'TAIL'; +$t->write_file_expand('nginx.conf', <<'EOC'); +%%TEST_GLOBALS%% +daemon off; +events {} +http { + %%TEST_GLOBALS_HTTP%% + server { + listen 127.0.0.1:8080; + server_name localhost; + default_type text/plain; + modsecurity on; + modsecurity_phase4_body_limit 64k; + modsecurity_phase4_log %%TESTDIR%%/phase4-regression.log; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyMimeType text/plain + SecResponseBodyLimit 2097152 + SecRule RESPONSE_BODY "@endsWith TAIL" "id:930001,phase:4,pass,log,msg:phase4-tail-seen" + '; + location /off { modsecurity_phase4_mode off; } + location /safe { modsecurity_phase4_mode safe; } + location /strict { modsecurity_phase4_mode strict; } + } +} +EOC + +for my $mode (qw/off safe strict/) { + $t->write_file('/' . $mode, $big); +} +$t->run(); +$t->plan(8); + +for my $mode (qw/off safe strict/) { + my $resp = http_get('/' . $mode); + like($resp, qr/HTTP\/1\.1 200 OK/, "$mode ignores the legacy connector limit"); + my ($headers, $body) = split /\x0d\x0a\x0d\x0a/, $resp, 2; + is($body, $big, "$mode forwards the complete large response"); +} +$t->stop(); +my $error_log = $t->read_file('error.log'); +is(scalar(() = $error_log =~ /phase4-tail-seen/g), 3, + 'the engine inspects the tail beyond the legacy limit in every mode'); +unlike($t->read_file('phase4-regression.log'), qr/A{100,}|TAIL/, + 'response bytes are absent from event logs'); diff --git a/tests/modsecurity-phase4-subrequest.t b/tests/modsecurity-phase4-subrequest.t new file mode 100644 index 00000000..347fa1e2 --- /dev/null +++ b/tests/modsecurity-phase4-subrequest.t @@ -0,0 +1,83 @@ +#!/usr/bin/perl +use warnings; use strict; +use Test::More; +use JSON::PP qw(decode_json); +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +# auth_request shares the main request's pool but owns a separate response. +# Its header-only image response must not consume the main transaction's +# response headers or MIME selection. Direct subrequest EOS is covered by +# the compiled owner-isolation regression; auth_request itself is header-only. +my $t = Test::Nginx->new()->has(qw/http auth_request/); +$t->write_file_expand('nginx.conf', <<'EOC'); +%%TEST_GLOBALS%% +daemon off; +events {} +http { + %%TEST_GLOBALS_HTTP%% + log_format auth_probe '$uri $status'; + server { + listen 127.0.0.1:8080; + server_name localhost; + default_type text/plain; + log_subrequest on; + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log %%TESTDIR%%/phase4-subrequest.log; + modsecurity_rules 'SecResponseBodyMimeTypesClear'; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyMimeType text/plain + SecRule RESPONSE_HEADERS:Content-Type "@contains image/png" "id:940002,phase:3,deny,log,status:403,msg:auth-response-headers-leaked" + SecRule RESPONSE_HEADERS:Content-Type "@contains text/plain" "id:940003,phase:3,pass,log,msg:main-response-headers-seen" + SecRule RESPONSE_BODY "@contains P4-MAIN-MARKER" "id:940001,phase:4,deny,log,status:403" + '; + location = /_auth { + internal; + modsecurity off; + default_type image/png; + access_log %%TESTDIR%%/auth.log auth_probe; + } + location = /control { auth_request /_auth; } + location = /main { auth_request /_auth; } + } +} +EOC + +$t->write_file('/_auth', 'BENIGN AUTH BODY'); +$t->write_file('/control', 'BENIGN MAIN BODY'); +$t->write_file('/main', 'P4-MAIN-MARKER'); +$t->run(); +$t->plan(16); + +my $control = http_get('/control'); +like($control, qr/^HTTP\/1\.1 200 OK/, 'benign main response passes after authorization'); +like($control, qr/BENIGN MAIN BODY/, 'main body remains intact'); +unlike($control, qr/BENIGN AUTH BODY/, 'authorization body is absent from main response'); +is(http_get('/main'), '', 'main Phase4 marker still triggers strict abort after authorization'); +$t->stop(); +my @auth_results = $t->read_file('auth.log') =~ /^\/_auth 200\r?$/gm; +is(scalar @auth_results, 2, 'both authorization subrequests completed successfully'); +my $error_log = $t->read_file('error.log'); +like($error_log, qr/main-response-headers-seen/, 'main response headers are inspected after the subrequest'); +unlike($error_log, qr/auth-response-headers-leaked/, 'auth response headers never enter the main transaction'); + +my $log = $t->read_file('phase4-subrequest.log'); +my @events; +my $valid_json = eval { + @events = map { decode_json($_) } grep { length $_ } split /\n/, $log; + 1; +}; +ok($valid_json, 'subrequest regression event is valid JSON'); +diag($@) unless $valid_json; +is(scalar @events, 1, 'only the matching main response produces a Phase4 event'); +is($events[0]->{uri}, '/main', 'event belongs to the main request'); +is($events[0]->{mode}, 'strict', 'main strict mode is retained'); +is($events[0]->{actual_action}, 'connection_abort', 'main intervention aborts the connection'); +ok($events[0]->{header_sent}, 'main response headers were committed'); +is($events[0]->{waf_status}, 403, 'main rule decision is retained'); +is($events[0]->{rule_id}, '940001', 'main body rule caused the event'); +unlike($log, qr/P4-MAIN-MARKER|BENIGN AUTH BODY|BENIGN MAIN BODY/, 'event log excludes both response bodies'); diff --git a/tests/modsecurity-proxy-h2.t b/tests/modsecurity-proxy-h2.t index e8ef0ad2..df0e43ba 100644 --- a/tests/modsecurity-proxy-h2.t +++ b/tests/modsecurity-proxy-h2.t @@ -86,6 +86,7 @@ http { } location /phase4 { modsecurity on; + modsecurity_phase4_mode strict; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On @@ -270,6 +271,15 @@ EOF select undef, undef, undef, 0.1; print $client 'AND-THIS'; + } elsif ($uri =~ m{^/phase4\?what=(redirect301|redirect302|block401|block403)$}) { + print $client <<"EOF"; +HTTP/1.1 200 OK +Content-Type: text/html +Connection: close + +phase4 trigger +EOF + } else { print $client <<"EOF"; diff --git a/tests/modsecurity-proxy.t b/tests/modsecurity-proxy.t index a412f5ea..a742029f 100644 --- a/tests/modsecurity-proxy.t +++ b/tests/modsecurity-proxy.t @@ -84,6 +84,7 @@ http { } location /phase4 { modsecurity on; + modsecurity_phase4_mode strict; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On @@ -191,6 +192,15 @@ EOF select undef, undef, undef, 0.1; print $client 'AND-THIS'; + } elsif ($uri =~ m{^/phase4\?what=(redirect301|redirect302|block401|block403)$}) { + print $client <<"EOF"; +HTTP/1.1 200 OK +Content-Type: text/html +Connection: close + +phase4 trigger +EOF + } else { print $client <<"EOF"; diff --git a/tests/modsecurity.t b/tests/modsecurity.t index fcb26f4f..ff31c229 100644 --- a/tests/modsecurity.t +++ b/tests/modsecurity.t @@ -95,6 +95,7 @@ http { } location /phase4 { modsecurity on; + modsecurity_phase4_mode strict; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On diff --git a/tests/test_header_protocol_runtime.py b/tests/test_header_protocol_runtime.py new file mode 100644 index 00000000..69758911 --- /dev/null +++ b/tests/test_header_protocol_runtime.py @@ -0,0 +1,198 @@ +"""Exercise the actual synthetic Transfer-Encoding resolver across build modes. + +The C fixture models NGINX callback boundaries only. It checks that HTTP/2 and +HTTP/3 bypass context/sanity/native mutations even when the host chunked flag +is set, while HTTP/1.x preserves the existing resolver result. +""" + +from __future__ import annotations + +import itertools +import os +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +HEADER_FILTER = ROOT / "src" / "ngx_http_modsecurity_header_filter.c" + + +def resolver_definition(source: str) -> str: + name = "ngx_http_modsecurity_resolv_header_transfer_encoding" + for match in re.finditer(rf"\b{name}\s*\(", source): + cursor = source.index(")", match.end()) + 1 + while source[cursor].isspace(): + cursor += 1 + if source[cursor] != "{": + continue + start = source.rfind("\n", 0, match.start()) + 1 + start = source.rfind("\n", 0, start - 1) + 1 + depth = 0 + for index in range(cursor, len(source)): + if source[index] == "{": + depth += 1 + elif source[index] == "}": + depth -= 1 + if depth == 0: + return source[start:index + 1] + raise AssertionError("production Transfer-Encoding resolver was not found") + + +PREAMBLE = r""" +#include +#include +#include +#include +#include +#include + +typedef unsigned char u_char; +typedef intptr_t ngx_int_t; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct { + int chunked; + int http_version; +#if (NGX_HTTP_V2) + void *stream; +#endif +} ngx_http_request_t; +typedef struct { void *modsec_transaction; } ngx_http_modsecurity_ctx_t; +#define ngx_string(value) { sizeof(value) - 1, (u_char *) value } +#if nginx_version >= 1025000 +#define NGX_HTTP_VERSION_30 3000 +#endif + +static int transaction; +static ngx_http_modsecurity_ctx_t context = { &transaction }; +static unsigned context_calls, native_calls, sanity_calls; +static int native_result; + +static ngx_http_modsecurity_ctx_t * +ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r) +{ + (void) r; + context_calls++; + return &context; +} + +#if (MODSECURITY_SANITY_CHECKS) +static int ngx_http_modsecurity_store_ctx_header(ngx_http_request_t *r, + ngx_str_t *name, ngx_str_t *value) +{ + (void) r; + if (name->len != 17 || value->len != 7) abort(); + sanity_calls++; + return 1; +} +#endif + +static int msc_add_n_response_header(void *owner, const unsigned char *name, + size_t name_size, const unsigned char *value, size_t value_size) +{ + if (owner != &transaction || name_size != 17 || value_size != 7 || + memcmp(name, "Transfer-Encoding", 17) != 0 || + memcmp(value, "chunked", 7) != 0) abort(); + native_calls++; + return native_result; +} +""" + +MAIN = r""" +int main(int argc, char **argv) +{ + ngx_http_request_t request; + ngx_str_t name = ngx_string("Transfer-Encoding"); + ngx_int_t result; + + if (argc != 5) return 2; + memset(&request, 0, sizeof(request)); + request.http_version = atoi(argv[1]); + request.chunked = atoi(argv[2]); +#if (NGX_HTTP_V2) + request.stream = atoi(argv[3]) ? &transaction : NULL; +#endif + native_result = atoi(argv[4]); + result = ngx_http_modsecurity_resolv_header_transfer_encoding(&request, + name, 0); + printf("%ld %u %u %u\n", (long) result, native_calls, sanity_calls, + context_calls); + return 0; +} +""" + + +class HeaderProtocolRuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler or shutil.which(compiler[0]) is None: + raise unittest.SkipTest("A C compiler is needed for header protocol regressions") + cls.directory = tempfile.TemporaryDirectory(prefix="modsec-header-protocol-") + cls.addClassCleanup(cls.directory.cleanup) + source = PREAMBLE + resolver_definition(HEADER_FILTER.read_text(encoding="utf-8")) + MAIN + path = Path(cls.directory.name) / "header-protocol.c" + path.write_text(source, encoding="utf-8") + cls.binaries = {} + for v2, sanity, version in itertools.product((0, 1), (0, 1), (1024000, 1031000)): + key = v2, sanity, version + binary = Path(cls.directory.name) / f"header-{v2}-{sanity}-{version}" + completed = subprocess.run( + compiler + ["-std=c99", "-Wall", "-Wextra", "-Werror", + f"-DNGX_HTTP_V2={v2}", + f"-DMODSECURITY_SANITY_CHECKS={sanity}", + f"-Dnginx_version={version}", str(path), "-o", str(binary)], + capture_output=True, text=True, check=False, + ) + if completed.returncode != 0: + raise AssertionError(completed.stdout + completed.stderr) + cls.binaries[key] = binary + + def execute(self, key, protocol, chunked, stream=0, result=73): + completed = subprocess.run( + [str(self.binaries[key]), str(protocol), str(chunked), str(stream), str(result)], + capture_output=True, text=True, check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + return tuple(map(int, completed.stdout.split())) + + def test_http1_chunked_preserves_native_and_sanity_callbacks(self): + for key in self.binaries: + with self.subTest(build=key): + self.assertEqual(self.execute(key, 1001, 1), (73, 1, key[1], 1)) + + def test_http1_unchunked_never_synthesizes_transfer_encoding(self): + for key in self.binaries: + with self.subTest(build=key): + self.assertEqual(self.execute(key, 1001, 0), (1, 0, 0, 0)) + + def test_http2_stream_bypasses_even_a_stale_chunked_flag(self): + for key in self.binaries: + if not key[0]: + continue + for chunked in (0, 1): + with self.subTest(build=key, chunked=chunked): + self.assertEqual(self.execute(key, 2000, chunked, stream=1), (1, 0, 0, 0)) + + def test_http3_bypasses_without_requiring_an_http2_stream_field(self): + for key in self.binaries: + if key[2] < 1025000: + continue + for chunked in (0, 1): + with self.subTest(build=key, chunked=chunked): + self.assertEqual(self.execute(key, 3000, chunked), (1, 0, 0, 0)) + + def test_http1_propagates_native_header_result(self): + for key in self.binaries: + for native_result in (-1, 0, 1): + with self.subTest(build=key, result=native_result): + self.assertEqual(self.execute(key, 1001, 1, result=native_result), + (native_result, 1, key[1], 1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_intervention_runtime.py b/tests/test_intervention_runtime.py new file mode 100644 index 00000000..b317c7bc --- /dev/null +++ b/tests/test_intervention_runtime.py @@ -0,0 +1,636 @@ +"""Exercise production intervention dispatch and retained allocation lifetimes. + +The actual module wrapper, redirect/log helpers, and request-context lookup are +compiled with host doubles. Both sanity-check build variants run each case. +Native NGINX/libModSecurity integration remains the Perl suites' responsibility. +""" + +from __future__ import annotations + +import os +import re +from pathlib import Path +import shlex +import shutil +import subprocess +import tempfile +import unittest + +from test_phase4_runtime import function_definition + +ROOT = Path(__file__).resolve().parents[1] +MODULE = ROOT / "src" / "ngx_http_modsecurity_module.c" + +PREAMBLE = r""" +#include +#include +#include +#include +#include +typedef unsigned char u_char; +typedef int ngx_int_t; +typedef struct ngx_http_request_s ngx_http_request_t; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct { + ngx_str_t key, value; + unsigned hash; +} ngx_table_elt_t; +typedef struct { + ngx_table_elt_t entries[4]; + unsigned count; +} ngx_list_t; +typedef struct ngx_pool_cleanup_s { + void (*handler)(void *); + void *data; + struct ngx_pool_cleanup_s *next; +} ngx_pool_cleanup_t; +typedef struct { + ngx_pool_cleanup_t *cleanup; + u_char storage[4096]; + size_t used; +} ngx_pool_t; +typedef struct { int unused; } Transaction; +typedef struct { + int status; + char *url, *log; + int disruptive; + double pause; +} ModSecurityIntervention; +typedef struct ngx_http_modsecurity_conf_s ngx_http_modsecurity_conf_t; +typedef struct { int named; ngx_str_t name; } ngx_http_core_loc_conf_t; +typedef struct { + ngx_http_request_t *r; + struct { const ngx_http_modsecurity_conf_t *conf; } lifetime; + struct { ngx_str_t target; ngx_int_t status, response_status; } error_page; + Transaction *modsec_transaction; + ModSecurityIntervention pending_request_intervention; + ngx_str_t last_intervention_log; + int last_intervention_status, intervention_late, intervention_failed; + int intervention_redirect_location_installed, logged; + int phase4_failed, phase4_error_emitting, phase4_started, phase4_processed, intervention_body_limit; + int phase4_append_active, request_policy_superseded, intervention_triggered; + int request_intervention_active, request_intervention_pending, native_request_body_limit_rejection; +} ngx_http_modsecurity_ctx_t; +struct ngx_http_modsecurity_conf_s { + void *phase4_log_file; + const void *rules_identity; + void *transaction_id; + int enable, use_error_log, phase4_mode, sanity_checks_enabled; +}; +typedef struct { void *log; } ngx_connection_t; +struct ngx_http_request_s { + int header_sent; + ngx_pool_t *pool; + ngx_connection_t *connection; + ngx_http_modsecurity_ctx_t *ctx; + ngx_http_modsecurity_conf_t *conf; + ngx_http_core_loc_conf_t *core; + int internal, err_status; + ngx_str_t uri; + struct { ngx_list_t headers; ngx_table_elt_t *location; } headers_out; +}; +#define NGX_OK 0 +#define NGX_ERROR (-1) +#define NGX_DECLINED (-5) +#define NGX_HTTP_MODSEC_PHASE4_MODE_OFF 0 +#define NGX_HTTP_BAD_REQUEST 400 +#define NGX_HTTP_FORBIDDEN 403 +#define NGX_HTTP_INTERNAL_SERVER_ERROR 500 +#define NGX_LOG_ERR 3 +#define NGX_LOG_WARN 4 +#define ngx_inline inline +#define ngx_http_modsecurity_module 1 +#define ngx_http_core_module 2 +#define ngx_http_get_module_ctx(r, module) ((r)->ctx) +#define ngx_http_get_module_loc_conf(r, module) ((module) == 2 ? (void *)(r)->core : (void *)(r)->conf) +#define ngx_http_set_ctx(r, c, module) ((r)->ctx = (c)) +#define ngx_strncmp(a, b, n) strncmp((const char *)(a), (const char *)(b), (n)) +#define ngx_memzero(data, size) memset((data), 0, (size)) +#define ngx_memcpy(dst, src, size) memcpy((dst), (src), (size)) +#define ngx_strlen(data) strlen((const char *)(data)) +#define ngx_str_null(str) do { (str)->len = 0; (str)->data = NULL; } while (0) +#define ngx_str_set(str, literal) do { \ + (str)->len = sizeof(literal) - 1; (str)->data = (u_char *)(literal); \ +} while (0) + +static int native_available = 1, native_status = 403, native_calls; +static int native_return = 1, native_disruptive = 1; +static int native_has_log = 1, native_has_url, native_initialized; +static char native_log[4096] = "native intervention message"; +static char native_url[256] = "https://example.test/new-location"; +static int freed_log, freed_url, bad_free, allocation_calls, fail_allocation_at; +static int list_calls, fail_list, clear_calls, error_log_calls; +static int status_calls, updated_status, early_log_calls, sanity_calls, fail_sanity; +static int early_log_result; +static char captured_error[256]; +static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r); +static void ngx_http_modsecurity_cleanup(void *data) { (void)data; } +static void ngx_http_modsecurity_capture_error_page(ngx_http_request_t *r, ngx_http_modsecurity_ctx_t *ctx, ngx_int_t status) +{ (void)r; (void)ctx; (void)status; } + +static int msc_intervention(Transaction *transaction, ModSecurityIntervention *out) +{ + (void)transaction; + ++native_calls; + native_initialized = out->status == 200 && out->log == NULL && + out->url == NULL && out->disruptive == 0 && out->pause == 0.0; + if (!native_available) return 0; + out->status = native_status; + out->log = native_has_log ? native_log : NULL; + out->url = native_has_url ? native_url : NULL; + out->disruptive = native_disruptive; + return native_return; +} +/* Poison released native memory without releasing storage: a retained native + * pointer remains readable but differs from the expected pool-owned value. */ +static void tracked_free(void *pointer) +{ + if (pointer == native_log) ++freed_log; + else if (pointer == native_url) ++freed_url; + else ++bad_free; + memset(pointer, '!', strlen((const char *)pointer)); +} +#define free(pointer) tracked_free(pointer) +static void *ngx_pnalloc(ngx_pool_t *pool, size_t bytes) +{ + u_char *result; + ++allocation_calls; + if (allocation_calls == fail_allocation_at || bytes > sizeof(pool->storage) - pool->used) + return NULL; + result = pool->storage + pool->used; + pool->used += bytes == 0 ? 1 : bytes; + return result; +} +static ngx_table_elt_t *ngx_list_push(ngx_list_t *list) +{ + ++list_calls; + if (fail_list || list->count == 4U) return NULL; + return &list->entries[list->count++]; +} +static void ngx_http_clear_location(ngx_http_request_t *r) +{ + ++clear_calls; + if (r->headers_out.location != NULL) r->headers_out.location->hash = 0; + r->headers_out.location = NULL; +} +static void ngx_log_error(int level, void *log, int error, const char *format, ...) +{ + va_list args; + (void)level; (void)log; (void)error; + ++error_log_calls; + va_start(args, format); + if (strcmp(format, "%s") == 0) + snprintf(captured_error, sizeof(captured_error), "%s", va_arg(args, const char *)); + va_end(args); +} +static void msc_update_status_code(Transaction *transaction, int status) +{ + (void)transaction; + ++status_calls; + updated_status = status; +} +static ngx_int_t ngx_http_modsecurity_log_handler(ngx_http_request_t *r) +{ + ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); + if (ctx != NULL) ctx->logged = 1; + ++early_log_calls; + return early_log_result; +} +#if MODSECURITY_SANITY_CHECKS +static ngx_int_t ngx_http_modsecurity_store_ctx_header(ngx_http_request_t *r, + ngx_str_t *name, ngx_str_t *value) +{ + (void)r; (void)name; (void)value; + ++sanity_calls; + return fail_sanity ? NGX_ERROR : NGX_OK; +} +#endif +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "line %d: %s\n", __LINE__, #condition); return 1; \ +} } while (0) +""" + +CASES = r""" +int main(int argc, char **argv) +{ + ngx_pool_t pool = {0}; + ngx_connection_t connection = {0}; + Transaction transaction = {0}; + ngx_http_modsecurity_ctx_t ctx = {0}; + ngx_http_modsecurity_conf_t conf = {0}; + ngx_http_request_t request = {0}; + ngx_table_elt_t previous_location = {0}; + ngx_pool_cleanup_t cleanup = {ngx_http_modsecurity_cleanup, &ctx, NULL}; + const char *expected_log = "native intervention message"; + const char *expected_url = "https://example.test/new-location"; + const char *limit_message = "Response body limit is marked to reject the request"; + int result; + CHECK(argc == 2); + request.pool = &pool; request.connection = &connection; + request.ctx = &ctx; request.conf = &conf; + ctx.r = &request; ctx.modsec_transaction = &transaction; + conf.phase4_log_file = &transaction; conf.use_error_log = 1; conf.enable = 1; + ctx.lifetime.conf = &conf; + previous_location.hash = 1; + request.headers_out.location = &previous_location; + ctx.intervention_late = 1; ctx.intervention_failed = 1; + ctx.intervention_redirect_location_installed = 1; + + if (strcmp(argv[1], "no-intervention") == 0) { + native_available = 0; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 0 && native_initialized && native_calls == 1); + CHECK(!ctx.intervention_late && !ctx.intervention_failed); + CHECK(!ctx.intervention_redirect_location_installed); + CHECK(allocation_calls == 0 && error_log_calls == 0); + CHECK(freed_log == 0 && freed_url == 0 && bad_free == 0); + } else if (strcmp(argv[1], "invalid-native-results") == 0) { + const int invalid[] = {-2, -1, 2, 99}; + size_t index; + int mode, committed; + native_has_url = 1; native_status = 302; + for (mode = 0; mode <= 2; ++mode) { + conf.phase4_mode = mode; + for (committed = 0; committed <= 1; ++committed) { + request.header_sent = committed; + for (index = 0; index < sizeof(invalid) / sizeof(invalid[0]); ++index) { + strcpy(native_log, expected_log); strcpy(native_url, expected_url); + native_return = invalid[index]; freed_log = 0; freed_url = 0; + ctx.last_intervention_status = 451; + ctx.last_intervention_log.data = (u_char *)"stale"; + ctx.last_intervention_log.len = 5U; + ctx.intervention_body_limit = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); + CHECK(result == (committed ? NGX_ERROR : 500)); + CHECK(ctx.intervention_failed && !ctx.intervention_late); + CHECK(!ctx.intervention_body_limit && !ctx.intervention_redirect_location_installed); + CHECK(ctx.last_intervention_status == 0); + CHECK(ctx.last_intervention_log.data == NULL && ctx.last_intervention_log.len == 0U); + CHECK(freed_log == 1 && freed_url == 1 && bad_free == 0); + CHECK(allocation_calls == 0 && status_calls == 0 && list_calls == 0); + CHECK(early_log_calls == 0 && captured_error[0] == '\0'); + CHECK(request.headers_out.location == &previous_location && previous_location.hash == 1); + } + } + } + } else if (strcmp(argv[1], "nondisruptive-result") == 0 || + strcmp(argv[1], "no-result-owned-outputs") == 0) { + native_has_url = 1; native_status = 302; + if (strcmp(argv[1], "nondisruptive-result") == 0) native_disruptive = 0; + else native_return = 0; + ctx.last_intervention_status = 451; + ctx.last_intervention_log.data = (u_char *)"stale"; + ctx.last_intervention_log.len = 5U; ctx.intervention_body_limit = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); + CHECK(result == 0 && native_calls == 1); + CHECK(!ctx.intervention_failed && !ctx.intervention_late && !ctx.intervention_body_limit); + CHECK(!ctx.intervention_redirect_location_installed && ctx.last_intervention_status == 0); + CHECK(ctx.last_intervention_log.data == NULL && ctx.last_intervention_log.len == 0U); + CHECK(freed_log == 1 && freed_url == 1 && bad_free == 0); + CHECK(allocation_calls == 0 && status_calls == 0 && list_calls == 0); + CHECK(early_log_calls == 0 && error_log_calls == 0); + CHECK(request.headers_out.location == &previous_location && previous_location.hash == 1); + } else if (strcmp(argv[1], "body-limit-rejection") == 0) { + int mode, committed; + for (mode = 0; mode <= 2; ++mode) { + conf.phase4_mode = mode; + for (committed = 0; committed <= 1; ++committed) { + strcpy(native_log, limit_message); freed_log = 0; + request.header_sent = committed; ctx.phase4_started = 1; ctx.phase4_append_active = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == (committed ? NGX_ERROR : 403)); + CHECK(ctx.intervention_failed && ctx.intervention_body_limit && !ctx.intervention_late); + CHECK(ctx.last_intervention_status == 403); + CHECK(!ctx.intervention_redirect_location_installed && freed_log == 1 && freed_url == 0); + CHECK(status_calls == 0 && list_calls == 0 && early_log_calls == 0); + } + } + } else if (strcmp(argv[1], "body-limit-language-independent") == 0) { + const char *messages[] = {"Andere Diagnose", "", NULL}; + int mode, committed; + size_t i; + ctx.phase4_started = 1; ctx.phase4_append_active = 1; + for (mode = 0; mode <= 2; ++mode) { + conf.phase4_mode = mode; + for (committed = 0; committed <= 1; ++committed) { + request.header_sent = committed; + for (i = 0; i < sizeof(messages) / sizeof(messages[0]); ++i) { + native_has_log = messages[i] != NULL; freed_log = 0; + if (messages[i]) strcpy(native_log, messages[i]); + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == (committed ? NGX_ERROR : 403)); + CHECK(ctx.intervention_body_limit && ctx.intervention_failed); + CHECK(freed_log == native_has_log && status_calls == 0 && list_calls == 0); + } + } + } + } else if (strcmp(argv[1], "body-limit-boundary-controls") == 0) { + ctx.phase4_started = 1; ctx.phase4_processed = 1; ctx.phase4_append_active = 1; + strcpy(native_log, limit_message); + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 403 && !ctx.intervention_body_limit && !ctx.intervention_failed); + CHECK(status_calls == 1 && freed_log == 1); + status_calls = 0; freed_log = 0; ctx.phase4_processed = 0; + strcpy(native_log, "[file \"/rule\"] [line \"1\"] [id \"123\"] [rev \"\"] [msg \"limit\"] [data \"\"] [severity \"2\"] [ver \"\"] [maturity \"0\"] [accuracy \"0\"] [hostname \"host\"] [uri \"/\"] [unique_id \"x\"] [ref \"\"]"); + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 500 && !ctx.intervention_body_limit && ctx.intervention_failed); + CHECK(status_calls == 0 && freed_log == 1); + } else if (strcmp(argv[1], "request-append-retains-limit") == 0) { + strcpy(native_log, "Request body limit is marked to reject the request"); + native_status = 413; + CHECK(ngx_http_modsecurity_capture_request_body_intervention(&request) == NGX_OK); + CHECK(ctx.request_intervention_pending && freed_log == 0); + native_available = 0; ctx.request_intervention_active = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 413 && ctx.native_request_body_limit_rejection); + CHECK(!ctx.request_intervention_pending && freed_log == 1 && status_calls == 1); + CHECK(!ctx.intervention_failed && ctx.last_intervention_status == 413); + } else if (strcmp(argv[1], "request-append-invalid-result") == 0) { + native_return = 2; + CHECK(ngx_http_modsecurity_capture_request_body_intervention(&request) == NGX_ERROR); + CHECK(!ctx.request_intervention_pending && freed_log == 1); + CHECK(status_calls == 0 && early_log_calls == 0); + } else if (strcmp(argv[1], "body-limit-other-phase") == 0 || + strcmp(argv[1], "body-limit-rule-lookalike") == 0) { + if (strcmp(argv[1], "body-limit-other-phase") == 0) strcpy(native_log, limit_message); + else { + ctx.phase4_started = 1; ctx.phase4_processed = 1; + strcpy(native_log, "[client 127.0.0.1] ModSecurity: Access denied with code 403 (phase 4). [id \"12345\"] [msg \"Response body limit is marked to reject the request\"]"); + } + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 403 && !ctx.intervention_failed && !ctx.intervention_body_limit); + CHECK(ctx.last_intervention_status == 403 && status_calls == 1); + CHECK(freed_log == 1 && freed_url == 0 && early_log_calls == 0); + } else if (strcmp(argv[1], "uncorrelated-append-intervention") == 0) { + int mode, committed; + ctx.phase4_started = 1; ctx.phase4_append_active = 1; native_status = 451; native_has_log = 0; + for (mode = 0; mode <= 2; ++mode) { + conf.phase4_mode = mode; + for (committed = 0; committed <= 1; ++committed) { + request.header_sent = committed; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == (committed ? NGX_ERROR : 500)); + CHECK(ctx.intervention_failed && !ctx.intervention_body_limit && !ctx.intervention_late); + CHECK(ctx.last_intervention_status == 0 && ctx.last_intervention_log.data == NULL); + CHECK(status_calls == 0 && early_log_calls == 0 && list_calls == 0); + CHECK(freed_log == 0 && freed_url == 0 && allocation_calls == 0); + } + } + } else if (strcmp(argv[1], "early-log-failure") == 0) { + early_log_result = NGX_ERROR; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); + CHECK(result == 500 && ctx.intervention_failed && !ctx.intervention_late); + CHECK(early_log_calls == 1 && freed_log == 1 && freed_url == 0); + CHECK(list_calls == 0 && !ctx.intervention_redirect_location_installed); + } else if (strcmp(argv[1], "status-precommit") == 0) { + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); + CHECK(result == 403 && native_initialized); + CHECK(ctx.last_intervention_status == 403 && updated_status == 403); + CHECK(status_calls == 1 && early_log_calls == 1 && ctx.logged); + CHECK(!ctx.intervention_failed && !ctx.intervention_late); + CHECK(freed_log == 1 && freed_url == 0 && bad_free == 0); + CHECK(ctx.last_intervention_log.data != (u_char *)native_log); + CHECK(ctx.last_intervention_log.len == strlen(expected_log)); + CHECK(strcmp((const char *)ctx.last_intervention_log.data, expected_log) == 0); + CHECK(strcmp(captured_error, expected_log) == 0); + } else if (strcmp(argv[1], "status-committed") == 0 || + strcmp(argv[1], "status-native-committed") == 0) { + request.header_sent = 1; + conf.phase4_mode = strcmp(argv[1], "status-native-committed") == 0 ? 0 : 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == NGX_ERROR && ctx.intervention_late && !ctx.intervention_failed); + CHECK(ctx.last_intervention_status == 403 && updated_status == 403); + CHECK(!ctx.intervention_redirect_location_installed); + CHECK(status_calls == 1 && early_log_calls == 0); + CHECK(freed_log == 1 && freed_url == 0); + CHECK(strcmp((const char *)ctx.last_intervention_log.data, expected_log) == 0); + } else if (strcmp(argv[1], "late-phase4-policy-deferral") == 0) { + int mode; + request.header_sent = 1; + ctx.phase4_started = 1; ctx.phase4_processed = 1; + for (mode = 0; mode <= 2; ++mode) { + conf.phase4_mode = mode; + strcpy(native_log, expected_log); freed_log = 0; + status_calls = 0; early_log_calls = 0; updated_status = 0; + ctx.logged = 0; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); + CHECK(result == NGX_ERROR && ctx.intervention_late && !ctx.intervention_failed); + CHECK(ctx.last_intervention_status == 403 && !ctx.intervention_body_limit); + CHECK(status_calls == (mode == 0) && early_log_calls == (mode == 0)); + CHECK(updated_status == (mode == 0 ? 403 : 0)); + CHECK(ctx.logged == (mode == 0) && list_calls == 0); + CHECK(freed_log == 1 && freed_url == 0 && !ctx.intervention_redirect_location_installed); + } + } else if (strcmp(argv[1], "status-log-only") == 0) { + native_status = 200; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); + CHECK(result == 0 && ctx.last_intervention_status == 200); + CHECK(!ctx.intervention_failed && !ctx.intervention_late); + CHECK(status_calls == 0 && early_log_calls == 0 && !ctx.logged); + CHECK(freed_log == 1 && freed_url == 0); + } else if (strcmp(argv[1], "redirect-owned") == 0) { + native_status = 302; native_has_url = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); + CHECK(result == 302 && !ctx.intervention_failed && !ctx.intervention_late); + CHECK(ctx.intervention_redirect_location_installed); + CHECK(freed_log == 1 && freed_url == 1 && bad_free == 0); + CHECK(request.headers_out.location != &previous_location && previous_location.hash == 0); + CHECK(request.headers_out.location->hash == 1 && clear_calls == 1); + CHECK(request.headers_out.location->key.len == 8U); + CHECK(memcmp(request.headers_out.location->key.data, "Location", 8U) == 0); + CHECK(request.headers_out.location->value.data != (u_char *)native_url); + CHECK(request.headers_out.location->value.len == strlen(expected_url)); + CHECK(memcmp(request.headers_out.location->value.data, expected_url, strlen(expected_url)) == 0); + CHECK(strcmp((const char *)ctx.last_intervention_log.data, expected_log) == 0); + CHECK(status_calls == 0 && early_log_calls == 0); + CHECK(sanity_calls == MODSECURITY_SANITY_CHECKS); + } else if (strcmp(argv[1], "redirect-committed") == 0) { + native_status = 302; native_has_url = 1; request.header_sent = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == NGX_ERROR && ctx.intervention_late && !ctx.intervention_failed); + CHECK(!ctx.intervention_redirect_location_installed); + CHECK(request.headers_out.location == &previous_location && previous_location.hash == 1); + CHECK(list_calls == 0 && clear_calls == 0 && allocation_calls == 1); + CHECK(freed_log == 1 && freed_url == 1 && sanity_calls == 0); + } else if (strcmp(argv[1], "redirect-crlf") == 0) { + native_status = 302; native_has_url = 1; + strcpy(native_url, "https://example.test/\r\nInjected: yes"); + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 400 && ctx.intervention_failed && !ctx.intervention_late); + CHECK(!ctx.intervention_redirect_location_installed); + CHECK(request.headers_out.location == &previous_location && list_calls == 0); + CHECK(freed_log == 1 && freed_url == 1 && bad_free == 0); + CHECK(allocation_calls == 1 && error_log_calls == 2); + } else if (strcmp(argv[1], "redirect-list-failure") == 0) { + native_status = 302; native_has_url = 1; fail_list = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 500 && ctx.intervention_failed && !ctx.intervention_late); + CHECK(!ctx.intervention_redirect_location_installed); + CHECK(request.headers_out.location == &previous_location && clear_calls == 0); + CHECK(freed_log == 1 && freed_url == 1 && list_calls == 1); + } else if (strcmp(argv[1], "redirect-allocation-failure") == 0) { + native_status = 302; native_has_url = 1; fail_allocation_at = 2; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 500 && ctx.intervention_failed && !ctx.intervention_late); + CHECK(!ctx.intervention_redirect_location_installed && list_calls == 0); + CHECK(freed_log == 1 && freed_url == 1 && allocation_calls == 2); + CHECK(strcmp((const char *)ctx.last_intervention_log.data, expected_log) == 0); + } else if (strcmp(argv[1], "redirect-sanity-failure") == 0) { + native_status = 302; native_has_url = 1; fail_sanity = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == (MODSECURITY_SANITY_CHECKS ? 500 : 302)); + CHECK(ctx.intervention_failed == MODSECURITY_SANITY_CHECKS); + CHECK(ctx.intervention_redirect_location_installed == !MODSECURITY_SANITY_CHECKS); + CHECK(!ctx.intervention_late && sanity_calls == MODSECURITY_SANITY_CHECKS); + CHECK(freed_log == 1 && freed_url == 1 && bad_free == 0); + CHECK(request.headers_out.location->value.len == strlen(expected_url)); + CHECK(memcmp(request.headers_out.location->value.data, expected_url, strlen(expected_url)) == 0); + } else if (strcmp(argv[1], "log-allocation-precommit") == 0 || + strcmp(argv[1], "log-allocation-committed") == 0) { + request.header_sent = strcmp(argv[1], "log-allocation-committed") == 0; + native_has_url = 1; native_status = 302; fail_allocation_at = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == (request.header_sent ? NGX_ERROR : 302)); + CHECK(!ctx.intervention_failed && ctx.intervention_late == request.header_sent); + CHECK(ctx.intervention_redirect_location_installed == !request.header_sent); + CHECK(error_log_calls == 2); + CHECK(ctx.last_intervention_log.len == 0U && ctx.last_intervention_log.data == NULL); + CHECK(freed_log == 1 && freed_url == 1 && list_calls == !request.header_sent); + } else if (strcmp(argv[1], "disabled-log-retention") == 0) { + conf.phase4_log_file = NULL; conf.use_error_log = 0; + ctx.last_intervention_log.data = (u_char *)"old message"; + ctx.last_intervention_log.len = 11; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 403 && !ctx.intervention_failed); + CHECK(ctx.last_intervention_log.data == NULL && ctx.last_intervention_log.len == 0U); + CHECK(allocation_calls == 0 && error_log_calls == 0); + CHECK(freed_log == 1 && freed_url == 0); + } else if (strcmp(argv[1], "missing-log") == 0) { + native_has_log = 0; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 403 && allocation_calls == 0); + CHECK(ctx.last_intervention_log.data == NULL && ctx.last_intervention_log.len == 0U); + CHECK(strcmp(captured_error, "(no log message was specified)") == 0); + CHECK(freed_log == 0 && freed_url == 0); + } else if (strcmp(argv[1], "missing-config-precommit") == 0 || + strcmp(argv[1], "missing-config-committed") == 0) { + request.conf = NULL; + request.header_sent = strcmp(argv[1], "missing-config-committed") == 0; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == (request.header_sent ? NGX_ERROR : 500)); + /* No policy owner is selected without configuration: its state stays untouched. */ + CHECK(ctx.intervention_failed && ctx.intervention_late); + CHECK(native_calls == 0 && freed_log == 0 && freed_url == 0); + } else if (strcmp(argv[1], "redirect-context-recovery") == 0) { + pool.cleanup = &cleanup; request.ctx = NULL; ctx.intervention_failed = 0; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 403 && native_calls == 1 && !ctx.intervention_failed); + CHECK(freed_log == 1 && ctx.last_intervention_status == 403); + } else if (strcmp(argv[1], "foreign-context-rejected") == 0) { + ngx_http_request_t other_request = {0}; + ctx.r = &other_request; pool.cleanup = &cleanup; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 500 && native_calls == 0); + CHECK(freed_log == 0 && freed_url == 0); + } else { + return 2; + } + CHECK(bad_free == 0 && freed_log <= 1 && freed_url <= 1); + return 0; +} +""" + + +def unit_program(module: str) -> str: + helpers = "" + for name in ("ngx_http_modsecurity_same_policy", "ngx_http_modsecurity_is_block_error_page"): + helpers += "\nstatic ngx_int_t\n" + function_definition(module, name) + helpers += "\nstatic ngx_http_modsecurity_ctx_t *\n" + function_definition( + module, "ngx_http_modsecurity_get_module_ctx" + ) + helpers += "\nstatic ngx_int_t\n" + function_definition( + module, "ngx_http_modsecurity_log_intervention" + ) + helpers += "\nstatic int\n" + function_definition( + module, "ngx_http_modsecurity_process_intervention_redirect" + ) + for helper in ( + "ngx_http_modsecurity_take_pending_request_intervention", + "ngx_http_modsecurity_is_response_body_limit_rejection", + "ngx_http_modsecurity_intervention_failure_status", + "ngx_http_modsecurity_collect_native_intervention", + "ngx_http_modsecurity_defer_late_phase4_intervention", + ): + if re.search(rf"\b{helper}\s*\(", module): + helpers += "\nstatic ngx_int_t\n" + function_definition(module, helper) + helpers += "\nngx_int_t\n" + function_definition(module, "ngx_http_modsecurity_capture_request_body_intervention") + reset = "ngx_http_modsecurity_reset_intervention" + if re.search(rf"\b{reset}\s*\(", module): + helpers += "\nstatic void\n" + function_definition(module, reset) + helpers += "\nint\n" + function_definition( + module, "ngx_http_modsecurity_process_intervention" + ) + metadata_path = (ROOT / "src" / "ngx_http_modsecurity_log_metadata.h").as_posix() + metadata = f'\n#include "{metadata_path}"\n' + return PREAMBLE + metadata + helpers + CASES + + +class InterventionRuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler or shutil.which(compiler[0]) is None: + raise RuntimeError("set CC to a GCC- or Clang-compatible C17 compiler") + temporary = tempfile.TemporaryDirectory(prefix="nginx-intervention-") + cls.addClassCleanup(temporary.cleanup) + directory = Path(temporary.name) + source = directory / "intervention.c" + source.write_text(unit_program(MODULE.read_text(encoding="utf-8")), encoding="utf-8") + cls.binaries = {} + for sanity in (0, 1): + binary = directory / (f"intervention-{sanity}.exe" if os.name == "nt" else f"intervention-{sanity}") + build = subprocess.run( + [*compiler, "-std=c17", "-Wall", "-Wextra", "-Werror", + f"-DMODSECURITY_SANITY_CHECKS={sanity}", str(source), "-o", str(binary)], + capture_output=True, text=True, timeout=60, check=False, + ) + if build.returncode: + raise AssertionError(f"intervention helper build (sanity={sanity}) failed:\n{build.stderr}") + cls.binaries[sanity] = binary + + def run_case(self, name: str) -> None: + for sanity, binary in self.binaries.items(): + with self.subTest(sanity=sanity): + result = subprocess.run( + [str(binary), name], capture_output=True, text=True, + timeout=10, check=False, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +def case_test(name: str): + def test(self: InterventionRuntimeTests) -> None: + self.run_case(name) + return test + + +for case in ( + "no-intervention", "status-precommit", "status-committed", "status-native-committed", + "status-log-only", + "late-phase4-policy-deferral", + "invalid-native-results", "nondisruptive-result", "no-result-owned-outputs", + "body-limit-rejection", "body-limit-other-phase", "body-limit-rule-lookalike", + "early-log-failure", "body-limit-language-independent", "body-limit-boundary-controls", + "request-append-retains-limit", "request-append-invalid-result", + "uncorrelated-append-intervention", + "redirect-owned", "redirect-committed", "redirect-crlf", "redirect-list-failure", + "redirect-allocation-failure", "redirect-sanity-failure", + "log-allocation-precommit", "log-allocation-committed", "disabled-log-retention", + "missing-log", "missing-config-precommit", "missing-config-committed", + "redirect-context-recovery", "foreign-context-rejected", +): + setattr(InterventionRuntimeTests, "test_" + case.replace("-", "_"), case_test(case)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_lifecycle_runtime.py b/tests/test_lifecycle_runtime.py new file mode 100644 index 00000000..74c4d474 --- /dev/null +++ b/tests/test_lifecycle_runtime.py @@ -0,0 +1,993 @@ +"""Exercise access routing and native audit completion through production code. + +The complete access and log handlers run with small NGINX/SDK doubles. These +tests prove their control flow, not live error_page or client transport behavior. +""" + +from __future__ import annotations + +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import tempfile +import unittest + +from test_phase4_runtime import function_definition + +ROOT = Path(__file__).resolve().parents[1] + +PREAMBLE = r""" +#include +#include +#include +#include +#include +typedef unsigned char u_char; +typedef long ngx_int_t; +typedef unsigned long ngx_uint_t; +typedef long off_t; +typedef long ssize_t; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct { int unused; } ngx_pool_t; +typedef struct { int unused; } Transaction; +typedef struct ngx_http_request_s ngx_http_request_t; +typedef struct ngx_list_part_s { void *elts; ngx_uint_t nelts; struct ngx_list_part_s *next; } ngx_list_part_t; +typedef struct { ngx_list_part_t part; } ngx_list_t; +typedef struct { ngx_str_t key, value; } ngx_table_elt_t; +typedef struct { u_char *pos, *last; int last_buf; } ngx_buf_t; +typedef struct ngx_chain_s { ngx_buf_t *buf; struct ngx_chain_s *next; } ngx_chain_t; +typedef struct { int fd; ngx_str_t name; } ngx_file_t; +typedef struct { off_t size; int regular; } ngx_file_info_t; +typedef struct { ngx_file_t file; off_t offset; } ngx_temp_file_t; +typedef struct { ngx_temp_file_t *temp_file; ngx_chain_t *bufs; } ngx_http_request_body_t; +typedef struct { void *sockaddr, *local_sockaddr, *log; ngx_str_t addr_text; int error; } ngx_connection_t; +typedef struct { + ngx_http_request_t *r; + struct { ngx_pool_t *pool; const void *conf; } lifetime; + Transaction *modsec_transaction; + int waiting_more_body, body_requested, request_body_processed; + int intervention_triggered, intervention_failed, request_intervention_active; + int request_intervention_pending, logged, native_logging_failed; + u_char *request_file_scratch; +} ngx_http_modsecurity_ctx_t; +typedef struct { int enable; } ngx_http_modsecurity_conf_t; +struct ngx_http_request_s { + ngx_pool_t *pool; ngx_connection_t *connection; ngx_http_request_t *main; + int count, method, http_version, header_sent, error_page; + ngx_str_t http_protocol, unparsed_uri, method_name; + struct { ngx_str_t server; ngx_list_t headers; } headers_in; + ngx_http_request_body_t *request_body; + int request_body_in_single_buf, request_body_in_persistent_file; + int request_body_in_file_only, request_body_in_clean_file; + void (*write_event_handler)(ngx_http_request_t *); + ngx_http_modsecurity_ctx_t *ctx, *recovered; + ngx_http_modsecurity_conf_t *conf; +}; +#define NGX_OK 0 +#define NGX_ERROR (-1) +#define NGX_AGAIN (-2) +#define NGX_DONE (-4) +#define NGX_DECLINED (-5) +#define NGX_HTTP_INTERNAL_SERVER_ERROR 500 +#define NGX_HTTP_SPECIAL_RESPONSE 300 +#define NGX_HTTP_VERSION_9 9 +#define NGX_HTTP_VERSION_10 1000 +#define NGX_HTTP_VERSION_11 1001 +#define NGX_SOCKADDR_STRLEN 64 +#define NGX_LOG_ERR 3 +#define NGX_INVALID_FILE (-1) +#define NGX_FILE_ERROR (-1) +#define ngx_is_file(info) ((info)->regular) +#define ngx_file_size(info) ((info)->size) +#define ngx_buf_special(buf) ((buf)->pos == NULL && (buf)->last == NULL) +#define dd(...) ((void)0) +#define ngx_http_get_module_ctx(r, module) ((r)->ctx) +#define ngx_http_get_module_loc_conf(r, module) ((r)->conf) +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "line %d: %s\n", __LINE__, #condition); return 1; \ +} } while (0) +static Transaction transaction; +static ngx_http_modsecurity_ctx_t created; +static int created_calls, recovered_calls, error_calls, prepare_calls, prepare_failure, recover_allowed; +static int native_results[4] = {1, 1, 1, 1}; +static int native_calls[4], intervention_results[5], intervention_calls[5]; +static int active_phase, append_result = 1, append_calls, capture_calls, pending_limit; +static int log_result = 1, log_calls, log_reenter, nested_log_result; +static int stat_calls, read_calls, short_read, fail_read, bad_stat, regular_file = 1, changed_file, fail_scratch; +static size_t max_read; +static off_t actual_file_size = 3; +static u_char file_bytes[65537], scratch[32768]; +static ngx_http_request_t *active_request; +static ngx_int_t ngx_http_modsecurity_log_handler(ngx_http_request_t *r); +static ngx_int_t ngx_http_modsecurity_log_context(ngx_http_modsecurity_ctx_t *ctx); +static void ngx_http_core_run_phases(ngx_http_request_t *r) { (void)r; } +static ngx_int_t ngx_http_modsecurity_prepare_location(ngx_http_request_t *r, const ngx_http_modsecurity_conf_t *conf) +{ + (void)r; (void)conf; ++prepare_calls; + return prepare_failure ? NGX_ERROR : NGX_OK; +} +static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r) +{ + ++recovered_calls; + if (r->ctx != NULL && r->ctx->r == r) return r->ctx; + return recover_allowed && r->recovered != NULL && r->recovered->r == r ? r->recovered : NULL; +} +static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_create_ctx(ngx_http_request_t *r) +{ + ++created_calls; memset(&created, 0, sizeof(created)); + created.r = r; created.modsec_transaction = &transaction; + created.lifetime.pool = r->pool; created.lifetime.conf = r->conf; + r->ctx = &created; + return &created; +} +static char *ngx_str_to_char(ngx_str_t value, ngx_pool_t *pool) +{ + (void)pool; + return value.len == 0 ? NULL : (char *)value.data; +} +static int ngx_inet_get_port(void *address) { (void)address; return 80; } +static ngx_int_t ngx_connection_local_sockaddr(ngx_connection_t *connection, ngx_str_t *out, int port) +{ + (void)connection; (void)port; out->data = (u_char *)"127.0.0.1"; out->len = 9; + return NGX_OK; +} +static ngx_pool_t *ngx_http_modsecurity_pcre_malloc_init(ngx_pool_t *pool) { return pool; } +static void ngx_http_modsecurity_pcre_malloc_done(ngx_pool_t *pool) { (void)pool; } +static void ngx_log_error(int level, void *log, int error, const char *format, ...) +{ + (void)level; (void)log; (void)error; (void)format; ++error_calls; +} +static int phase_result(int phase) { active_phase = phase; ++native_calls[phase]; return native_results[phase]; } +static int msc_process_connection(Transaction *t, const char *client, int cp, const char *server, int sp) +{ + (void)t; (void)client; (void)cp; (void)server; (void)sp; return phase_result(0); +} +static int msc_process_uri(Transaction *t, const char *uri, const char *method, const char *version) +{ + (void)t; (void)uri; (void)method; (void)version; return phase_result(1); +} +static int msc_add_n_request_header(Transaction *t, const u_char *name, size_t nl, const u_char *value, size_t vl) +{ + (void)t; (void)name; (void)nl; (void)value; (void)vl; return 1; +} +static int msc_process_request_headers(Transaction *t) { (void)t; return phase_result(2); } +static int msc_process_request_body(Transaction *t) { (void)t; return phase_result(3); } +static int msc_append_request_body(Transaction *t, const u_char *data, size_t size) +{ + (void)t; (void)data; (void)size; active_phase = 4; ++append_calls; return append_result; +} +static ngx_int_t ngx_http_modsecurity_capture_request_body_intervention(ngx_http_request_t *r) +{ + ++capture_calls; + if (intervention_results[4] < 0) return NGX_ERROR; + if (pending_limit) r->ctx->request_intervention_pending = 1; + return NGX_OK; +} +static int ngx_http_modsecurity_process_intervention(Transaction *t, ngx_http_request_t *r, int early) +{ + (void)t; (void)early; ++intervention_calls[active_phase]; + if (!r->ctx->request_intervention_active) abort(); + if (active_phase == 3 && r->ctx->request_intervention_pending) { + r->ctx->request_intervention_pending = 0; + return 413; + } + return intervention_results[active_phase]; +} +static ngx_int_t ngx_http_read_client_request_body(ngx_http_request_t *r, void (*callback)(ngx_http_request_t *)) +{ + (void)r; (void)callback; return NGX_OK; +} +static void *ngx_pnalloc(ngx_pool_t *pool, size_t bytes) +{ + (void)pool; return !fail_scratch && bytes == sizeof(scratch) ? scratch : NULL; +} +static int ngx_fd_info(int fd, ngx_file_info_t *info) +{ + ++stat_calls; + if (fd != 7 || bad_stat) return NGX_FILE_ERROR; + info->size = actual_file_size - (changed_file && stat_calls > 1 ? 1 : 0); + info->regular = regular_file; + return 0; +} +static ssize_t ngx_read_file(ngx_file_t *file, u_char *out, size_t amount, off_t offset) +{ + (void)file; ++read_calls; + if (amount > max_read) max_read = amount; + if (fail_read || offset < 0 || (uintmax_t)offset + amount > sizeof(file_bytes)) return -1; + if (short_read) return (ssize_t)amount - 1; + memcpy(out, file_bytes + offset, amount); + return (ssize_t)amount; +} +static int msc_process_logging(Transaction *t) +{ + (void)t; ++log_calls; + if (log_reenter && log_calls == 1) nested_log_result = ngx_http_modsecurity_log_handler(active_request); + return log_result; +} +static ngx_int_t ngx_http_modsecurity_log_request_contexts(ngx_http_request_t *r) +{ + if (r == NULL || r->pool == NULL) return NGX_ERROR; + return r->ctx != NULL ? ngx_http_modsecurity_log_context(r->ctx) : NGX_OK; +} +""" + +CASES = r""" +static int run_access(const char *scenario, int parameter) +{ + ngx_pool_t pool = {0}; + ngx_connection_t connection = {0}; + ngx_http_modsecurity_conf_t conf = {1}; + ngx_http_modsecurity_ctx_t previous = {0}; + ngx_http_request_t other_request = {0}; + ngx_http_request_body_t body = {0}; + ngx_http_request_t request = {0}; + u_char bytes[] = "abc"; + ngx_buf_t buffer = {bytes, bytes + 3, 0}; + ngx_chain_t chain = {&buffer, NULL}; + ngx_temp_file_t file = {{7, {8, (u_char *)"body.tmp"}}, 3}; + int phase, before_calls, after_calls, allowed = 0; + ngx_int_t result; + request.pool = &pool; request.connection = &connection; + request.main = &request; request.conf = &conf; request.request_body = &body; + request.http_version = NGX_HTTP_VERSION_11; + request.unparsed_uri.data = (u_char *)"/protected"; request.unparsed_uri.len = 10; + request.method_name.data = (u_char *)"POST"; request.method_name.len = 4; + connection.addr_text.data = (u_char *)"127.0.0.1"; connection.addr_text.len = 9; + request.error_page = 1; + if (strcmp(scenario, "body-callback-no-context") == 0) { + ngx_http_modsecurity_request_read(&request); + CHECK(created_calls == 0 && native_calls[3] == 0); + return 0; + } + if (strcmp(scenario, "fresh-error-page") == 0 || strcmp(scenario, "foreign-direct") == 0) { + previous.r = strcmp(scenario, "foreign-direct") == 0 ? &other_request : &request; + previous.modsec_transaction = &transaction; + previous.request_body_processed = 1; previous.intervention_triggered = 1; + if (strcmp(scenario, "foreign-direct") == 0) request.ctx = &previous; + else request.recovered = &previous; + intervention_results[3] = parameter; + } else if (strcmp(scenario, "same-policy") == 0 || strcmp(scenario, "block-page") == 0) { + previous.r = &request; previous.modsec_transaction = &transaction; + previous.request_body_processed = 1; + previous.intervention_triggered = strcmp(scenario, "block-page") == 0; + request.recovered = &previous; recover_allowed = 1; + } else if (strcmp(scenario, "p1-status") == 0) intervention_results[2] = parameter; + else if (strcmp(scenario, "p2-status") == 0) intervention_results[3] = parameter; + else if (strcmp(scenario, "intervention-error") == 0) { + intervention_results[parameter] = -1; + if (parameter == 4) body.bufs = &chain; + } else if (strcmp(scenario, "native-error") == 0) native_results[parameter / 10] = parameter % 10 - 1; + else if (strcmp(scenario, "append") == 0) { + body.bufs = &chain; append_result = parameter; allowed = parameter == 0 || parameter == 1; + } else if (strcmp(scenario, "file") == 0 || strcmp(scenario, "file-large") == 0 || strcmp(scenario, "file-reject") == 0) { + body.temp_file = &file; append_result = parameter; allowed = parameter == 0 || parameter == 1; + if (strcmp(scenario, "file-large") == 0) file.offset = actual_file_size = sizeof(file_bytes); + if (strcmp(scenario, "file-reject") == 0) pending_limit = 1; + } else if (strcmp(scenario, "file-error") == 0) { + body.temp_file = &file; + if (parameter == 0) file.file.fd = NGX_INVALID_FILE; + if (parameter == 1) file.offset = -1; + if (parameter == 2) bad_stat = 1; + if (parameter == 3) regular_file = 0; + if (parameter == 4) actual_file_size = 2; + if (parameter == 5) short_read = 1; + if (parameter == 6) fail_read = 1; + if (parameter == 7) changed_file = 1; + if (parameter == 8) fail_scratch = 1; + } else if (strcmp(scenario, "disabled") == 0) conf.enable = 0; + else if (strcmp(scenario, "prepare-error") == 0) prepare_failure = 1; + else return 2; + result = ngx_http_modsecurity_access_handler(&request); + CHECK(prepare_calls == 1); + CHECK(created.request_intervention_active == 0); + if (strcmp(scenario, "fresh-error-page") == 0 || strcmp(scenario, "foreign-direct") == 0) { + CHECK(created_calls == 1 && recovered_calls == 1); + CHECK(native_calls[2] == 1 && native_calls[3] == 1); + CHECK(previous.request_body_processed && previous.intervention_triggered); + CHECK(result == (parameter > 0 ? parameter : NGX_DECLINED)); + } else if (strcmp(scenario, "same-policy") == 0 || strcmp(scenario, "block-page") == 0) { + CHECK(result == NGX_DECLINED && created_calls == 0); + CHECK(native_calls[0] == 0 && native_calls[2] == 0 && native_calls[3] == 0); + } else if (strcmp(scenario, "disabled") == 0 || strcmp(scenario, "prepare-error") == 0) { + CHECK(result == (prepare_failure ? 500 : NGX_DECLINED) && created_calls == 0); + } else if (strcmp(scenario, "p1-status") == 0) { + CHECK(result == parameter && created.intervention_triggered && native_calls[3] == 0); + } else if (strcmp(scenario, "p2-status") == 0 || strcmp(scenario, "file-reject") == 0) { + CHECK(result == (pending_limit ? 413 : parameter) && created.intervention_triggered); + CHECK(native_calls[3] == 1 && created.request_body_processed); + CHECK(!created.request_intervention_pending); + } else if (allowed) { + CHECK(result == NGX_DECLINED && native_calls[3] == 1 && created.request_body_processed); + CHECK(!created.intervention_failed); + if (body.temp_file != NULL) CHECK(stat_calls == 2 && read_calls == append_calls && capture_calls == append_calls); + if (strcmp(scenario, "file-large") == 0) CHECK(append_calls == 3 && max_read == 32768U); + } else { + CHECK(result == 500 && created.intervention_failed && created.intervention_triggered); + if (strcmp(scenario, "native-error") == 0) CHECK(intervention_calls[parameter / 10] == 0); + if (body.temp_file != NULL || strcmp(scenario, "append") == 0) CHECK(native_calls[3] == 0); + before_calls = append_calls + log_calls; + for (phase = 0; phase < 4; ++phase) before_calls += native_calls[phase]; + CHECK(ngx_http_modsecurity_access_handler(&request) == 500); + after_calls = append_calls + log_calls; + for (phase = 0; phase < 4; ++phase) after_calls += native_calls[phase]; + CHECK(before_calls == after_calls); + request.header_sent = 1; + CHECK(ngx_http_modsecurity_access_handler(&request) == NGX_ERROR && connection.error); + } + return 0; +} +static int run_logging(const char *scenario, int parameter) +{ + ngx_pool_t pool = {0}; + ngx_connection_t connection = {0}; + ngx_http_modsecurity_ctx_t ctx = {0}; + ngx_http_request_t request = {0}; + ngx_int_t result; + request.pool = &pool; request.connection = &connection; request.ctx = &ctx; + ctx.r = &request; ctx.modsec_transaction = &transaction; ctx.lifetime.pool = &pool; + active_request = &request; log_result = parameter; + if (strcmp(scenario, "log-missing") == 0) ctx.modsec_transaction = NULL; + if (strcmp(scenario, "log-no-context") == 0) request.ctx = NULL; + if (strcmp(scenario, "log-reenter") == 0) log_reenter = 1; + result = ngx_http_modsecurity_log_handler(&request); + if (strcmp(scenario, "log-no-context") == 0) { CHECK(result == NGX_OK && log_calls == 0); return 0; } + CHECK(ctx.logged && result == (parameter == 1 && ctx.modsec_transaction != NULL ? NGX_OK : NGX_ERROR)); + CHECK(ctx.native_logging_failed == (result != NGX_OK)); + CHECK(log_calls == (ctx.modsec_transaction != NULL ? 1 : 0)); + if (log_reenter) CHECK(nested_log_result == NGX_ERROR); + CHECK(ngx_http_modsecurity_log_handler(&request) == result); + CHECK(log_calls == (ctx.modsec_transaction != NULL ? 1 : 0)); + CHECK(!connection.error && !request.header_sent); + return 0; +} +int main(int argc, char **argv) +{ + CHECK(argc == 3); + return strncmp(argv[1], "log-", 4) == 0 ? run_logging(argv[1], atoi(argv[2])) : run_access(argv[1], atoi(argv[2])); +} +""" + + +def unit_program(access: str, logging: str) -> str: + helpers = "" + for name, result in ( + ("ngx_http_modsecurity_request_failure", "static ngx_int_t"), + ("ngx_http_modsecurity_request_intervention", "static int"), + ("ngx_http_modsecurity_append_request_chunk", "static ngx_int_t"), + ("ngx_http_modsecurity_append_request_body", "static ngx_int_t"), + ("ngx_http_modsecurity_inspect_request_body_file", "static ngx_int_t"), + ("ngx_http_modsecurity_request_read", "void"), + ("ngx_http_modsecurity_access_handler", "ngx_int_t"), + ): + helpers += "\n" + result + "\n" + function_definition(access, name) + for name in ("ngx_http_modsecurity_log_context", "ngx_http_modsecurity_log_handler"): + helpers += "\nstatic ngx_int_t\n" + function_definition(logging, name) + return PREAMBLE + helpers + CASES + + +OWNERSHIP_PREAMBLE = r""" +#include +#include +#include +#include +typedef unsigned char u_char; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct { int unused; } Transaction; +typedef long ngx_int_t; +typedef struct { int unused; } ngx_array_t; +typedef struct { int unused; } ngx_http_modsecurity_header_t; +typedef struct ngx_http_request_s ngx_http_request_t; +typedef struct { + ngx_http_request_t *r; + Transaction *modsec_transaction; + struct { void *pool; const void *conf; } lifetime; + struct { char *log, *url; } pending_request_intervention; + int request_intervention_pending, logged; + ngx_array_t *sanity_headers_out; +} ngx_http_modsecurity_ctx_t; +typedef struct { void (*handler)(void *); void *data; } ngx_pool_cleanup_t; +typedef struct { + unsigned char storage[1024]; size_t used; ngx_pool_cleanup_t cleanup; +} ngx_pool_t; +typedef struct { void *rules_set, *transaction_id; } ngx_http_modsecurity_conf_t; +typedef struct { void *modsec; } ngx_http_modsecurity_main_conf_t; +typedef struct { void *log; } ngx_connection_t; +struct ngx_http_request_s { + ngx_pool_t *pool; ngx_connection_t *connection; + ngx_http_modsecurity_conf_t *conf; ngx_http_modsecurity_main_conf_t *main_conf; + ngx_http_modsecurity_ctx_t *ctx; +}; +#define NGX_OK 0 +#define NGX_CONF_ERROR ((void *)-1) +#define dd(...) ((void)0) +#define ngx_http_get_module_main_conf(r, module) ((r)->main_conf) +#define ngx_http_get_module_loc_conf(r, module) ((r)->conf) +#define ngx_http_set_ctx(r, value, module) ((r)->ctx = (value)) +#define ngx_memcpy(dst, src, size) memcpy((dst), (src), (size)) +#define ngx_memzero(dst, size) memset((dst), 0, (size)) +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "line %d: %s\n", __LINE__, #condition); return 1; \ +} } while (0) +static Transaction transaction; +static int allocation_calls, fail_allocation_at, complex_failure, native_failure; +static int registration_failure, registration_calls, native_calls, cleanup_calls; +static int sanity_failure, sanity_allocations, sanity_destroys; +static int custom_id_copied, custom_id_terminated, audit_attempts; +static char custom_id[4] = {'i', 'd', '7', '!'}; +static void ngx_http_modsecurity_cleanup(void *data); +static ngx_int_t ngx_http_modsecurity_log_context(ngx_http_modsecurity_ctx_t *ctx) +{ + if (!ctx->logged) { ctx->logged = 1; ++audit_attempts; } + return NGX_OK; +} +static void *ngx_pnalloc(ngx_pool_t *pool, size_t bytes) +{ + void *result; + ++allocation_calls; + if (allocation_calls == fail_allocation_at || bytes > sizeof(pool->storage) - pool->used) + return NULL; + result = pool->storage + pool->used; + pool->used += bytes; + return result; +} +static void *ngx_pcalloc(ngx_pool_t *pool, size_t bytes) +{ + void *result = ngx_pnalloc(pool, bytes); + if (result != NULL) memset(result, 0, bytes); + return result; +} +static int ngx_http_complex_value(ngx_http_request_t *r, void *expression, ngx_str_t *out) +{ + (void)r; (void)expression; + out->data = (u_char *)custom_id; out->len = 3; + return complex_failure ? -1 : NGX_OK; +} +static Transaction *msc_new_transaction(void *modsec, void *rules, void *log) +{ + (void)modsec; (void)rules; (void)log; + ++native_calls; + return native_failure ? NULL : &transaction; +} +static Transaction *msc_new_transaction_with_id(void *modsec, void *rules, char *id, void *log) +{ + custom_id_copied = id != custom_id && memcmp(id, custom_id, 3) == 0; + custom_id_terminated = id[3] == '\0'; + return msc_new_transaction(modsec, rules, log); +} +static void msc_transaction_cleanup(Transaction *value) +{ + if (value != &transaction) abort(); + ++cleanup_calls; +} +static ngx_pool_cleanup_t *ngx_pool_cleanup_add(ngx_pool_t *pool, size_t size) +{ + (void)size; + ++registration_calls; + return registration_failure ? NULL : &pool->cleanup; +} +#if MODSECURITY_SANITY_CHECKS +static ngx_array_t sanity_array; +static ngx_array_t *ngx_array_create(ngx_pool_t *pool, unsigned count, size_t bytes) +{ + (void)pool; (void)count; (void)bytes; + ++sanity_allocations; + return sanity_failure ? NULL : &sanity_array; +} +static void ngx_array_destroy(ngx_array_t *array) +{ + if (array != &sanity_array) abort(); + ++sanity_destroys; +} +#endif +""" + +OWNERSHIP_CASES = r""" +int main(int argc, char **argv) +{ + ngx_pool_t pool = {0}; + ngx_connection_t connection = {0}; + ngx_http_modsecurity_conf_t conf = {0}; + ngx_http_modsecurity_main_conf_t main_conf = {0}; + ngx_http_request_t request = {0}; + ngx_http_modsecurity_ctx_t *ctx; + int expected_failure = 1; + CHECK(argc == 2); + request.pool = &pool; request.connection = &connection; + request.conf = &conf; request.main_conf = &main_conf; + conf.rules_set = &transaction; main_conf.modsec = &transaction; + if (strcmp(argv[1], "context-allocation") == 0) fail_allocation_at = 1; + else if (strcmp(argv[1], "complex-value") == 0) { + conf.transaction_id = &conf; complex_failure = 1; + } else if (strcmp(argv[1], "custom-id-allocation") == 0) { + conf.transaction_id = &conf; fail_allocation_at = 2; + } else if (strcmp(argv[1], "native-allocation") == 0) native_failure = 1; + else if (strcmp(argv[1], "cleanup-registration") == 0) registration_failure = 1; + else if (strcmp(argv[1], "sanity-allocation") == 0) { + sanity_failure = 1; expected_failure = MODSECURITY_SANITY_CHECKS; + } else if (strcmp(argv[1], "custom-id-owned") == 0) { + conf.transaction_id = &conf; expected_failure = 0; + } else if (strcmp(argv[1], "cleanup-once") == 0) expected_failure = 0; + else return 2; + + ctx = ngx_http_modsecurity_create_ctx(&request); + if (expected_failure) { + CHECK(ctx == NULL && request.ctx == NULL); + CHECK(pool.cleanup.handler == NULL && pool.cleanup.data == NULL); + if (registration_failure || sanity_failure) CHECK(cleanup_calls == 1 && audit_attempts == 1); + else CHECK(cleanup_calls == 0); + if (fail_allocation_at || complex_failure) CHECK(native_calls == 0); + return 0; + } + CHECK(ctx != NULL && ctx != NGX_CONF_ERROR && ctx == request.ctx); + CHECK(native_calls == 1 && registration_calls == 1 && cleanup_calls == 0); + CHECK(ctx->lifetime.pool == &pool && ctx->lifetime.conf == &conf); + CHECK(pool.cleanup.handler == ngx_http_modsecurity_cleanup && pool.cleanup.data == ctx); + if (conf.transaction_id != NULL) CHECK(custom_id_copied && custom_id_terminated); + ngx_http_modsecurity_cleanup(NULL); + pool.cleanup.handler(pool.cleanup.data); + pool.cleanup.handler(pool.cleanup.data); + CHECK(ctx->modsec_transaction == NULL && cleanup_calls == 1 && audit_attempts == 1); + CHECK(sanity_allocations == MODSECURITY_SANITY_CHECKS); + CHECK(sanity_destroys == MODSECURITY_SANITY_CHECKS); + return 0; +} +""" + + +def ownership_program(module: str) -> str: + helpers = "\nstatic void\n" + function_definition( + module, "ngx_http_modsecurity_cleanup" + ) + helpers += "\nngx_http_modsecurity_ctx_t *\n" + function_definition( + module, "ngx_http_modsecurity_create_ctx" + ) + return OWNERSHIP_PREAMBLE + helpers + OWNERSHIP_CASES + + +POLICY_PREAMBLE = r""" +#include +#include +#include +#include +typedef unsigned char u_char; +typedef long ngx_int_t; +typedef unsigned long ngx_uint_t; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct { int result, logs, cleanups; } Transaction; +typedef struct ngx_http_request_s ngx_http_request_t; +typedef struct ngx_pool_cleanup_s { + void (*handler)(void *); void *data; struct ngx_pool_cleanup_s *next; +} ngx_pool_cleanup_t; +typedef struct { ngx_pool_cleanup_t *cleanup; } ngx_pool_t; +typedef struct { void *elts; ngx_uint_t nelts; } ngx_array_t; +typedef struct { ngx_str_t literal; int failed; } ngx_http_complex_value_t; +typedef struct { ngx_int_t status, overwrite; ngx_http_complex_value_t value; } ngx_http_err_page_t; +typedef struct { ngx_array_t *error_pages; int named; ngx_str_t name; } ngx_http_core_loc_conf_t; +typedef struct ngx_http_modsecurity_conf_s { + void *rules_set; const void *rules_identity; + int enable, use_error_log, sanity_checks_enabled; + void *transaction_id, *phase4_log_file; ngx_uint_t phase4_mode; +} ngx_http_modsecurity_conf_t; +typedef struct { + ngx_http_request_t *r; + struct { ngx_pool_t *pool; const ngx_http_modsecurity_conf_t *conf; } lifetime; + struct { ngx_str_t target; ngx_int_t status, response_status; } error_page; + Transaction *modsec_transaction; + struct { char *log, *url; } pending_request_intervention; + int logged, native_logging_failed, request_policy_superseded; + int request_intervention_active, request_intervention_pending, native_request_body_limit_rejection; + int intervention_triggered, intervention_failed, phase4_failed; + ngx_int_t last_intervention_status; + ngx_array_t *sanity_headers_out; +} ngx_http_modsecurity_ctx_t; +typedef struct { void *log; int error; } ngx_connection_t; +struct ngx_http_request_s { + ngx_pool_t *pool; ngx_connection_t *connection; + ngx_http_modsecurity_ctx_t *ctx; + ngx_http_modsecurity_conf_t *conf; + ngx_http_core_loc_conf_t *core; + int internal, error_page, err_status, header_sent; ngx_str_t uri; +}; +enum { ngx_http_modsecurity_module = 1, ngx_http_core_module = 2 }; +#define NGX_OK 0 +#define NGX_ERROR (-1) +#define NGX_HTTP_BAD_REQUEST 400 +#define NGX_LOG_ERR 3 +#define ngx_http_get_module_ctx(r, module) ((r)->ctx) +#define ngx_http_set_ctx(r, value, module) ((r)->ctx = (value)) +#define ngx_http_get_module_loc_conf(r, module) \ + ((module) == ngx_http_core_module ? (void *)(r)->core : (void *)(r)->conf) +#define ngx_strncmp(a, b, n) strncmp((const char *)(a), (const char *)(b), (n)) +#define ngx_memzero(dst, size) memset((dst), 0, (size)) +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "line %d: %s\n", __LINE__, #condition); return 1; \ +} } while (0) +static ngx_pool_t *last_pool_seen; +static void ngx_http_modsecurity_cleanup(void *data); +static ngx_pool_t *ngx_http_modsecurity_pcre_malloc_init(ngx_pool_t *pool) +{ + last_pool_seen = pool; + return pool; +} +static void ngx_http_modsecurity_pcre_malloc_done(ngx_pool_t *pool) { (void)pool; } +static void ngx_log_error(int level, void *log, int error, const char *format, ...) +{ + (void)level; (void)log; (void)error; (void)format; +} +static int msc_process_logging(Transaction *transaction) +{ + ++transaction->logs; + return transaction->result; +} +static void msc_transaction_cleanup(Transaction *transaction) +{ + ++transaction->cleanups; + if (transaction->cleanups != 1) abort(); +} +static int ngx_http_complex_value(ngx_http_request_t *r, ngx_http_complex_value_t *value, ngx_str_t *out) +{ + (void)r; *out = value->literal; return value->failed ? NGX_ERROR : NGX_OK; +} +static u_char *ngx_strlchr(u_char *first, u_char *last, u_char value) +{ + return memchr(first, value, (size_t)(last - first)); +} +#if MODSECURITY_SANITY_CHECKS +static void ngx_array_destroy(ngx_array_t *array) { (void)array; } +#endif +""" + +POLICY_CASES = r""" +int main(int argc, char **argv) +{ + Transaction first = {1, 0, 0}, second = {1, 0, 0}, foreign_transaction = {1, 0, 0}; + ngx_connection_t connection = {0}; + ngx_http_modsecurity_conf_t original = {0}, destination; + ngx_http_core_loc_conf_t core = {0}; + ngx_http_request_t request = {0}, other = {0}; + ngx_http_modsecurity_ctx_t origin = {0}, target = {0}, foreign = {0}; + ngx_pool_cleanup_t source_cleanup = {ngx_http_modsecurity_cleanup, &origin, NULL}; + ngx_pool_cleanup_t target_cleanup = {ngx_http_modsecurity_cleanup, &target, &source_cleanup}; + ngx_pool_cleanup_t foreign_cleanup = {ngx_http_modsecurity_cleanup, &foreign, NULL}; + ngx_pool_t pool = {&source_cleanup}; + ngx_http_err_page_t error_page = {403, -1, {{24, (u_char *)"/block.html?observation=1"}, 0}}; + ngx_array_t pages = {&error_page, 1}; + ngx_http_modsecurity_ctx_t *selected; + int parameter, expected; + CHECK(argc == 3); + parameter = atoi(argv[2]); + original.enable = 1; original.rules_set = &first; original.rules_identity = &first; + destination = original; destination.rules_set = &second; + request.pool = &pool; request.connection = &connection; + request.conf = &destination; request.core = &core; + request.uri.data = (u_char *)"/block.html"; request.uri.len = 11; + origin.r = &request; origin.lifetime.pool = &pool; origin.lifetime.conf = &original; + origin.modsec_transaction = &first; + target.r = &request; target.lifetime.pool = &pool; target.lifetime.conf = &destination; + target.modsec_transaction = &second; + foreign.r = &other; foreign.lifetime.pool = &pool; foreign.lifetime.conf = &original; + foreign.modsec_transaction = &foreign_transaction; + source_cleanup.next = &foreign_cleanup; + + if (strcmp(argv[1], "same-policy") == 0) { + CHECK(ngx_http_modsecurity_same_policy(&original, &destination)); + CHECK(ngx_http_modsecurity_prepare_location(&request, &destination) == NGX_OK); + CHECK(!origin.request_policy_superseded); + CHECK(ngx_http_modsecurity_get_module_ctx(&request) == &origin && request.ctx == &origin); + CHECK(ngx_http_modsecurity_log_handler(&request) == NGX_OK); + CHECK(ngx_http_modsecurity_log_handler(&request) == NGX_OK); + CHECK(first.logs == 1 && second.logs == 0 && foreign_transaction.logs == 0); + } else if (strcmp(argv[1], "policy-key") == 0) { + if (parameter == 0) destination.rules_identity = &second; + if (parameter == 1) destination.enable = 0; + if (parameter == 2) destination.use_error_log = 1; + if (parameter == 3) destination.transaction_id = &second; + if (parameter == 4) destination.phase4_mode = 1; + if (parameter == 5) destination.phase4_log_file = &second; + if (parameter == 6) destination.sanity_checks_enabled = 1; + expected = parameter != 6 || MODSECURITY_SANITY_CHECKS; + CHECK(ngx_http_modsecurity_prepare_location(&request, &destination) == NGX_OK); + CHECK(origin.request_policy_superseded == expected); + CHECK(ngx_http_modsecurity_get_module_ctx(&request) == (expected ? NULL : &origin)); + } else if (strcmp(argv[1], "changed-policy") == 0 || strcmp(argv[1], "failed-audit") == 0 || strcmp(argv[1], "disabled-target") == 0) { + destination.rules_identity = &second; + if (strcmp(argv[1], "disabled-target") == 0) destination.enable = 0; + CHECK(ngx_http_modsecurity_prepare_location(&request, &destination) == NGX_OK); + CHECK(origin.request_policy_superseded && ngx_http_modsecurity_get_module_ctx(&request) == NULL); + if (destination.enable) { pool.cleanup = &target_cleanup; request.ctx = ⌖ } + if (strcmp(argv[1], "failed-audit") == 0) first.result = 0; + expected = first.result == 1 ? NGX_OK : NGX_ERROR; + CHECK(ngx_http_modsecurity_log_handler(&request) == expected); + CHECK(ngx_http_modsecurity_log_handler(&request) == expected); + CHECK(first.logs == 1 && second.logs == (destination.enable ? 1 : 0)); + CHECK(foreign_transaction.logs == 0 && !connection.error && !request.header_sent); + } else if (strcmp(argv[1], "cleanup-fallback") == 0) { + request.pool = NULL; + ngx_http_modsecurity_cleanup(&origin); + ngx_http_modsecurity_cleanup(&origin); + CHECK(first.logs == 1 && first.cleanups == 1 && origin.modsec_transaction == NULL); + CHECK(last_pool_seen == &pool && foreign_transaction.logs == 0); + } else if (strcmp(argv[1], "block-page") == 0) { + destination.rules_identity = &second; + core.error_pages = &pages; + origin.request_intervention_active = 1; + origin.intervention_triggered = 1; origin.last_intervention_status = 403; + request.internal = 1; request.error_page = 1; request.err_status = 403; + if (parameter == 6) origin.native_request_body_limit_rejection = 1; + if (parameter == 7) origin.request_intervention_active = 0; + if (parameter == 8 || parameter == 9) { + error_page.value.literal.data = (u_char *)"@block"; error_page.value.literal.len = 6; + core.named = parameter == 8; core.name = error_page.value.literal; + } + if (parameter == 10 || parameter == 11) { + error_page.overwrite = parameter == 10 ? 200 : 0; + request.err_status = error_page.overwrite; + } + if (parameter == 14) { + error_page.value.literal.data = (u_char *)"https://example.test/error"; + error_page.value.literal.len = 26; + } + if (parameter == 15) error_page.value.failed = 1; + if (parameter != 13) ngx_http_modsecurity_capture_error_page(&request, &origin, 403); + origin.request_intervention_active = 0; + if (parameter == 1) { request.uri.data = (u_char *)"/different"; request.uri.len = 10; } + if (parameter == 2) request.err_status = 451; + if (parameter == 3) request.internal = 0; + if (parameter == 4) origin.intervention_failed = 1; + if (parameter == 5) origin.last_intervention_status = 451; + if (parameter == 12) origin.r = &other; + expected = parameter == 0 || parameter == 8 || parameter == 10 || parameter == 11; + selected = ngx_http_modsecurity_get_module_ctx(&request); + CHECK(selected == (expected ? &origin : NULL)); + CHECK(first.logs == 0 && second.logs == 0 && foreign_transaction.logs == 0); + } else return 2; + return 0; +} +""" + + +def policy_program(module: str, logging: str) -> str: + helpers = "\nngx_int_t\n" + function_definition(logging, "ngx_http_modsecurity_log_context") + for name, result in ( + ("ngx_http_modsecurity_same_policy", "static int"), + ("ngx_http_modsecurity_capture_error_page", "static void"), + ("ngx_http_modsecurity_is_block_error_page", "static int"), + ("ngx_http_modsecurity_prepare_location", "ngx_int_t"), + ("ngx_http_modsecurity_get_module_ctx", "ngx_http_modsecurity_ctx_t *"), + ("ngx_http_modsecurity_log_request_contexts", "ngx_int_t"), + ("ngx_http_modsecurity_cleanup", "static void"), + ): + helpers += "\n" + result + "\n" + function_definition(module, name) + helpers += "\nngx_int_t\n" + function_definition(logging, "ngx_http_modsecurity_log_handler") + return POLICY_PREAMBLE + helpers + POLICY_CASES + + +class LifecycleRuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler or shutil.which(compiler[0]) is None: + raise RuntimeError("set CC to a GCC- or Clang-compatible C17 compiler") + workspace = tempfile.TemporaryDirectory(prefix="nginx-lifecycle-") + cls.addClassCleanup(workspace.cleanup) + directory = Path(workspace.name) + source = directory / "lifecycle.c" + source.write_text(unit_program( + (ROOT / "src/ngx_http_modsecurity_access.c").read_text(encoding="utf-8"), + (ROOT / "src/ngx_http_modsecurity_log.c").read_text(encoding="utf-8"), + ), encoding="utf-8") + cls.binary = directory / ("lifecycle.exe" if os.name == "nt" else "lifecycle") + build = subprocess.run( + [*compiler, "-std=c17", "-Wall", "-Wextra", "-Werror", + str(source), "-o", str(cls.binary)], + capture_output=True, text=True, check=False, timeout=60, + ) + if build.returncode: + raise AssertionError(f"Lifecycle helper build failed:\n{build.stderr}") + + def case(self, name: str, parameter: int) -> None: + result = subprocess.run( + [str(self.binary), name, str(parameter)], + capture_output=True, text=True, check=False, timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_internal_error_page_uses_fresh_location_context(self): + for status in (0, 403): + with self.subTest(status=status): + self.case("fresh-error-page", status) + + def test_foreign_direct_context_does_not_skip_access_processing(self): + self.case("foreign-direct", 403) + + def test_body_callback_without_a_recoverable_context_is_safe(self): + self.case("body-callback-no-context", 0) + + def test_error_page_preserves_p1_and_p2_status_decisions(self): + for scenario in ("p1-status", "p2-status"): + for status in (302, 403, 451): + with self.subTest(scenario=scenario, status=status): + self.case(scenario, status) + + def test_negative_interventions_fail_and_remain_terminal(self): + for phase in range(5): + with self.subTest(phase=phase): + self.case("intervention-error", phase) + + def test_native_phase_requires_exact_success(self): + for phase in range(4): + for result in (-1, 0, 2): + with self.subTest(phase=phase, result=result): + self.case("native-error", phase * 10 + result + 1) + + def test_native_append_partial_is_not_phase_failure(self): + for result in (-1, 0, 1, 2): + with self.subTest(result=result): + self.case("append", result) + + def test_file_and_memory_partial_results_have_equal_semantics(self): + for result in (-1, 0, 1, 2): + with self.subTest(result=result): + self.case("file", result) + + def test_file_chunks_are_bounded_and_partial_inspection_continues(self): + self.case("file-large", 0) + + def test_file_descriptor_read_and_allocation_failures_remain_errors(self): + for failure in range(9): + with self.subTest(failure=failure): + self.case("file-error", failure) + + def test_file_native_reject_is_dispatched_after_successful_eos(self): + self.case("file-reject", 1) + + def test_same_policy_and_native_block_page_do_not_repeat_request_phases(self): + for scenario in ("same-policy", "block-page"): + with self.subTest(scenario=scenario): + self.case(scenario, 0) + + def test_location_preparation_runs_before_disabled_or_failed_entry(self): + for scenario in ("disabled", "prepare-error"): + with self.subTest(scenario=scenario): + self.case(scenario, 0) + + def test_audit_result_is_checked_and_attempted_once(self): + for result in (-1, 0, 1, 2): + with self.subTest(result=result): + self.case("log-result", result) + + def test_audit_reentry_cannot_repeat_native_call(self): + self.case("log-reenter", 1) + + def test_missing_native_transaction_does_not_call_audit(self): + self.case("log-missing", 1) + + def test_disabled_context_does_not_call_audit(self): + self.case("log-no-context", 1) + + +class LifecycleOwnershipTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler or shutil.which(compiler[0]) is None: + raise RuntimeError("set CC to a GCC- or Clang-compatible C17 compiler") + workspace = tempfile.TemporaryDirectory(prefix="nginx-ownership-") + cls.addClassCleanup(workspace.cleanup) + directory = Path(workspace.name) + source = directory / "ownership.c" + source.write_text(ownership_program( + (ROOT / "src/ngx_http_modsecurity_module.c").read_text(encoding="utf-8") + ), encoding="utf-8") + cls.binaries = {} + for sanity in (0, 1): + binary = directory / (f"ownership-{sanity}.exe" if os.name == "nt" else f"ownership-{sanity}") + build = subprocess.run( + [*compiler, "-std=c17", "-Wall", "-Wextra", "-Werror", + f"-DMODSECURITY_SANITY_CHECKS={sanity}", str(source), "-o", str(binary)], + capture_output=True, text=True, check=False, timeout=60, + ) + if build.returncode: + raise AssertionError(f"Ownership helper build failed:\n{build.stderr}") + cls.binaries[sanity] = binary + + def case(self, name: str) -> None: + for sanity, binary in self.binaries.items(): + with self.subTest(sanity=sanity): + result = subprocess.run( + [str(binary), name], capture_output=True, text=True, + check=False, timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_context_and_native_allocation_failures_are_null(self): + for name in ("context-allocation", "native-allocation"): + with self.subTest(name=name): + self.case(name) + + def test_custom_id_failures_do_not_create_native_transaction(self): + for name in ("complex-value", "custom-id-allocation"): + with self.subTest(name=name): + self.case(name) + + def test_partial_creation_failure_cleans_native_owner(self): + for name in ("cleanup-registration", "sanity-allocation"): + with self.subTest(name=name): + self.case(name) + + def test_custom_id_copy_is_terminated_before_native_call(self): + self.case("custom-id-owned") + + def test_cleanup_is_null_safe_and_idempotent(self): + self.case("cleanup-once") + + +class LifecyclePolicyTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler or shutil.which(compiler[0]) is None: + raise RuntimeError("set CC to a GCC- or Clang-compatible C17 compiler") + workspace = tempfile.TemporaryDirectory(prefix="nginx-policy-") + cls.addClassCleanup(workspace.cleanup) + directory = Path(workspace.name) + source = directory / "policy.c" + source.write_text(policy_program( + (ROOT / "src/ngx_http_modsecurity_module.c").read_text(encoding="utf-8"), + (ROOT / "src/ngx_http_modsecurity_log.c").read_text(encoding="utf-8"), + ), encoding="utf-8") + cls.binaries = {} + for sanity in (0, 1): + binary = directory / (f"policy-{sanity}.exe" if os.name == "nt" else f"policy-{sanity}") + build = subprocess.run( + [*compiler, "-std=c17", "-Wall", "-Wextra", "-Werror", + f"-DMODSECURITY_SANITY_CHECKS={sanity}", str(source), "-o", str(binary)], + capture_output=True, text=True, check=False, timeout=60, + ) + if build.returncode: + raise AssertionError(f"Policy helper build failed:\n{build.stderr}") + cls.binaries[sanity] = binary + + def case(self, name: str, parameter: int = 0) -> None: + for sanity, binary in self.binaries.items(): + with self.subTest(sanity=sanity): + result = subprocess.run( + [str(binary), name, str(parameter)], capture_output=True, + text=True, check=False, timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_inherited_equivalent_policy_reuses_one_audited_owner(self): + self.case("same-policy") + + def test_semantic_policy_changes_require_a_new_owner(self): + for parameter in range(7): + with self.subTest(parameter=parameter): + self.case("policy-key", parameter) + + def test_changed_or_disabled_target_retains_source_audit(self): + for name in ("changed-policy", "disabled-target"): + with self.subTest(name=name): + self.case(name) + + def test_owner_audit_failure_is_sticky_without_skipping_other_owners(self): + self.case("failed-audit") + + def test_cleanup_audits_once_with_the_retained_pool(self): + self.case("cleanup-fallback") + + def test_exact_native_block_destination_can_reuse_its_origin(self): + for parameter in (0, 8, 10, 11): + with self.subTest(parameter=parameter): + self.case("block-page", parameter) + + def test_unrelated_routes_cannot_claim_the_block_page_exemption(self): + for parameter in (1, 2, 3, 4, 5, 6, 7, 9, 12, 13, 14, 15): + with self.subTest(parameter=parameter): + self.case("block-page", parameter) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_log_metadata_runtime.py b/tests/test_log_metadata_runtime.py new file mode 100644 index 00000000..e22b3777 --- /dev/null +++ b/tests/test_log_metadata_runtime.py @@ -0,0 +1,346 @@ +"""Compile the real standalone metadata/JSON header with small NGINX doubles. + +Expected records follow the complete native RuleMessage framing contract. +Python's strict UTF-8 decoder and JSON parser independently check the escaped +output. Invalid input bytes are represented as U+00xx; valid UTF-8 is retained. +These are helper regressions, not native HTTP or audit-log integration tests. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + +PREAMBLE = r""" +#include +#include +#include +#include +#include + +typedef unsigned char u_char; +typedef intptr_t ngx_int_t; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct { int fail; unsigned calls; } ngx_pool_t; +#define ngx_inline inline + +static void *allocations[8]; +static unsigned allocation_count; + +static void *ngx_pnalloc(ngx_pool_t *pool, size_t size) +{ + void *result; + + pool->calls++; + if (pool->fail || allocation_count == 8) { + return NULL; + } + result = malloc(size == 0 ? 1 : size); + if (result != NULL) { + allocations[allocation_count++] = result; + } + return result; +} + +#include "ngx_http_modsecurity_log_metadata.h" +""" + +PROGRAM = PREAMBLE + r""" +static int from_hex(char value) +{ + if (value >= '0' && value <= '9') return value - '0'; + if (value >= 'a' && value <= 'f') return value - 'a' + 10; + if (value >= 'A' && value <= 'F') return value - 'A' + 10; + return -1; +} + +int main(int argc, char **argv) +{ + ngx_pool_t pool = {0, 0}; + ngx_str_t input, output = {5, (u_char *) "stale"}; + u_char *bytes; + size_t size, index; + ngx_int_t result; + int high, low; + + if (argc < 3 || strlen(argv[2]) % 2 != 0) return 2; + size = strlen(argv[2]) / 2; + /* Deliberately do not append a NUL: the production API is len-bounded. */ + bytes = malloc(size == 0 ? 1 : size); + if (bytes == NULL) return 2; + for (index = 0; index < size; index++) { + high = from_hex(argv[2][index * 2]); + low = from_hex(argv[2][index * 2 + 1]); + if (high < 0 || low < 0) return 2; + bytes[index] = (u_char) (high * 16 + low); + } + input.data = bytes; + input.len = size; + if (argc > 3) pool.fail = atoi(argv[3]); + if (argc > 4) { + input.len = (size_t) strtoull(argv[4], NULL, 10); + if (input.len > size) return 2; + } + if (strcmp(argv[1], "json-oversize") == 0) input.len = SIZE_MAX; + if (strcmp(argv[1], "json-null") == 0 || + strcmp(argv[1], "metadata-null") == 0) input.data = NULL; + + if (strncmp(argv[1], "json", 4) == 0) { + ngx_http_modsecurity_json_escape(&pool, &input, &output); + if (output.data == NULL) { + if (output.len != 0) return 3; + fputs("NULL", stdout); + } else { + if (output.data[output.len] != '\0') return 3; + fwrite(output.data, 1, output.len, stdout); + } + } else if (strncmp(argv[1], "metadata", 8) == 0) { + result = ngx_http_modsecurity_parse_rule_metadata(input, &output); + if (pool.calls != 0) return 3; + printf("%ld\n", (long) result); + if (result != 1 && output.len != 0) return 3; + fwrite(output.data, 1, output.len, stdout); + } else if (strcmp(argv[1], "extract") == 0) { + ngx_http_modsecurity_extract_rule_id(&pool, &input, &output); + /* The copied ID must survive overwriting the temporary input. */ + memset(bytes, 'x', size); + if (output.data == NULL) { + if (output.len != 0) return 3; + fputs("NULL", stdout); + } else { + if (output.data[output.len] != '\0') return 3; + fwrite(output.data, 1, output.len, stdout); + } + } else if (strcmp(argv[1], "summary") == 0) { + output = ngx_http_modsecurity_sanitize_intervention(&pool, input); + if (pool.calls != 0) return 3; + fwrite(output.data, 1, output.len, stdout); + } else { + return 2; + } + free(bytes); + for (index = 0; index < allocation_count; index++) free(allocations[index]); + return 0; +} +""" + +PARSER_ONLY = PREAMBLE + r""" +int main(void) +{ + ngx_str_t input = {0, (u_char *) ""}; + ngx_str_t output; + + return ngx_http_modsecurity_parse_rule_metadata(input, &output) != 0; +} +""" + + +def engine_record(*, rule_id: str = "942100", changes=None, tags=(), tail=False) -> bytes: + fields = [ + ("file", "/rules/response.conf"), + ("line", "12"), + ("id", rule_id), + ("rev", ""), + ("msg", "response rule"), + ("data", r"Matched \x22data\x22"), + ("severity", "2"), + ("ver", "test"), + ("maturity", "0"), + ("accuracy", "0"), + ] + fields += [("tag", value) for value in tags] + fields += [ + ("hostname", "example.invalid"), + ("uri", "/response"), + ("unique_id", "transaction"), + ("ref", "o0,4v0,4"), + ] + if changes: + fields = [(name, changes.get(name, value)) for name, value in fields] + if tail: + fields += [("hostname", "example.invalid"), ("uri", "/response"), + ("unique_id", "transaction")] + return " ".join(f'[{name} "{value}"]' for name, value in fields).encode("ascii") + + +class LogMetadataRuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.compiler = shlex.split(os.environ.get("CC", "cc")) + if not cls.compiler or shutil.which(cls.compiler[0]) is None: + raise unittest.SkipTest("A C compiler is needed for metadata helper regressions") + cls.directory = tempfile.TemporaryDirectory(prefix="modsec-log-metadata-") + cls.addClassCleanup(cls.directory.cleanup) + cls.binary = Path(cls.directory.name) / "metadata" + cls.parser_binary = Path(cls.directory.name) / "parser-only" + for name, source, binary in ( + ("metadata.c", PROGRAM, cls.binary), + ("parser-only.c", PARSER_ONLY, cls.parser_binary), + ): + path = Path(cls.directory.name) / name + path.write_text(source, encoding="utf-8") + completed = subprocess.run( + cls.compiler + ["-std=c99", "-Wall", "-Wextra", "-Werror", + "-I", str(ROOT / "src"), str(path), "-o", str(binary)], + capture_output=True, text=True, check=False, + ) + if completed.returncode != 0: + raise AssertionError(completed.stdout + completed.stderr) + + def execute(self, operation: str, data: bytes, *, fail=False, length=None) -> bytes: + command = [str(self.binary), operation, data.hex(), str(int(fail))] + if length is not None: + command.append(str(length)) + completed = subprocess.run(command, capture_output=True, check=False) + self.assertEqual(completed.returncode, 0, completed.stderr.decode("utf-8", "replace")) + return completed.stdout + + def parse(self, data: bytes, **kwargs): + result, rule_id = self.execute("metadata", data, **kwargs).split(b"\n", 1) + return int(result), rule_id + + def escaped(self, data: bytes, **kwargs): + output = self.execute("json", data, **kwargs) + text = output.decode("utf-8", "strict") + return output, json.loads('"' + text + '"') + + def test_header_can_be_included_for_only_the_parser_without_unused_warnings(self): + completed = subprocess.run([str(self.parser_binary)], capture_output=True, check=False) + self.assertEqual(completed.returncode, 0, completed.stderr) + + def test_complete_record_and_optional_error_log_tail(self): + for tail in (False, True): + with self.subTest(tail=tail): + self.assertEqual(self.parse(engine_record(tail=tail)), (1, b"942100")) + + def test_decimal_ids_are_preserved_without_integer_conversion(self): + value = "000" + "9" * 100 + self.assertEqual(self.parse(engine_record(rule_id=value)), (1, value.encode("ascii"))) + + def test_reflected_bare_id_before_actual_metadata_does_not_win(self): + data = b'Matched data: id "1" Operator msg "reflected" ' + engine_record() + self.assertEqual(self.parse(data), (1, b"942100")) + self.assertEqual(self.execute("summary", data), b"id:present msg:present op:-") + + def test_bare_presence_snippets_remain_redacted(self): + for data in (b'id "1"', b'Operator', b'msg "body"', b'id "1" Operator'): + with self.subTest(data=data): + self.assertEqual(self.parse(data), (0, b"")) + self.assertEqual(self.execute("summary", data), b"redacted") + + def test_forged_or_duplicate_records_are_unknown(self): + real = engine_record() + for data in ( + engine_record(rule_id="1") + b" " + real, + b'[file "malformed" ' + real, + engine_record(changes={"uri": '/[file "forged'}), + b"prefix" + real, + ): + with self.subTest(data=data): + self.assertEqual(self.parse(data), (-1, b"")) + self.assertEqual(self.execute("summary", data), b"redacted") + + def test_complete_order_and_terminal_framing_are_required(self): + record = engine_record() + invalid = ( + record[:-1], + record + b" trailing", + record.replace(b'[rev ""] ', b""), + record.replace(b'[line "12"] [id "942100"]', b'[id "942100"] [line "12"]'), + record + b' [hostname "example.invalid"]', + ) + for data in invalid: + with self.subTest(data=data): + self.assertEqual(self.parse(data), (-1, b"")) + + def test_native_numeric_fields_must_be_nonempty_decimal(self): + for field in ("line", "id", "severity", "maturity", "accuracy"): + for value in ("", "-1", "1x"): + with self.subTest(field=field, value=value): + self.assertEqual(self.parse(engine_record(changes={field: value})), (-1, b"")) + + def test_raw_quotes_and_hex_escaped_fields_follow_engine_framing(self): + record = engine_record(changes={"file": '/rules/quoted"name.conf'}, + tags=(r"attack-\x22tag\x22", "second")) + self.assertEqual(self.parse(record), (1, b"942100")) + for value in (r"bad\z", r"bad\x2", r"bad\xGG", 'raw"quote'): + with self.subTest(value=value): + self.assertEqual(self.parse(engine_record(changes={"data": value})), (-1, b"")) + + def test_metadata_is_length_bounded_and_rejects_embedded_controls(self): + record = engine_record() + self.assertEqual(self.parse(record + b' [file "outside"]', length=len(record)), + (1, b"942100")) + for character in (b"\0", b"\n", b"\x80"): + data = record.replace(b"response rule", b"response" + character + b"rule") + self.assertEqual(self.parse(data), (-1, b"")) + self.assertEqual(self.execute("metadata-null", b""), b"0\n") + self.assertEqual(self.execute("metadata-null", b"x"), b"-1\n") + + def test_extracted_id_is_an_owned_nul_terminated_copy(self): + self.assertEqual(self.execute("extract", engine_record()), b"942100") + self.assertEqual(self.execute("extract", b'id "1"'), b"") + self.assertEqual(self.execute("extract", engine_record(), fail=True), b"NULL") + + def test_summary_is_complete_redacted_and_needs_no_allocation(self): + record = engine_record(changes={"msg": "Operator private matched-data"}) + expected = b"id:present msg:present op:-" + self.assertEqual(self.execute("summary", record), expected) + self.assertEqual(self.execute("summary", record, fail=True), expected) + self.assertNotIn(b"private", self.execute("summary", record)) + + def test_all_bytes_produce_strict_json_without_losing_controls(self): + data = bytes(range(256)) + escaped, value = self.escaped(data) + self.assertEqual(value, "".join(chr(character) for character in range(256))) + self.assertIn(b"\\u0000", escaped) + self.assertIn(b"\\u001f", escaped) + self.assertIn(b"\\u00ff", escaped) + + def test_valid_utf8_is_preserved_byte_for_byte(self): + text = "Grüße 日本語 😀\u2028\u2029" + data = text.encode("utf-8") + escaped, value = self.escaped(data) + self.assertEqual(escaped, data) + self.assertEqual(value, text) + + def test_valid_utf8_boundary_sequences_are_retained(self): + data = b"\xc2\x80\xdf\xbf\xe0\xa0\x80\xed\x9f\xbf\xee\x80\x80\xef\xbf\xbf\xf0\x90\x80\x80\xf4\x8f\xbf\xbf" + escaped, value = self.escaped(data) + self.assertEqual(escaped, data) + self.assertEqual(value, data.decode("utf-8", "strict")) + + def test_invalid_and_truncated_utf8_bytes_are_explicitly_escaped(self): + invalid = ( + b"\x80\xbf", b"\xc0\xaf", b"\xc1\xbf", b"\xe0\x80\xaf", + b"\xed\xa0\x80", b"\xf0\x80\x80\x80", b"\xf4\x90\x80\x80", + b"\xf5\x80\x80\x80", b"\xff", b"\xc2", b"\xe1\x80", b"\xf1\x80\x80", + ) + for data in invalid: + with self.subTest(data=data): + escaped, value = self.escaped(data) + self.assertTrue(escaped.isascii()) + self.assertEqual(value, "".join(chr(character) for character in data)) + + def test_json_respects_declared_length(self): + self.assertEqual(self.escaped(b"ok\xff", length=2), (b"ok", "ok")) + + def test_json_allocation_failure_is_distinct_from_valid_empty(self): + self.assertEqual(self.execute("json", b"ordinary", fail=True), b"NULL") + self.assertEqual(self.execute("json", b"", fail=True), b"") + self.assertEqual(self.execute("json-null", b""), b"") + self.assertEqual(self.execute("json-null", b"x"), b"NULL") + self.assertEqual(self.execute("json-oversize", b"x"), b"NULL") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_nginx_patch_runtime.py b/tests/test_nginx_patch_runtime.py new file mode 100644 index 00000000..15bba629 --- /dev/null +++ b/tests/test_nginx_patch_runtime.py @@ -0,0 +1,165 @@ +"""Exercise the real NGINX patch applier at its filesystem/process boundary. + +The minimal official close-stream guard is an application fixture, not a +native HTTP/2 runtime. The multiplexed .t regression proves reset delivery. +""" + +from __future__ import annotations + +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +HELPER = ROOT / ".github/scripts/apply-nginx-http2-patch.sh" +PATCH = ROOT / "patches/nginx-http2-close-stream-flush.patch" + +PREFIX = ( + "/* Minimal official ngx_http_v2_close_stream application context. */\n" + "void\nngx_http_v2_close_stream(void)\n{\n" + " h2c->processing--;\n\n" +) +OLD_GUARD = ( + " if (h2c->processing || h2c->blocked) {\n" + " return;\n" + " }\n" +) +NEW_GUARD = ( + " if (h2c->processing || h2c->blocked) {\n" + " /*\n" + " * A reset queued while finalizing this stream must not wait for\n" + " * another stream to finish or for a new socket read event. Let the\n" + " * existing write handler flush it after this request is destroyed.\n" + " */\n" + " if (h2c->last_out && !h2c->blocked) {\n" + " ngx_post_event(h2c->connection->write, &ngx_posted_events);\n" + " }\n\n" + " return;\n" + " }\n" +) +SUFFIX = "\n ev = h2c->connection->read;\n}\n" + + +def with_eol(text: str, ending: bytes) -> bytes: + return text.encode("ascii").replace(b"\n", ending) + + +class NginxPatchRuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.shell = shutil.which("sh") + patch = shutil.which("patch") + if cls.shell is None or patch is None: + raise RuntimeError("POSIX sh and GNU patch are required") + version = subprocess.run( + [patch, "--version"], capture_output=True, text=True, + timeout=10, check=False, + ) + if version.returncode or "GNU patch" not in version.stdout: + raise RuntimeError("the patch-applier regressions require GNU patch") + cls.helper_bytes = HELPER.read_bytes() + cls.patch_bytes = PATCH.read_bytes().replace(b"\r\n", b"\n") + + def fixture(self, source_eol: bytes, patch_eol: bytes, *, spaces=False): + temporary = tempfile.TemporaryDirectory(prefix="nginx patch fixture ") + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + repository = root / ("connector checkout" if spaces else "connector") + helper = repository / ".github/scripts/apply-nginx-http2-patch.sh" + patch = repository / "patches/nginx-http2-close-stream-flush.patch" + source_root = root / ("nginx source checkout" if spaces else "nginx") + source = source_root / "src/http/v2/ngx_http_v2.c" + helper.parent.mkdir(parents=True) + patch.parent.mkdir(parents=True) + source.parent.mkdir(parents=True) + helper.write_bytes(self.helper_bytes) + patch.write_bytes(self.patch_bytes.replace(b"\n", patch_eol)) + source.write_bytes(with_eol(PREFIX + OLD_GUARD + SUFFIX, source_eol)) + return root, helper, patch, source_root, source + + def invoke(self, helper: Path, *arguments: Path): + return subprocess.run( + [self.shell, str(helper), *(str(argument) for argument in arguments)], + capture_output=True, text=True, timeout=15, check=False, + ) + + def assert_no_patch_debris(self, root: Path): + leftovers = [str(path.relative_to(root)) for path in root.rglob("*") + if path.is_file() and path.suffix in (".rej", ".orig")] + self.assertEqual(leftovers, [], "failed/dry-run application leaves no rejects/backups") + + def check_application(self, source_eol: bytes, patch_eol: bytes, *, spaces=False): + root, helper, patch, source_root, source = self.fixture( + source_eol, patch_eol, spaces=spaces, + ) + original_patch = patch.read_bytes() + result = self.invoke(helper, source_root) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + expected = with_eol(PREFIX + NEW_GUARD + SUFFIX, source_eol) + self.assertEqual(source.read_bytes(), expected, + "only the requested hunk changes and the source EOL stays intact") + self.assertEqual(patch.read_bytes(), original_patch, "input patch is never rewritten") + again = self.invoke(helper, source_root) + self.assertEqual(again.returncode, 0, again.stdout + again.stderr) + self.assertEqual(source.read_bytes(), expected, "second application is idempotent") + self.assertEqual(patch.read_bytes(), original_patch) + self.assert_no_patch_debris(root) + + def test_lf_source_lf_patch(self): + self.check_application(b"\n", b"\n") + + def test_crlf_source_lf_patch(self): + self.check_application(b"\r\n", b"\n") + + def test_lf_source_crlf_patch(self): + self.check_application(b"\n", b"\r\n") + + def test_crlf_source_crlf_patch(self): + self.check_application(b"\r\n", b"\r\n") + + def test_unmatched_hunk_preserves_source_without_rejects_or_backups(self): + for ending in (b"\n", b"\r\n"): + with self.subTest(source_eol=repr(ending)): + root, helper, patch, source_root, source = self.fixture(ending, b"\n") + invalid = with_eol( + PREFIX + OLD_GUARD.replace("h2c->processing || h2c->blocked", "h2c->processing") + + SUFFIX, ending, + ) + source.write_bytes(invalid) + original_patch = patch.read_bytes() + result = self.invoke(helper, source_root) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(source.read_bytes(), invalid) + self.assertEqual(patch.read_bytes(), original_patch) + self.assert_no_patch_debris(root) + + def test_missing_http2_source_fails_without_creating_it(self): + root, helper, patch, source_root, source = self.fixture(b"\n", b"\n") + source.unlink() + result = self.invoke(helper, source_root) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(source.exists()) + self.assertTrue(patch.is_file()) + self.assert_no_patch_debris(root) + + def test_usage_rejects_missing_or_extra_arguments(self): + root, helper, patch, source_root, source = self.fixture(b"\n", b"\n") + original = source.read_bytes() + for arguments in ((), (source_root, source_root)): + with self.subTest(argument_count=len(arguments)): + result = self.invoke(helper, *arguments) + self.assertEqual(result.returncode, 2) + self.assertIn("Usage:", result.stderr) + self.assertEqual(source.read_bytes(), original) + self.assertTrue(patch.is_file()) + self.assert_no_patch_debris(root) + + def test_source_and_repository_paths_with_spaces(self): + self.check_application(b"\r\n", b"\r\n", spaces=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_phase4_runtime.py b/tests/test_phase4_runtime.py new file mode 100644 index 00000000..2d06710d --- /dev/null +++ b/tests/test_phase4_runtime.py @@ -0,0 +1,908 @@ +"""Compile and exercise the production Phase4 helpers with small host doubles. + +These tests cover accounting, buffer representations, EOS, and intervention +dispatch. Native NGINX/libModSecurity HTTP tests remain in modsecurity*.t. +There is no dependency on the Common connector or its runtime framework. +""" + +from __future__ import annotations + +import os +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +BODY = ROOT / "src" / "ngx_http_modsecurity_body_filter.c" +MODULE = ROOT / "src" / "ngx_http_modsecurity_module.c" + + +def matching_delimiter(source: str, opening: int, left: str, right: str) -> int: + depth = 0 + for index in range(opening, len(source)): + if source[index] == left: + depth += 1 + elif source[index] == right: + depth -= 1 + if depth == 0: + return index + raise AssertionError(f"unterminated {left}{right} pair") + + +def function_definition(source: str, name: str) -> str: + for match in re.finditer(rf"\b{re.escape(name)}\s*\(", source): + opening = source.index("(", match.start()) + closing = matching_delimiter(source, opening, "(", ")") + cursor = closing + 1 + while cursor < len(source) and source[cursor].isspace(): + cursor += 1 + if cursor >= len(source) or source[cursor] != "{": + continue + end = matching_delimiter(source, cursor, "{", "}") + start = source.rfind("\n", 0, match.start()) + 1 + return source[start:end + 1] + raise AssertionError(f"{name} definition was not found") + + +PREAMBLE = r""" +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 64 +#endif +#include +#include +#include +#include +#include +#include +#ifdef _MSC_VER +typedef intptr_t ssize_t; +#endif +typedef unsigned char u_char; +typedef int ngx_int_t; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct ngx_http_request_s ngx_http_request_t; +typedef struct ngx_http_modsecurity_conf_s ngx_http_modsecurity_conf_t; +typedef struct { int named; ngx_str_t name; } ngx_http_core_loc_conf_t; +typedef struct ngx_pool_cleanup_s { + void (*handler)(void *); + void *data; + struct ngx_pool_cleanup_s *next; +} ngx_pool_cleanup_t; +typedef struct { ngx_pool_cleanup_t *cleanup; } ngx_pool_t; +typedef struct { int error; void *log; int buffered; } ngx_connection_t; +typedef struct { u_char *data; size_t length; } ngx_file_t; +typedef struct { + u_char *pos, *last; + off_t file_pos, file_last; + ngx_file_t *file; + unsigned memory, in_file, last_buf, last_in_chain; + unsigned flush, sync; +} ngx_buf_t; +typedef struct ngx_chain_s { + ngx_buf_t *buf; + struct ngx_chain_s *next; +} ngx_chain_t; +typedef struct { + size_t response_body_bytes_seen, response_body_bytes_inspected; + int response_body_seen, response_body_truncated, phase4_processed; + int phase4_strict_abort, phase4_headers_checked, intervention_triggered; + int intervention_late, intervention_failed, response_replaced, last_intervention_status; + int phase4_failed, phase4_error_emitting, phase4_started, intervention_body_limit; + int phase4_append_active, replacement_body_terminal_sent; + int logged, request_policy_superseded; + struct { ngx_pool_t *pool; const ngx_http_modsecurity_conf_t *conf; } lifetime; + struct { ngx_str_t target; ngx_int_t status, response_status; } error_page; + ngx_http_request_t *r; + void *modsec_transaction; + u_char *phase4_file_scratch; +} ngx_http_modsecurity_ctx_t; +struct ngx_http_modsecurity_conf_s { + unsigned phase4_mode; + size_t phase4_body_limit; + int enable, use_error_log; + void *rules_identity, *transaction_id, *phase4_log_file; +}; +struct ngx_http_request_s { + int header_sent, filter_finalize; + int buffered, internal, err_status; + ngx_str_t uri; + ngx_http_core_loc_conf_t *core_conf; + struct ngx_http_request_s *main; + ngx_pool_t *pool; + ngx_connection_t *connection; + ngx_http_modsecurity_ctx_t *ctx; + ngx_http_modsecurity_conf_t *conf; +}; +#define NGX_OK 0 +#define NGX_ERROR (-1) +#define NGX_AGAIN (-2) +#define NGX_DECLINED (-5) +#define NGX_HTTP_INTERNAL_SERVER_ERROR 500 +#define NGX_HTTP_FORBIDDEN 403 +#define NGX_HTTP_MODSEC_PHASE4_MODE_OFF 0 +#define NGX_HTTP_MODSEC_PHASE4_MODE_SAFE 1 +#define NGX_HTTP_MODSEC_PHASE4_MODE_STRICT 2 +#define NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK 32768U +#define ngx_buf_in_memory(buffer) ((buffer)->memory) +#define ngx_http_get_module_ctx(r, module) ((r)->ctx) +#define ngx_http_core_module 1 +#define ngx_http_get_module_loc_conf(r, module) \ + ((module) == ngx_http_core_module ? (void *)(r)->core_conf : (void *)(r)->conf) +#define ngx_http_set_ctx(r, value, module) ((r)->ctx = (value)) +#define ngx_strncmp(a, b, n) strncmp((const char *)(a), (const char *)(b), (n)) +#define ngx_log_error(...) (++diagnostic_calls) +static int ngx_http_modsecurity_module; +static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r); +static void ngx_http_modsecurity_cleanup(void *data) { (void)data; } +static int allocation_calls, read_calls, append_calls, process_calls; +static int fail_allocation, fail_read, short_read, fail_append, append_result = 1; +static int engine_result = 1, native_result, native_late, native_failed, native_body_limit; +static int log_result, event_calls, pcre_enter, pcre_leave; +static int diagnostic_calls, native_only_after_eos, append_collections, eos_with_append_active; +static int forward_calls, forwarded_links, forward_result; +static int verify_writer, writer_rejected_zero, forwarded_controls, forwarded_eos; +static int control_link_calls, control_buffer_calls, fail_control_link, fail_control_buffer; +static ngx_chain_t control_links[16]; +static ngx_buf_t control_buffers[16]; +static int finalize_calls, emulate_core_response, core_response_result; +static ngx_chain_t *core_response; +ngx_int_t ngx_http_modsecurity_body_filter(ngx_http_request_t *, ngx_chain_t *); +static size_t read_max, appended_bytes; +static u_char scratch[32768], inspected[200000]; +static const char *last_action, *last_reason; + +static ngx_chain_t *ngx_alloc_chain_link(ngx_pool_t *pool) +{ + (void)pool; + if (fail_control_link || control_link_calls >= 16) return NULL; + return &control_links[control_link_calls++]; +} +static ngx_buf_t *ngx_calloc_buf(ngx_pool_t *pool) +{ + (void)pool; + if (fail_control_buffer || control_buffer_calls >= 16) return NULL; + return &control_buffers[control_buffer_calls++]; +} +static void *ngx_pnalloc(ngx_pool_t *pool, size_t bytes) +{ + (void)pool; + ++allocation_calls; + return !fail_allocation && bytes <= sizeof(scratch) ? scratch : NULL; +} +static ssize_t ngx_read_file(ngx_file_t *file, u_char *out, + size_t bytes, off_t offset) +{ + ++read_calls; + if (bytes > read_max) read_max = bytes; + if (fail_read || offset < 0) return -1; + if ((size_t)offset > file->length || bytes > file->length - (size_t)offset) + return 0; + if (short_read && bytes != 0U) --bytes; + memcpy(out, file->data + offset, bytes); + return (ssize_t)bytes; +} +static int msc_append_response_body(void *transaction, u_char *data, size_t bytes) +{ + (void)transaction; + ++append_calls; + if (fail_append || bytes > sizeof(inspected) - appended_bytes) return -1; + memcpy(inspected + appended_bytes, data, bytes); + appended_bytes += bytes; + return append_result; +} +static int msc_process_response_body(void *transaction) +{ + ngx_http_modsecurity_ctx_t *ctx = transaction; + if (ctx->phase4_append_active) ++eos_with_append_active; + ++process_calls; + return engine_result; +} +static ngx_pool_t *ngx_http_modsecurity_pcre_malloc_init(ngx_pool_t *pool) +{ + ++pcre_enter; + return pool; +} +static void ngx_http_modsecurity_pcre_malloc_done(ngx_pool_t *pool) +{ + (void)pool; + ++pcre_leave; +} +static int ngx_http_modsecurity_process_intervention(void *transaction, + ngx_http_request_t *r, int early) +{ + ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); + (void)transaction; (void)early; + if (ctx == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; + if (ctx->phase4_append_active) ++append_collections; + if (native_only_after_eos && !ctx->phase4_processed) return 0; + ctx->intervention_late = native_late; + ctx->intervention_failed = native_failed; + ctx->intervention_body_limit = native_body_limit; + return native_result; +} +static ngx_int_t ngx_http_filter_finalize_request(ngx_http_request_t *r, + int *module, int status) +{ + (void)module; + ++finalize_calls; + if (!r->header_sent && emulate_core_response) { + int previous = r->filter_finalize; + r->filter_finalize = 1; + core_response_result = ngx_http_modsecurity_body_filter(r, core_response); + r->filter_finalize = previous; + } + return r->header_sent ? NGX_ERROR : status; +} +static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, + ngx_http_modsecurity_conf_t *conf, const char *wanted, + const char *actual, const char *reason) +{ + (void)r; (void)conf; (void)wanted; + ++event_calls; + last_action = actual; last_reason = reason; + return log_result; +} +static ngx_int_t ngx_http_next_body_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + (void)r; + ++forward_calls; + for (; in != NULL; in = in->next) { + ngx_buf_t *buffer = in->buf; + int special = buffer != NULL && !buffer->memory && !buffer->in_file && + (buffer->flush || buffer->sync || buffer->last_buf); + int empty = buffer == NULL || (buffer->memory + ? buffer->pos == buffer->last + : (!buffer->in_file || buffer->file_pos == buffer->file_last)); + if (verify_writer && empty && !special) { + ++writer_rejected_zero; + return NGX_ERROR; + } + ++forwarded_links; + if (special) ++forwarded_controls; + if (buffer != NULL && (buffer->last_buf || + (r != r->main && buffer->last_in_chain))) ++forwarded_eos; + } + return forward_result; +} +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "line %d: %s\n", __LINE__, #condition); return 1; \ +} } while (0) +""" + +CASES = r""" +int main(int argc, char **argv) +{ + ngx_http_modsecurity_ctx_t ctx = {0}; + ngx_http_modsecurity_conf_t conf = { + .phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_SAFE, + .phase4_body_limit = 1048576U, .enable = 1 + }; + ngx_pool_t pool = {0}; + ngx_connection_t connection = {0}; + ngx_http_request_t request = {0}; + ngx_http_request_t main_request = {0}; + u_char payload[70000]; + ngx_file_t file = {payload, sizeof(payload)}; + ngx_buf_t buffer = {0}; + ngx_buf_t final = {0}; + ngx_chain_t tail = {&final, NULL}; + ngx_chain_t chain = {&buffer, &tail}; + size_t allowed = 999U; + size_t i; + CHECK(argc == 2); + memset(payload, 'A', sizeof(payload)); + request.pool = &pool; request.connection = &connection; + request.ctx = &ctx; request.conf = &conf; request.main = &request; ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; + buffer.file = &file; buffer.in_file = 1; buffer.file_last = sizeof(payload); + + if (strcmp(argv[1], "request-ownership") == 0) { + ngx_http_modsecurity_ctx_t other_ctx = {0}; + ngx_http_request_t subrequest = {0}; + ngx_pool_cleanup_t owner_cleanup = {ngx_http_modsecurity_cleanup, &ctx, NULL}; + ngx_pool_cleanup_t other_cleanup = {ngx_http_modsecurity_cleanup, &other_ctx, &owner_cleanup}; + other_ctx.r = &subrequest; other_ctx.modsec_transaction = &other_ctx; + other_ctx.lifetime.pool = &pool; other_ctx.lifetime.conf = &conf; + subrequest.conf = &conf; + subrequest.pool = &pool; subrequest.main = &request; + pool.cleanup = &other_cleanup; + CHECK(ngx_http_modsecurity_get_module_ctx(&request) == &ctx); + request.ctx = NULL; /* Same-request internal redirect clears module context. */ + CHECK(ngx_http_modsecurity_get_module_ctx(&request) == &ctx); + CHECK(ngx_http_modsecurity_get_module_ctx(&subrequest) == &other_ctx); + subrequest.ctx = &ctx; /* Reject a foreign direct context too. */ + CHECK(ngx_http_modsecurity_get_module_ctx(&subrequest) == &other_ctx); + pool.cleanup = &owner_cleanup; /* Auth subrequest has no own transaction. */ + subrequest.ctx = NULL; + CHECK(ngx_http_modsecurity_get_module_ctx(&subrequest) == NULL); + subrequest.ctx = NULL; subrequest.connection = &connection; + final.last_in_chain = 1; + CHECK(ngx_http_modsecurity_body_filter(&subrequest, &tail) == NGX_OK); + CHECK(process_calls == 0 && !ctx.phase4_processed && forward_calls == 1); + final.last_in_chain = 0; final.last_buf = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &tail) == NGX_OK); + CHECK(process_calls == 1 && ctx.phase4_processed && forward_calls == 2); + } else if (strcmp(argv[1], "all-modes-large") == 0) { + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1048577U, &allowed) == NGX_OK); + CHECK(allowed == 1048577U && ctx.response_body_bytes_seen == allowed); + CHECK(ctx.response_body_seen && !ctx.response_body_truncated); + CHECK(ctx.response_body_bytes_inspected == 0U); + } + } else if (strcmp(argv[1], "all-modes-multiple") == 0) { + size_t chunk; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + for (chunk = 0; chunk < 3; ++chunk) { + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1048576U, &allowed) == NGX_OK); + CHECK(allowed == 1048576U); + ctx.response_body_bytes_inspected += allowed; + } + CHECK(ctx.response_body_bytes_seen == 3U * 1048576U); + CHECK(ctx.response_body_bytes_inspected == ctx.response_body_bytes_seen); + CHECK(!ctx.response_body_truncated); + } + } else if (strcmp(argv[1], "legacy-boundary") == 0) { + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1048575U, &allowed) == NGX_OK); + ctx.response_body_bytes_inspected += allowed; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1U, &allowed) == NGX_OK); + ctx.response_body_bytes_inspected += allowed; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1U, &allowed) == NGX_OK); + ctx.response_body_bytes_inspected += allowed; + CHECK(allowed == 1U && !ctx.response_body_truncated); + CHECK(ctx.response_body_bytes_seen == 1048577U); + CHECK(ctx.response_body_bytes_inspected == 1048577U); + } + } else if (strcmp(argv[1], "memory-over-legacy-limit") == 0) { + conf.phase4_body_limit = 65536U; + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + sizeof(payload); + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + append_calls = 0; appended_bytes = 0; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(append_calls == 1 && appended_bytes == sizeof(payload)); + CHECK(read_calls == 0 && allocation_calls == 0); + CHECK(ctx.response_body_bytes_seen == sizeof(payload)); + CHECK(ctx.response_body_bytes_inspected == sizeof(payload)); + CHECK(!ctx.response_body_truncated); + CHECK(memcmp(inspected, payload, sizeof(payload)) == 0); + CHECK(buffer.pos == payload && buffer.last == payload + sizeof(payload)); + } + } else if (strcmp(argv[1], "overflow") == 0) { + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + ctx.response_body_bytes_seen = SIZE_MAX - 1U; + ctx.response_body_bytes_inspected = SIZE_MAX - 1U; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 2U, &allowed) == NGX_OK); + CHECK(allowed == 2U && !ctx.response_body_truncated); + CHECK(ctx.response_body_bytes_seen == SIZE_MAX); + CHECK(ctx.response_body_bytes_inspected == SIZE_MAX - 1U); + CHECK(ngx_http_modsecurity_append_response_body_chunk(&ctx, + payload, 2U) == NGX_OK); + CHECK(ctx.response_body_bytes_inspected == SIZE_MAX); + CHECK(append_calls == (int)i + 1 && ctx.phase4_append_active); + } + } else if (strcmp(argv[1], "invalid-accounting") == 0) { + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + ctx.response_body_bytes_inspected = 1U; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1U, &allowed) == NGX_ERROR); + CHECK(allowed == 0U && ctx.response_body_truncated); + CHECK(ctx.response_body_bytes_seen == 0U); + } + } else if (strcmp(argv[1], "null-empty-mode") == 0) { + CHECK(ngx_http_modsecurity_plan_limited_response_body(NULL, &conf, + 1U, &allowed) == NGX_ERROR && allowed == 0U); + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, NULL, + 1U, &allowed) == NGX_ERROR && allowed == 0U); + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1U, NULL) == NGX_ERROR); + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 0U, &allowed) == NGX_OK && allowed == 0U); + CHECK(!ctx.response_body_seen); + conf.phase4_mode = 99U; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1U, &allowed) == NGX_ERROR && allowed == 0U); + } else if (strcmp(argv[1], "file-chunks") == 0) { + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(read_calls == 3 && append_calls == 3 && read_max == 32768U); + CHECK(allocation_calls == 1 && appended_bytes == sizeof(payload)); + CHECK(memcmp(inspected, payload, sizeof(payload)) == 0); + CHECK(buffer.file_pos == 0 && buffer.file_last == sizeof(payload)); + buffer.file_last = 1; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(allocation_calls == 1 && appended_bytes == sizeof(payload) + 1U); + } else if (strcmp(argv[1], "mixed-over-legacy-limit") == 0) { + conf.phase4_body_limit = 65536U; + buffer.memory = 1; buffer.pos = payload; buffer.last = payload + sizeof(payload); + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + append_calls = 0; appended_bytes = 0; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(appended_bytes == sizeof(payload) && append_calls == 1); + CHECK(read_calls == 0 && allocation_calls == 0); + CHECK(ctx.response_body_bytes_seen == sizeof(payload)); + CHECK(ctx.response_body_bytes_inspected == sizeof(payload)); + CHECK(!ctx.response_body_truncated); + CHECK(memcmp(inspected, payload, sizeof(payload)) == 0); + CHECK(buffer.pos == payload && buffer.last == payload + sizeof(payload)); + CHECK(buffer.file_pos == 0 && buffer.file_last == sizeof(payload)); + } + } else if (strcmp(argv[1], "file-over-legacy-limit") == 0) { + conf.phase4_body_limit = 65536U; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + read_calls = 0; append_calls = 0; allocation_calls = 0; + appended_bytes = 0; read_max = 0; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(read_calls == 3 && append_calls == 3 && read_max == 32768U); + CHECK(allocation_calls == 1 && appended_bytes == sizeof(payload)); + CHECK(ctx.response_body_bytes_seen == sizeof(payload)); + CHECK(ctx.response_body_bytes_inspected == sizeof(payload)); + CHECK(!ctx.response_body_truncated); + CHECK(memcmp(inspected, payload, sizeof(payload)) == 0); + CHECK(buffer.file_pos == 0 && buffer.file_last == sizeof(payload)); + } + } else if (strcmp(argv[1], "file-invalid") == 0) { + buffer.file_pos = -1; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + buffer.file_pos = 10; buffer.file_last = 9; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + buffer.file_pos = 0; buffer.file_last = 10; buffer.file = NULL; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, NULL) == NGX_ERROR); + buffer.memory = 1; buffer.pos = NULL; buffer.last = payload + 10; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + buffer.pos = payload + 10; buffer.last = payload; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(ngx_http_modsecurity_append_response_body_chunk(&ctx, + NULL, 1U) == NGX_ERROR); + CHECK(read_calls == 0 && append_calls == 0); + } else if (strcmp(argv[1], "allocation-failure") == 0) { + fail_allocation = 1; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(read_calls == 0 && appended_bytes == 0U); + } else if (strcmp(argv[1], "short-read") == 0 || strcmp(argv[1], "read-error") == 0) { + short_read = strcmp(argv[1], "short-read") == 0; + fail_read = !short_read; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(read_calls == 1 && append_calls == 0 && appended_bytes == 0U); + } else if (strcmp(argv[1], "append-error") == 0) { + fail_append = 1; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(ctx.response_body_bytes_inspected == 0U); + } + } else if (strcmp(argv[1], "append-zero") == 0) { + append_result = 0; + conf.phase4_body_limit = 1U; + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + append_calls = 0; appended_bytes = 0; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(ctx.response_body_bytes_inspected == 10U && append_calls == 1); + CHECK(!ctx.response_body_truncated); + } + } else if (strcmp(argv[1], "final-once") == 0) { + CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == NGX_OK); + CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == NGX_OK); + CHECK(process_calls == 1 && pcre_enter == 1 && pcre_leave == 1); + CHECK(ctx.phase4_processed); + } else if (strcmp(argv[1], "engine-error") == 0) { + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + request.header_sent = 1; connection.error = 0; engine_result = -1; + CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == NGX_ERROR); + CHECK(ctx.phase4_processed && ctx.intervention_triggered && connection.error); + } + CHECK(process_calls == 3 && pcre_enter == pcre_leave); + } else if (strcmp(argv[1], "process-zero") == 0) { + engine_result = 0; + CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == 500); + CHECK(ctx.intervention_triggered && ctx.phase4_processed); + CHECK(process_calls == 1 && pcre_enter == pcre_leave); + } else if (strcmp(argv[1], "main-eos") == 0 || strcmp(argv[1], "subrequest-eos") == 0) { + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; buffer.last_in_chain = 1; + chain.next = NULL; + if (strcmp(argv[1], "subrequest-eos") == 0) request.main = &main_request; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(appended_bytes == 10U && process_calls == (request.main != &request)); + chain.buf = &final; final.last_buf = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(process_calls == 1 && appended_bytes == 10U); + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(process_calls == 1 && forward_calls == 3); + } else if (strcmp(argv[1], "chain-restored") == 0) { + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; final.last_buf = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(chain.next == &tail && tail.next == NULL); + CHECK(forward_calls == 1 && forwarded_links == 2 && process_calls == 1); + CHECK(appended_bytes == 10U && buffer.pos == payload); + } else if (strcmp(argv[1], "forward-again") == 0) { + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; final.last_buf = 1; + forward_result = NGX_AGAIN; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_AGAIN); + CHECK(chain.next == &tail && forwarded_links == 2 && process_calls == 1); + CHECK(buffer.pos == payload && appended_bytes == 10U); + forward_result = NGX_OK; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(forwarded_links == 4 && appended_bytes == 10U && process_calls == 1); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_OK); + CHECK(process_calls == 1 && chain.next == &tail); + } else if (strcmp(argv[1], "terminal-trailing") == 0) { + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; buffer.last_buf = 1; + final.memory = 1; final.pos = payload + 10; final.last = payload + 20; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(process_calls == 1 && appended_bytes == 10U); + CHECK(chain.next == &tail && forwarded_links == 2); + } else if (strcmp(argv[1], "replaced-body") == 0) { + ctx.response_replaced = 1; + buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 10; + final.last_buf = 1; verify_writer = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(buffer.pos == buffer.last && buffer.in_file); + CHECK(buffer.file_last == buffer.file_pos); + CHECK(append_calls == 0 && process_calls == 0 && forwarded_links == 1); + CHECK(forwarded_controls == 1 && forwarded_eos == 1 && writer_rejected_zero == 0); + CHECK(ctx.replacement_body_terminal_sent && chain.next == &tail); + } else if (strcmp(argv[1], "replaced-data-only") == 0) { + ctx.response_replaced = 1; verify_writer = 1; chain.next = NULL; + buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 10; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(buffer.pos == buffer.last && buffer.file_pos == buffer.file_last); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_OK); + CHECK(forward_calls == 0 && control_link_calls == 0 && writer_rejected_zero == 0); + CHECK(append_calls == 0 && process_calls == 0 && !ctx.replacement_body_terminal_sent); + } else if (strcmp(argv[1], "replaced-buffered-drain") == 0) { + ctx.response_replaced = 1; verify_writer = 1; chain.next = NULL; + request.buffered = 1; forward_result = NGX_AGAIN; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_AGAIN); + CHECK(forward_calls == 1 && forwarded_links == 0 && writer_rejected_zero == 0); + request.buffered = 0; connection.buffered = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_AGAIN); + CHECK(forward_calls == 2 && forwarded_links == 0); + connection.buffered = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_OK); + CHECK(forward_calls == 2 && control_link_calls == 0); + } else if (strcmp(argv[1], "replaced-terminal-again") == 0) { + ctx.response_replaced = 1; verify_writer = 1; chain.next = NULL; + buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 10; + buffer.last_buf = 1; request.buffered = 1; forward_result = NGX_AGAIN; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_AGAIN); + CHECK(ctx.replacement_body_terminal_sent && forwarded_eos == 1); + CHECK(control_link_calls == 1 && forwarded_controls == 1 && writer_rejected_zero == 0); + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_AGAIN); + CHECK(forward_calls == 2 && forwarded_eos == 1 && forwarded_links == 1); + request.buffered = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_OK); + CHECK(forward_calls == 2 && control_link_calls == 1 && chain.next == NULL); + } else if (strcmp(argv[1], "replaced-subrequest-eos") == 0) { + ctx.response_replaced = 1; verify_writer = 1; request.main = &main_request; + chain.next = NULL; buffer.last_in_chain = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(forwarded_controls == 1 && forwarded_eos == 1 && writer_rejected_zero == 0); + CHECK(ctx.replacement_body_terminal_sent && control_buffers[0].sync); + CHECK(!control_buffers[0].last_buf && control_buffers[0].last_in_chain); + } else if (strcmp(argv[1], "replaced-main-chain-marker") == 0) { + ctx.response_replaced = 1; verify_writer = 1; chain.next = NULL; + buffer.last_in_chain = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(!ctx.replacement_body_terminal_sent && forwarded_eos == 0); + CHECK(control_buffers[0].sync && control_buffers[0].last_in_chain); + final.last_buf = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &tail) == NGX_OK); + CHECK(ctx.replacement_body_terminal_sent && forwarded_eos == 1); + CHECK(forwarded_controls == 2 && writer_rejected_zero == 0); + } else if (strcmp(argv[1], "replaced-control-allocation-failure") == 0) { + for (i = 0; i < 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; + ctx.response_replaced = 1; request.header_sent = 1; connection.error = 0; + buffer.file_pos = 0; buffer.last_buf = 1; chain.next = NULL; + fail_control_link = i == 0; fail_control_buffer = i != 0; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(ctx.phase4_failed && connection.error && forward_calls == 0); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_ERROR); + } + } else if (strcmp(argv[1], "counter-saturation-stream") == 0) { + chain.next = NULL; request.header_sent = 1; + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; + buffer.pos = payload; buffer.last = payload + 10; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + ctx.response_body_bytes_seen = SIZE_MAX - 1U; + ctx.response_body_bytes_inspected = SIZE_MAX - 1U; + append_calls = 0; appended_bytes = 0; process_calls = 0; forward_calls = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(ctx.response_body_bytes_seen == SIZE_MAX && ctx.response_body_bytes_inspected == SIZE_MAX); + CHECK(append_calls == 1 && appended_bytes == 10U && process_calls == 1 && forward_calls == 1); + CHECK(!ctx.phase4_failed && !ctx.response_body_truncated && !connection.error); + } + } else if (strcmp(argv[1], "large-file-first-chunk") == 0) { + if (sizeof(off_t) < 8) return 0; + buffer.file_last = (off_t)UINT32_MAX + (off_t)sizeof(payload); + native_result = 403; native_failed = 1; native_body_limit = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == 403); + CHECK(read_calls == 1 && append_calls == 1 && appended_bytes == 32768U); + CHECK(ctx.intervention_body_limit && process_calls == 0 && forward_calls == 0); + } else if (strcmp(argv[1], "append-boundary") == 0) { + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; + buffer.pos = payload; buffer.last = payload + 10; chain.next = NULL; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(append_collections == 1 && eos_with_append_active == 0); + CHECK(!ctx.phase4_append_active && ctx.phase4_processed && forward_calls == 1); + } else if (strcmp(argv[1], "empty-memory-eos") == 0) { + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; + chain.next = NULL; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; + conf.phase4_mode = (unsigned)i; + process_calls = 0; forward_calls = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(process_calls == 1 && append_calls == 0 && forward_calls == 1); + CHECK(ctx.phase4_processed && !ctx.phase4_failed); + CHECK(ctx.response_body_bytes_seen == 0U && !ctx.response_body_seen); + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(process_calls == 1 && forward_calls == 2); + } + } else if (strcmp(argv[1], "memory-missing-end") == 0) { + buffer.in_file = 0; buffer.memory = 1; buffer.pos = payload; + request.header_sent = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(append_calls == 0 && forward_calls == 0 && ctx.phase4_failed); + } else if (strcmp(argv[1], "invalid-append-results") == 0) { + const int invalid[] = {-2, -1, 2, 99}; + size_t invalid_index; + request.header_sent = 1; chain.next = NULL; + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; + for (i = 0; i <= 2; ++i) { + conf.phase4_mode = (unsigned)i; + for (invalid_index = 0; invalid_index < sizeof(invalid) / sizeof(invalid[0]); ++invalid_index) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; + append_calls = 0; appended_bytes = 0; connection.error = 0; + append_result = invalid[invalid_index]; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(append_calls == 1 && process_calls == 0 && forward_calls == 0); + CHECK(ctx.phase4_failed && connection.error); + } + } + } else if (strcmp(argv[1], "failure-reentry") == 0) { + request.header_sent = 1; chain.next = NULL; + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; + buffer.pos = payload; buffer.last = payload + 10; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; + conf.phase4_mode = (unsigned)i; + process_calls = 0; append_calls = 0; appended_bytes = 0; + engine_result = 0; connection.error = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(ctx.phase4_failed && connection.error); + engine_result = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_ERROR); + CHECK(process_calls == 1 && append_calls == 1 && forward_calls == 0); + } + } else if (strcmp(argv[1], "strict-reentry") == 0 || + strcmp(argv[1], "sink-failure-reentry") == 0) { + request.header_sent = 1; chain.next = NULL; + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; + buffer.pos = payload; buffer.last = payload + 10; + native_result = -1; native_late = 1; ctx.last_intervention_status = 403; + native_only_after_eos = 1; + if (strcmp(argv[1], "strict-reentry") == 0) conf.phase4_mode = 2; + else log_result = NGX_ERROR; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == + (conf.phase4_mode == 2 ? NGX_ERROR : NGX_OK)); + CHECK(ctx.phase4_failed == (conf.phase4_mode == 2)); + CHECK(connection.error == (conf.phase4_mode == 2) && event_calls == 1); + native_result = 0; log_result = NGX_OK; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == + (conf.phase4_mode == 2 ? NGX_ERROR : NGX_OK)); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == + (conf.phase4_mode == 2 ? NGX_ERROR : NGX_OK)); + CHECK(append_calls == 1 && process_calls == 1); + CHECK(forward_calls == (conf.phase4_mode == 2 ? 0 : 3)); + CHECK(event_calls == 1); + } else if (strcmp(argv[1], "core-error-once") == 0) { + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; + buffer.pos = payload; buffer.last = payload + 10; + chain.next = NULL; final.last_buf = 1; + engine_result = 0; emulate_core_response = 1; core_response = &tail; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == 500); + CHECK(finalize_calls == 1 && core_response_result == NGX_OK); + CHECK(process_calls == 1 && append_calls == 1 && forward_calls == 1); + CHECK(ctx.phase4_failed && !ctx.phase4_error_emitting); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_ERROR); + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(finalize_calls == 1 && forward_calls == 1); + } else if (strcmp(argv[1], "file-chunk-rejection") == 0) { + request.header_sent = 1; + native_result = 403; native_failed = 1; native_body_limit = 1; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; + conf.phase4_mode = (unsigned)i; + read_calls = 0; append_calls = 0; appended_bytes = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(ctx.phase4_failed && ctx.intervention_body_limit); + CHECK(read_calls == 1 && append_calls == 1 && appended_bytes == 32768U); + CHECK(process_calls == 0 && forward_calls == 0 && event_calls == 0); + } + } else if (strcmp(argv[1], "late-safe") == 0 || strcmp(argv[1], "late-strict") == 0 || strcmp(argv[1], "late-off") == 0) { + request.header_sent = 1; native_result = -1; native_late = 1; + ctx.last_intervention_status = 403; + if (strcmp(argv[1], "late-off") == 0) conf.phase4_mode = 0; + if (strcmp(argv[1], "late-strict") == 0) conf.phase4_mode = 2; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == + (conf.phase4_mode == 1 ? NGX_OK : NGX_ERROR)); + CHECK(event_calls == (conf.phase4_mode != 0)); + CHECK(ctx.intervention_triggered == (conf.phase4_mode != 1)); + if (conf.phase4_mode == 1) CHECK(strcmp(last_action, "log_only") == 0); + if (conf.phase4_mode == 2) { + CHECK(strcmp(last_action, "connection_abort") == 0); + CHECK(ctx.phase4_strict_abort && connection.error); + } + } else if (strcmp(argv[1], "native-failure-safe") == 0) { + request.header_sent = 1; native_result = 500; native_failed = 1; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == NGX_ERROR); + CHECK(connection.error && ctx.intervention_triggered && event_calls == 0); + } else if (strcmp(argv[1], "native-failure-uncommitted") == 0) { + native_result = 500; native_failed = 1; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == 500); + CHECK(ctx.intervention_triggered && event_calls == 0); + } else if (strcmp(argv[1], "uncommitted-deny") == 0) { + const ngx_int_t statuses[] = {403, 302}; + for (size_t i = 0; i < sizeof(statuses) / sizeof(statuses[0]); i++) { + ctx.phase4_headers_checked = 0; ctx.intervention_triggered = 0; + event_calls = 0; native_result = statuses[i]; + ctx.last_intervention_status = statuses[i]; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == statuses[i]); + CHECK(ctx.intervention_triggered && event_calls == 1); + CHECK(strcmp(last_action, "deny_status") == 0); + CHECK(strcmp(last_reason, "response_not_committed") == 0); + } + } else if (strcmp(argv[1], "log-failure-safe") == 0) { + request.header_sent = 1; native_result = -1; native_late = 1; log_result = NGX_ERROR; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == NGX_OK); + CHECK(!ctx.intervention_triggered && !ctx.phase4_failed && !connection.error); + CHECK(event_calls == 1 && diagnostic_calls == 1); + } else if (strcmp(argv[1], "log-failure-strict") == 0) { + conf.phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_STRICT; + request.header_sent = 1; native_result = -1; native_late = 1; log_result = NGX_ERROR; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == NGX_ERROR); + CHECK(ctx.phase4_failed && ctx.phase4_strict_abort && connection.error && event_calls == 1); + } else { + return 2; + } + return 0; +} +""" + + +def unit_program(body: str, module: str) -> str: + functions = ( + "response_body_failure", "plan_limited_response_body", + "append_response_body_chunk", "append_limited_response_body", + "append_file_response_body", "append_response_body_buffer", + "phase4_handle_intervention", "process_response_intervention", + "process_final_response_body", + ) + definitions = [ + "\nstatic ngx_int_t\n" + function_definition(body, "ngx_http_modsecurity_" + name) + + "\n" for name in functions + ] + gate = "ngx_http_modsecurity_phase4_error_gate" + if re.search(rf"\b{gate}\s*\(", body): + definitions.insert(0, "\nstatic ngx_int_t\n" + function_definition(body, gate) + "\n") + discard_type = "void" if re.search( + r"static\s+void\s+ngx_http_modsecurity_discard_replaced_response_body\s*\(", body + ) else "ngx_int_t" + definitions.append("\nstatic " + discard_type + "\n" + function_definition( + body, "ngx_http_modsecurity_discard_replaced_response_body" + )) + definitions.append("\nngx_int_t\n" + function_definition( + body, "ngx_http_modsecurity_body_filter" + )) + declarations = "\n".join(definition[:definition.index("{")] + ";" for definition in definitions) + context_lookup = "".join( + "\nstatic ngx_int_t\n" + function_definition(module, name) + "\n" + for name in ("ngx_http_modsecurity_same_policy", "ngx_http_modsecurity_is_block_error_page") + ) + context_lookup += "\nstatic ngx_http_modsecurity_ctx_t *\n" + function_definition( + module, "ngx_http_modsecurity_get_module_ctx" + ) + "\n" + return PREAMBLE + declarations + context_lookup + "".join(definitions) + CASES + + +class Phase4RuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler or shutil.which(compiler[0]) is None: + raise RuntimeError("set CC to a GCC- or Clang-compatible C17 compiler") + directory = tempfile.TemporaryDirectory(prefix="nginx-phase4-") + cls.addClassCleanup(directory.cleanup) + source = Path(directory.name) / "phase4.c" + cls.binary = Path(directory.name) / ("phase4.exe" if os.name == "nt" else "phase4") + source.write_text(unit_program(BODY.read_text(encoding="utf-8"), MODULE.read_text(encoding="utf-8")), encoding="utf-8") + build = subprocess.run( + [*compiler, "-std=c17", "-Wall", "-Wextra", "-Werror", + str(source), "-o", str(cls.binary)], + capture_output=True, text=True, timeout=60, check=False, + ) + if build.returncode: + raise AssertionError("Phase4 helper compilation failed:\n" + build.stderr) + + def run_case(self, name: str) -> None: + result = subprocess.run( + [str(self.binary), name], capture_output=True, text=True, + timeout=10, check=False, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +def case_test(name: str): + def test(self: Phase4RuntimeTests) -> None: + self.run_case(name) + return test + + +for case in ( + "request-ownership", "all-modes-large", "all-modes-multiple", "legacy-boundary", "memory-over-legacy-limit", + "overflow", "invalid-accounting", "null-empty-mode", "file-chunks", + "mixed-over-legacy-limit", "file-over-legacy-limit", "file-invalid", "allocation-failure", + "short-read", "read-error", "append-error", "append-zero", "final-once", + "engine-error", "process-zero", + "main-eos", "subrequest-eos", "chain-restored", "forward-again", + "terminal-trailing", "replaced-body", + "replaced-data-only", "replaced-buffered-drain", "replaced-terminal-again", + "replaced-subrequest-eos", "replaced-main-chain-marker", "replaced-control-allocation-failure", + "counter-saturation-stream", "large-file-first-chunk", "append-boundary", + "empty-memory-eos", "memory-missing-end", "invalid-append-results", + "failure-reentry", "strict-reentry", "sink-failure-reentry", + "core-error-once", "file-chunk-rejection", + "late-safe", "late-strict", "late-off", "native-failure-safe", + "native-failure-uncommitted", "uncommitted-deny", "log-failure-safe", "log-failure-strict", +): + setattr(Phase4RuntimeTests, "test_" + case.replace("-", "_"), case_test(case)) + + +if __name__ == "__main__": + unittest.main()