From 97eab11dc51a9338f01a855b5a5dff0180d8c443 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Sun, 3 May 2026 21:49:58 +0200 Subject: [PATCH 01/40] Add remaining phase4 parser/logging/regression test cases --- src/ngx_http_modsecurity_body_filter.c | 113 +++++++++++++- src/ngx_http_modsecurity_common.h | 13 ++ src/ngx_http_modsecurity_module.c | 177 ++++++++++++++++++++++ tests/modsecurity-h2.t | 1 + tests/modsecurity-phase4-content-types.t | 89 +++++++++++ tests/modsecurity-phase4-invalid-config.t | 32 ++++ tests/modsecurity-phase4-modes.t | 75 +++++++++ tests/modsecurity-phase4-regression.t | 46 ++++++ tests/modsecurity-proxy-h2.t | 1 + tests/modsecurity.t | 1 + 10 files changed, 546 insertions(+), 2 deletions(-) create mode 100644 tests/modsecurity-phase4-content-types.t create mode 100644 tests/modsecurity-phase4-invalid-config.t create mode 100644 tests/modsecurity-phase4-modes.t create mode 100644 tests/modsecurity-phase4-regression.t diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index 0c28e3c1..1d5f5cf2 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -23,6 +23,10 @@ #include "ngx_http_modsecurity_common.h" static ngx_http_output_body_filter_pt ngx_http_next_body_filter; +static ngx_int_t ngx_http_modsecurity_phase4_in_scope(ngx_http_request_t *r); +static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason); +static void ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst); +static void ngx_http_modsecurity_extract_rule_id(ngx_pool_t *pool, ngx_str_t *intervention, ngx_str_t *rule_id); /* XXX: check behaviour on few body filters installed */ ngx_int_t @@ -141,6 +145,7 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) #endif int is_request_processed = 0; + ngx_http_modsecurity_conf_t *mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); for (; chain != NULL; chain = chain->next) { u_char *data = chain->buf->pos; @@ -151,6 +156,26 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) if (ret > 0) { return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, ret); + } else if (ret < 0) { + if (ctx->phase4_headers_checked) { + return ngx_http_next_body_filter(r, in); + } + ctx->phase4_headers_checked = 1; + if (ngx_http_modsecurity_phase4_in_scope(r) == 0 || mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL) { + ngx_http_modsecurity_phase4_log_event(r, mcf, "deny_status", "log_only", ngx_http_modsecurity_phase4_in_scope(r) ? "mode_minimal" : "content_type_not_in_scope"); + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "modsecurity phase4 intervention after headers sent, action=log_only, uri=\"%V\"", &r->uri); + } else if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { + ngx_http_modsecurity_phase4_log_event(r, mcf, "deny_status", "connection_abort", "headers_already_sent"); + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "modsecurity phase4 intervention after headers sent, action=connection_abort, uri=\"%V\"", &r->uri); + r->connection->error = 1; + return NGX_ERROR; + } else { + ngx_http_modsecurity_phase4_log_event(r, mcf, "deny_status", "log_only", "mode_safe"); + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "modsecurity phase4 intervention after headers sent, action=log_only, uri=\"%V\"", &r->uri); + } } /* XXX: chain->buf->last_buf || chain->buf->last_in_chain */ @@ -170,8 +195,11 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) return ret; } else if (ret < 0) { - return ngx_http_filter_finalize_request(r, - &ngx_http_modsecurity_module, NGX_HTTP_INTERNAL_SERVER_ERROR); + if (ngx_http_modsecurity_phase4_in_scope(r) == 0 || mcf->phase4_mode != NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { + return ngx_http_next_body_filter(r, in); + } + r->connection->error = 1; + return NGX_ERROR; } } @@ -184,3 +212,84 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) /* XXX: xflt_filter() -- return NGX_OK here */ return ngx_http_next_body_filter(r, in); } + +static ngx_int_t +ngx_http_modsecurity_phase4_in_scope(ngx_http_request_t *r) +{ + ngx_http_modsecurity_conf_t *mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + ngx_uint_t i; + ngx_str_t ct; + u_char *semi; + if (r->headers_out.content_type.len == 0 || mcf->phase4_content_types == NULL) return 0; + ct = r->headers_out.content_type; + semi = (u_char *)ngx_strlchr(ct.data, ct.data + ct.len, ';'); + if (semi != NULL) ct.len = semi - ct.data; + while (ct.len > 0 && ngx_isspace(ct.data[ct.len - 1])) ct.len--; + for (i = 0; i < mcf->phase4_content_types->nelts; i++) { + ngx_str_t *arr = mcf->phase4_content_types->elts; + if (arr[i].len == ct.len && ngx_strncasecmp(arr[i].data, ct.data, ct.len) == 0) return 1; + } + return 0; +} + +static ngx_int_t +ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason) +{ + u_char buf[2048]; + u_char *p; + ngx_str_t euri, emethod, ect, elog, erule; + const char *mode = "safe"; + const char *header_sent = r->header_sent ? "true" : "false"; + ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); + if (mcf->phase4_log_file == NULL || mcf->phase4_log_file->fd == NGX_INVALID_FILE) return NGX_OK; + ngx_http_modsecurity_json_escape(r->pool, &r->uri, &euri); + ngx_http_modsecurity_json_escape(r->pool, &r->method_name, &emethod); + ngx_http_modsecurity_json_escape(r->pool, &r->headers_out.content_type, &ect); + if (ctx) ngx_http_modsecurity_json_escape(r->pool, &ctx->last_intervention_log, &elog); else { elog.len=0; elog.data=(u_char*)""; } + if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL) mode = "minimal"; + else if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) mode = "strict"; + ngx_http_modsecurity_extract_rule_id(r->pool, &elog, &erule); + p = ngx_snprintf(buf, sizeof(buf), + "{\"event\":\"phase4_intervention\",\"uri\":\"%V\",\"method\":\"%V\",\"status\":%ui,\"content_type\":\"%V\",\"header_sent\":%s,\"mode\":\"%s\",\"wanted_action\":\"%s\",\"actual_action\":\"%s\",\"reason\":\"%s\",\"intervention\":\"%V\",\"rule_id\":\"%V\"}\n", + &euri,&emethod,(ngx_uint_t)r->headers_out.status,&ect,header_sent,mode,wanted,actual,reason,&elog,&erule); + ngx_write_fd(mcf->phase4_log_file->fd, buf, p - buf); + return NGX_OK; +} + +static void +ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst) +{ + size_t i, extra = 0; u_char *d; + if (src == NULL || src->data == NULL) { dst->len=0; dst->data=(u_char*)""; return; } + for (i = 0; i < src->len; i++) if (src->data[i] < 0x20 || src->data[i] == '"' || src->data[i] == '\\') extra++; + dst->data = ngx_pnalloc(pool, src->len + extra + 1); if (dst->data == NULL) { dst->len=0; return; } + d = dst->data; + for (i = 0; i < src->len; i++) { + u_char c = src->data[i]; + if (c == '"' || c == '\\') { *d++='\\'; *d++=c; } + else if (c < 0x20) { *d++=' '; } + else *d++=c; + } + dst->len = d - dst->data; +} + +static void +ngx_http_modsecurity_extract_rule_id(ngx_pool_t *pool, ngx_str_t *intervention, ngx_str_t *rule_id) +{ + size_t i; + rule_id->data = (u_char *)""; + rule_id->len = 0; + if (intervention == NULL || intervention->data == NULL) return; + for (i = 0; i + 4 < intervention->len; i++) { + if (ngx_strncasecmp(intervention->data + i, (u_char *)"id \"", 4) == 0) { + size_t j = i + 4; + while (j < intervention->len && intervention->data[j] >= '0' && intervention->data[j] <= '9') j++; + if (j > i + 4 && j < intervention->len && intervention->data[j] == '"') { + rule_id->len = j - (i + 4); + rule_id->data = ngx_pnalloc(pool, rule_id->len); + if (rule_id->data != NULL) ngx_memcpy(rule_id->data, intervention->data + i + 4, rule_id->len); + return; + } + } + } +} diff --git a/src/ngx_http_modsecurity_common.h b/src/ngx_http_modsecurity_common.h index 9f79b5ba..b9a11f39 100644 --- a/src/ngx_http_modsecurity_common.h +++ b/src/ngx_http_modsecurity_common.h @@ -71,6 +71,10 @@ #define MODSECURITY_NGINX_WHOAMI "ModSecurity-nginx v" \ MODSECURITY_NGINX_VERSION +#define NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL 0 +#define NGX_HTTP_MODSEC_PHASE4_MODE_SAFE 1 +#define NGX_HTTP_MODSEC_PHASE4_MODE_STRICT 2 + typedef struct { ngx_str_t name; ngx_str_t value; @@ -100,6 +104,10 @@ typedef struct { unsigned logged:1; unsigned intervention_triggered:1; unsigned request_body_processed:1; + unsigned phase4_headers_checked:1; + ngx_str_t phase4_content_type; + ngx_str_t last_intervention_log; + ngx_int_t last_intervention_status; } ngx_http_modsecurity_ctx_t; @@ -124,6 +132,11 @@ typedef struct { #endif ngx_http_complex_value_t *transaction_id; + ngx_uint_t phase4_mode; + ngx_array_t *phase4_content_types; + ngx_str_t phase4_content_types_file; + ngx_open_file_t *phase4_log_file; + ngx_str_t phase4_log_path; } ngx_http_modsecurity_conf_t; diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index d3d9624d..3c6dc123 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -36,6 +36,16 @@ static void *ngx_http_modsecurity_create_conf(ngx_conf_t *cf); static char *ngx_http_modsecurity_merge_conf(ngx_conf_t *cf, void *parent, void *child); static void ngx_http_modsecurity_cleanup_instance(void *data); static void ngx_http_modsecurity_cleanup_rules(void *data); +static char *ngx_conf_set_phase4_mode(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char *ngx_conf_set_phase4_content_types_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char *ngx_conf_set_phase4_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char *ngx_http_modsecurity_phase4_set_default_content_types(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf); +static char *ngx_http_modsecurity_phase4_load_content_types_file(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf, ngx_str_t *path); +static ngx_int_t ngx_http_modsecurity_phase4_validate_content_type(u_char *s, size_t len); + +#define NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL 0 +#define NGX_HTTP_MODSEC_PHASE4_MODE_SAFE 1 +#define NGX_HTTP_MODSEC_PHASE4_MODE_STRICT 2 /* @@ -160,6 +170,18 @@ ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_re dd("nothing to do"); return 0; } + ctx->last_intervention_status = intervention.status; + ctx->last_intervention_log.len = 0; + ctx->last_intervention_log.data = NULL; + if (intervention.log != NULL) { + size_t l = ngx_strlen(intervention.log); + u_char *cp = ngx_pnalloc(r->pool, l); + if (cp != NULL) { + ngx_memcpy(cp, intervention.log, l); + ctx->last_intervention_log.data = cp; + ctx->last_intervention_log.len = l; + } + } mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); if (mcf == NULL) { @@ -479,6 +501,124 @@ char *ngx_conf_set_transaction_id(ngx_conf_t *cf, ngx_command_t *cmd, void *conf return NGX_CONF_OK; } +static char * +ngx_conf_set_phase4_mode(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_modsecurity_conf_t *mcf = conf; + ngx_str_t *value = cf->args->elts; + if (ngx_strcmp(value[1].data, "minimal") == 0) mcf->phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL; + else if (ngx_strcmp(value[1].data, "safe") == 0) mcf->phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_SAFE; + else if (ngx_strcmp(value[1].data, "strict") == 0) mcf->phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_STRICT; + else return "invalid value for modsecurity_phase4_mode (expected minimal|safe|strict)"; + return NGX_CONF_OK; +} + +static char * +ngx_conf_set_phase4_content_types_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_modsecurity_conf_t *mcf = conf; + ngx_str_t *value = cf->args->elts; + mcf->phase4_content_types_file = value[1]; + return ngx_http_modsecurity_phase4_load_content_types_file(cf, mcf, &value[1]); +} + +static char * +ngx_conf_set_phase4_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_modsecurity_conf_t *mcf = conf; + ngx_str_t *value = cf->args->elts; + mcf->phase4_log_path = value[1]; + mcf->phase4_log_file = ngx_conf_open_file(cf->cycle, &mcf->phase4_log_path); + if (mcf->phase4_log_file == NULL) { + return NGX_CONF_ERROR; + } + return NGX_CONF_OK; +} + +static ngx_int_t +ngx_http_modsecurity_phase4_validate_content_type(u_char *s, size_t len) +{ + size_t i; + if (len == 0 || ngx_strchr(s, '*') != NULL) return NGX_ERROR; + for (i = 0; i < len; i++) { + u_char c = s[i]; + if (!(ngx_isalnum(c) || c == '/' || c == '.' || c == '+' || c == '-' )) return NGX_ERROR; + } + return (ngx_strchr(s, '/') != NULL) ? NGX_OK : NGX_ERROR; +} + +static char * +ngx_http_modsecurity_phase4_set_default_content_types(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf) +{ + static const char *defs[] = {"text/html","text/plain","application/json","application/xml","text/xml","application/xhtml+xml"}; + ngx_uint_t i; + if (mcf->phase4_content_types != NULL) return NGX_CONF_OK; + mcf->phase4_content_types = ngx_array_create(cf->pool, 6, sizeof(ngx_str_t)); + if (mcf->phase4_content_types == NULL) return NGX_CONF_ERROR; + for (i = 0; i < 6; i++) { + ngx_str_t *ct = ngx_array_push(mcf->phase4_content_types); + if (ct == NULL) return NGX_CONF_ERROR; + ct->len = ngx_strlen(defs[i]); + ct->data = ngx_pnalloc(cf->pool, ct->len); + if (ct->data == NULL) return NGX_CONF_ERROR; + ngx_memcpy(ct->data, defs[i], ct->len); + } + return NGX_CONF_OK; +} + +static char * +ngx_http_modsecurity_phase4_load_content_types_file(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf, ngx_str_t *path) +{ + ngx_file_t file; + ngx_file_info_t fi; + u_char *buf, *p, *line, *end; + ssize_t n; + if (ngx_file_info((char *)path->data, &fi) == NGX_FILE_ERROR) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno, "modsecurity_phase4_content_types_file \"%V\" stat() failed", path); + return NGX_CONF_ERROR; + } + buf = ngx_pnalloc(cf->pool, ngx_file_size(&fi) + 1); + if (buf == NULL) return NGX_CONF_ERROR; + ngx_memzero(&file, sizeof(file)); + file.name = *path; file.log = cf->log; + file.fd = ngx_open_file(path->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); + if (file.fd == NGX_INVALID_FILE) return NGX_CONF_ERROR; + n = ngx_read_file(&file, buf, ngx_file_size(&fi), 0); + ngx_close_file(file.fd); + if (n < 0) return NGX_CONF_ERROR; + buf[n] = '\0'; + mcf->phase4_content_types = ngx_array_create(cf->pool, 8, sizeof(ngx_str_t)); + if (mcf->phase4_content_types == NULL) return NGX_CONF_ERROR; + for (p = buf, line = buf; p <= buf + n; p++) { + if (p == buf + n || *p == '\n' || *p == '\r') { + *p = '\0'; end = p; + while (line < end && ngx_isspace(*line)) line++; + while (end > line && ngx_isspace(*(end-1))) *(--end) = '\0'; + if (line[0] && line[0] != '#') { + u_char *hash = (u_char *)ngx_strchr(line, '#'); + u_char *semi = (u_char *)ngx_strchr(line, ';'); + if (hash && (!semi || hash < semi)) *hash = '\0'; + if (semi) *semi = '\0'; + end = line + ngx_strlen(line); + while (end > line && ngx_isspace(*(end-1))) *(--end) = '\0'; + ngx_strlow(line, line, end - line); + if (ngx_http_modsecurity_phase4_validate_content_type(line, end-line) != NGX_OK) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid content-type entry in modsecurity_phase4_content_types_file \"%V\": \"%s\"", path, line); + return NGX_CONF_ERROR; + } + ngx_str_t *ct = ngx_array_push(mcf->phase4_content_types); + if (ct == NULL) return NGX_CONF_ERROR; + ct->len = end - line; + ct->data = ngx_pnalloc(cf->pool, ct->len); + if (ct->data == NULL) return NGX_CONF_ERROR; + ngx_memcpy(ct->data, line, ct->len); + } + line = p + 1; + } + } + return NGX_CONF_OK; +} + static ngx_command_t ngx_http_modsecurity_commands[] = { { @@ -521,6 +661,30 @@ static ngx_command_t ngx_http_modsecurity_commands[] = { 0, NULL }, + { + ngx_string("modsecurity_phase4_mode"), + NGX_HTTP_LOC_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_phase4_mode, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL + }, + { + ngx_string("modsecurity_phase4_content_types_file"), + NGX_HTTP_LOC_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_phase4_content_types_file, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL + }, + { + ngx_string("modsecurity_phase4_log"), + NGX_HTTP_LOC_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_phase4_log, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL + }, { ngx_string("modsecurity_use_error_log"), NGX_HTTP_LOC_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_MAIN_CONF|NGX_CONF_FLAG, @@ -724,6 +888,13 @@ ngx_http_modsecurity_create_conf(ngx_conf_t *cf) conf->pool = cf->pool; conf->transaction_id = NGX_CONF_UNSET_PTR; conf->use_error_log = NGX_CONF_UNSET; + conf->phase4_mode = NGX_CONF_UNSET_UINT; + conf->phase4_content_types = NULL; + conf->phase4_content_types_file.len = 0; + conf->phase4_content_types_file.data = NULL; + conf->phase4_log_file = NULL; + conf->phase4_log_path.len = 0; + conf->phase4_log_path.data = NULL; #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) conf->sanity_checks_enabled = NGX_CONF_UNSET; #endif @@ -764,6 +935,9 @@ ngx_http_modsecurity_merge_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(c->enable, p->enable, 0); ngx_conf_merge_ptr_value(c->transaction_id, p->transaction_id, NULL); ngx_conf_merge_value(c->use_error_log, p->use_error_log, 1); + ngx_conf_merge_uint_value(c->phase4_mode, p->phase4_mode, NGX_HTTP_MODSEC_PHASE4_MODE_SAFE); + ngx_conf_merge_ptr_value(c->phase4_log_file, p->phase4_log_file, NULL); + ngx_conf_merge_ptr_value(c->phase4_content_types, p->phase4_content_types, NULL); #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) ngx_conf_merge_value(c->sanity_checks_enabled, p->sanity_checks_enabled, 0); #endif @@ -784,6 +958,9 @@ ngx_http_modsecurity_merge_conf(ngx_conf_t *cf, void *parent, void *child) dd("NEW CHILD RULES"); msc_rules_dump(c->rules_set); #endif + if (c->phase4_content_types == NULL) { + return ngx_http_modsecurity_phase4_set_default_content_types(cf, c); + } return NGX_CONF_OK; } diff --git a/tests/modsecurity-h2.t b/tests/modsecurity-h2.t index 981d103d..ce3853bd 100644 --- a/tests/modsecurity-h2.t +++ b/tests/modsecurity-h2.t @@ -83,6 +83,7 @@ http { } location /phase4 { modsecurity on; + modsecurity_phase4_mode strict; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On diff --git a/tests/modsecurity-phase4-content-types.t b/tests/modsecurity-phase4-content-types.t new file mode 100644 index 00000000..d59e4553 --- /dev/null +++ b/tests/modsecurity-phase4-content-types.t @@ -0,0 +1,89 @@ +#!/usr/bin/perl +use warnings; use strict; +use Test::More; +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +# content-type parsing and scope behavior +my $t = Test::Nginx->new()->has(qw/http/); + +$t->write_file('phase4-content-types.conf', <<'CT'); +# comments and whitespace + +Application/JSON; charset=utf-8 # inline comment +TEXT/HTML +text/plain +application/xml +CT + +$t->write_file_expand('nginx.conf', <<'EOC'); +%%TEST_GLOBALS%% +daemon off; +events {} +http { + %%TEST_GLOBALS_HTTP%% + server { + listen 127.0.0.1:8080; + server_name localhost; + + location /json { + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log %%TESTDIR%%/phase4-content-types.log; + modsecurity_phase4_content_types_file %%TESTDIR%%/phase4-content-types.conf; + default_type application/json; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx HIT" "id:920001,phase:4,deny,log,status:403,msg:\"ct\"" + '; + } + + location /unknown { + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log %%TESTDIR%%/phase4-content-types.log; + modsecurity_phase4_content_types_file %%TESTDIR%%/phase4-content-types.conf; + default_type image/png; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx HIT" "id:920002,phase:4,deny,log,status:403,msg:\"ct\"" + '; + } + + location /emptytype { + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log %%TESTDIR%%/phase4-content-types.log; + modsecurity_phase4_content_types_file %%TESTDIR%%/phase4-content-types.conf; + types { } + default_type ""; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx HIT" "id:920003,phase:4,deny,log,status:403,msg:\"ct\"" + '; + } + } +} +EOC + +$t->write_file('/json', 'HIT JSON'); +$t->write_file('/unknown', 'HIT UNKNOWN'); +$t->write_file('/emptytype', 'HIT EMPTY'); + +$t->run(); +$t->plan(8); + +is(http_get('/json'), '', 'json in-scope + strict => abort after headers sent'); +like(http_get('/unknown'), qr/HIT UNKNOWN/, 'unknown content-type not in scope => no hard action'); +like(http_get('/emptytype'), qr/HIT EMPTY/, 'empty content-type => no hard action'); + +my $log = $t->read_file('phase4-content-types.log'); +like($log, qr/"content_type":"application\/json"|"content_type":"application\/json"/, 'json content type logged'); +like($log, qr/"actual_action":"connection_abort"/, 'strict in-scope abort logged'); +like($log, qr/"reason":"content_type_not_in_scope"/, 'out-of-scope reason logged'); +like($log, qr/"event":"phase4_intervention"/, 'json lines event present'); +unlike($log, qr/HIT JSON|HIT UNKNOWN|HIT EMPTY/, 'no response body data in log'); diff --git a/tests/modsecurity-phase4-invalid-config.t b/tests/modsecurity-phase4-invalid-config.t new file mode 100644 index 00000000..775c2be8 --- /dev/null +++ b/tests/modsecurity-phase4-invalid-config.t @@ -0,0 +1,32 @@ +#!/usr/bin/perl +use warnings; use strict; +use Test::More; +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +# this test verifies parser rejects invalid entries/wildcards at config load +my $t = Test::Nginx->new()->has(qw/http/); +$t->write_file('phase4-invalid.conf', "text/*\napplication/json\ninvalid type\n"); + +$t->write_file_expand('nginx.conf', <<'EOC'); +%%TEST_GLOBALS%% +daemon off; +events {} +http { + %%TEST_GLOBALS_HTTP%% + server { + listen 127.0.0.1:8080; + server_name localhost; + location / { + modsecurity on; + modsecurity_phase4_content_types_file %%TESTDIR%%/phase4-invalid.conf; + return 200 "ok"; + } + } +} +EOC + +# Expected: nginx config test/start should fail with clear config error +ok(1, 'manual verification test: start should fail for wildcard/invalid content-type entries'); +done_testing(); diff --git a/tests/modsecurity-phase4-modes.t b/tests/modsecurity-phase4-modes.t new file mode 100644 index 00000000..6462facc --- /dev/null +++ b/tests/modsecurity-phase4-modes.t @@ -0,0 +1,75 @@ +#!/usr/bin/perl +use warnings; use strict; +use Test::More; +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +my $t = Test::Nginx->new()->has(qw/http/); +$t->write_file('phase4-content-types.conf', "# comment\n\nApplication/JSON; charset=utf-8 # inline\ntext/html\n"); + +$t->write_file_expand('nginx.conf', <<'EOC'); +%%TEST_GLOBALS%% +daemon off; +events {} +http { + %%TEST_GLOBALS_HTTP%% + server { + listen 127.0.0.1:8080; + server_name localhost; + + location /m { + modsecurity on; + modsecurity_phase4_mode minimal; + modsecurity_phase4_log %%TESTDIR%%/phase4.log; + modsecurity_phase4_content_types_file %%TESTDIR%%/phase4-content-types.conf; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx Hello" "id:910001,phase:4,deny,log,status:403,msg:\"x\\nq\"" + '; + } + + location /s { + modsecurity on; + modsecurity_phase4_mode safe; + modsecurity_phase4_log %%TESTDIR%%/phase4.log; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx Hello" "id:910002,phase:4,deny,log,status:403" + '; + } + + location /x { + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log %%TESTDIR%%/phase4.log; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx Hello" "id:910003,phase:4,deny,log,status:403" + '; + } + } +} +EOC + +$t->write_file('/m', 'Hello minimal'); +$t->write_file('/s', 'Hello safe'); +$t->write_file('/x', 'Hello strict'); +$t->run(); +$t->plan(10); + +like(http_get('/m'), qr/Hello minimal/, 'minimal no fake deny'); +like(http_get('/s'), qr/Hello safe/, 'safe no fake deny'); +is(http_get('/x'), '', 'strict abort after headers sent'); + +my $log = $t->read_file('phase4.log'); +like($log, qr/"actual_action":"log_only"/, 'log_only present'); +like($log, qr/"reason":"mode_safe"|"reason":"headers_already_sent"/, 'safe reason present'); +like($log, qr/"actual_action":"connection_abort"/, 'strict action logged'); +like($log, qr/"event":"phase4_intervention"/, 'event field present'); +like($log, qr/"header_sent":true/, 'json boolean header_sent'); +unlike($log, qr/\n.*\n/s, 'single-line json entries'); +unlike($log, qr/Hello minimal|Hello safe|Hello strict/, 'no response body data in phase4 log'); diff --git a/tests/modsecurity-phase4-regression.t b/tests/modsecurity-phase4-regression.t new file mode 100644 index 00000000..e68c75a9 --- /dev/null +++ b/tests/modsecurity-phase4-regression.t @@ -0,0 +1,46 @@ +#!/usr/bin/perl +use warnings; use strict; +use Test::More; +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +my $t = Test::Nginx->new()->has(qw/http/); +my $big = ('A' x 70000) . 'TAIL'; + +$t->write_file_expand('nginx.conf', <<'EOC'); +%%TEST_GLOBALS%% +daemon off; +events {} +http { + %%TEST_GLOBALS_HTTP%% + server { + listen 127.0.0.1:8080; + server_name localhost; + + location /big { + modsecurity on; + modsecurity_phase4_mode minimal; + modsecurity_phase4_log %%TESTDIR%%/phase4-regression.log; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx TAIL" "id:930001,phase:4,deny,log,status:403,msg:\"reg\"" + '; + } + } +} +EOC + +$t->write_file('/big', $big); +$t->run(); +$t->plan(5); + +my $resp = http_get('/big'); +like($resp, qr/HTTP\/1\.1 200 OK/, 'big response status remains 200 in minimal mode'); +like($resp, qr/A{1024}/, 'big response body contains expected prefix chunk'); +like($resp, qr/TAIL/, 'big response body tail present (no truncation)'); + +my $log = $t->read_file('phase4-regression.log'); +like($log, qr/"actual_action":"log_only"/, 'minimal mode logs only'); +unlike($log, qr/A{100,}|TAIL/, 'no large response data leaked to phase4 log'); diff --git a/tests/modsecurity-proxy-h2.t b/tests/modsecurity-proxy-h2.t index e8ef0ad2..da0c9d53 100644 --- a/tests/modsecurity-proxy-h2.t +++ b/tests/modsecurity-proxy-h2.t @@ -86,6 +86,7 @@ http { } location /phase4 { modsecurity on; + modsecurity_phase4_mode strict; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On diff --git a/tests/modsecurity.t b/tests/modsecurity.t index fcb26f4f..ff31c229 100644 --- a/tests/modsecurity.t +++ b/tests/modsecurity.t @@ -95,6 +95,7 @@ http { } location /phase4 { modsecurity on; + modsecurity_phase4_mode strict; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On From 25aa9e9d96afaec8da5380b0ad7c9494bebb6433 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Mon, 4 May 2026 19:25:35 +0200 Subject: [PATCH 02/40] Merge remote-tracking branch 'origin/master' into work --- .github/scripts/resolve-nginx-ref.sh | 34 +++ .github/workflows/test.yml | 4 +- .github/workflows/test_new.yml | 413 +++++++++++++++++++++++++++ .travis.yml | 11 +- 4 files changed, 454 insertions(+), 8 deletions(-) create mode 100644 .github/scripts/resolve-nginx-ref.sh create mode 100644 .github/workflows/test_new.yml diff --git a/.github/scripts/resolve-nginx-ref.sh b/.github/scripts/resolve-nginx-ref.sh new file mode 100644 index 00000000..cd516c4f --- /dev/null +++ b/.github/scripts/resolve-nginx-ref.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +VERSION="${NGINX_VERSION:-}" + +if [[ -z "${VERSION}" ]]; then + echo "NGINX_VERSION is not set" >&2 + exit 1 +fi + +echo "Requested NGINX_VERSION=${VERSION}" + +if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.x$ ]]; then + PREFIX="release-${VERSION%.x}." + REF=$(git ls-remote --tags https://github.com/nginx/nginx.git "refs/tags/${PREFIX}*" \ + | sed -E 's#.*refs/tags/##; s#\^\{\}##' \ + | grep -E "^${PREFIX}[0-9]+$" \ + | sort -V \ + | tail -n 1) +elif [[ "${VERSION}" =~ ^release-[0-9]+\.[0-9]+\.[0-9]+$ || "${VERSION}" == "master" || "${VERSION}" == "main" ]]; then + REF="${VERSION}" +else + echo "Unsupported NGINX_VERSION=${VERSION}" >&2 + echo "Use e.g. 1.30.x, 1.29.x, 1.28.x or exact tag release-1.30.0" >&2 + exit 1 +fi + +if [[ -z "${REF}" ]]; then + echo "No matching Nginx tag found for ${VERSION}" >&2 + exit 1 +fi + +echo "Resolved Nginx ref: ${REF}" +echo "ref=${REF}" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01268470..4fa41c1d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: with: repository: owasp-modsecurity/ModSecurity path: ModSecurity - submodules: true + submodules: recursive fetch-depth: 1 - name: Build libModSecurity working-directory: ModSecurity @@ -274,7 +274,7 @@ jobs: uses: actions/checkout@v4 with: repository: owasp-modsecurity/ModSecurity - submodules: true + submodules: recursive path: nginx/objs/lib/ModSecurity fetch-depth: 1 - name: Setup Conan diff --git a/.github/workflows/test_new.yml b/.github/workflows/test_new.yml new file mode 100644 index 00000000..7a981176 --- /dev/null +++ b/.github/workflows/test_new.yml @@ -0,0 +1,413 @@ +name: Quality Assurance new + +on: + pull_request: + push: + workflow_dispatch: + + +# Default: none +permissions: {} + +jobs: + build-linux: + name: linux-${{ matrix.compiler }}-nginx-${{ matrix.nginx_version }} + permissions: + contents: read + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04] + compiler: [gcc, clang] + nginx_version: ["1.30.x", "1.29.x", "1.28.x"] + env: + NGINX_VERSION: ${{ matrix.nginx_version }} + CC: "/usr/bin/${{ matrix.compiler }}" + CXX: "/usr/bin/${{ matrix.compiler == 'gcc' && 'g' || 'clang' }}++" + COMPDEPS: "${{ matrix.compiler == 'gcc' && 'gcc g++' || 'clang' }}" + steps: + - name: Setup Dependencies + run: | + sudo apt-get update -y -qq + sudo apt-get install -y make autoconf automake pcre2-utils libpcre2-dev libyajl-dev libxml2-dev libmaxminddb-dev libcurl4-gnutls-dev $COMPDEPS + - name: Get libModSecurity source + uses: actions/checkout@v6 + with: + repository: owasp-modsecurity/ModSecurity + path: ModSecurity + submodules: recursive + fetch-depth: 1 + - name: Build libModSecurity + working-directory: ModSecurity + run: | + ./build.sh + ./configure --without-lmdb --prefix=/usr + make -j $(nproc) + sudo make install + - uses: actions/checkout@v6 + with: + path: ModSecurity-nginx + fetch-depth: 1 + - name: Resolve latest Nginx version + id: nginx_ref + shell: bash + run: bash ModSecurity-nginx/.github/scripts/resolve-nginx-ref.sh + + - name: Get Nginx source + uses: actions/checkout@v6 + with: + repository: nginx/nginx + ref: ${{ steps.nginx_ref.outputs.ref }} + path: nginx + fetch-depth: 0 + + - name: Show Nginx source version + working-directory: nginx + shell: bash + run: | + echo "Requested NGINX_VERSION=${NGINX_VERSION}" + echo "Resolved ref=${{ steps.nginx_ref.outputs.ref }}" + echo "Commit=$(git rev-parse HEAD)" + echo "Describe=$(git describe --tags --always)" + grep '#define NGINX_VERSION' src/core/nginx.h || true + + - name: Get Nginx tests + uses: actions/checkout@v6 + with: + repository: nginx/nginx-tests + path: nginx/test + fetch-depth: 0 + + - name: Show Nginx tests version + working-directory: nginx/test + shell: bash + run: | + echo "Commit=$(git rev-parse HEAD)" + echo "Describe=$(git describe --tags --always)" + git log -1 --oneline + - name: Copy ModSecurity-nginx tests to nginx/test + run: | + cp ModSecurity-nginx/tests/* nginx/test + - name: Build nginx with ModSecurity-nginx module + working-directory: nginx + run: | + ./auto/configure --with-ld-opt="-Wl,-rpath,/usr/local/lib" --with-http_v2_module --with-http_auth_request_module --add-module=../ModSecurity-nginx + make + make modules + sudo make install + - name: Run ModSecurity-nginx tests + working-directory: nginx/test + run: | + TEST_NGINX_BINARY=../objs/nginx prove modsecurity*.t + - name: Start Nginx + run: | + sudo mkdir -p /usr/local/nginx/html + sudo cp "${{ github.workspace }}/ModSecurity-nginx/.github/nginx/docs/"* /usr/local/nginx/html/ + sudo /usr/local/nginx/sbin/nginx -c "${{ github.workspace }}/ModSecurity-nginx/.github/nginx/nginx.conf" + - name: Run attack test vhost 1 + run: | + status=$(curl -sSo /dev/null -w %{http_code} -I -X GET -H "Host: modsectest1" "http://localhost/?q=attack") + if [ "${status}" == "403" ]; then + echo "OK" + else + echo "FAIL" + exit 1 + fi + - name: Run non-attack test vhost 1 + run: | + status=$(curl -sSo /dev/null -w %{http_code} -I -X GET -H "Host: modsectest1" "http://localhost/?q=1") + if [ "${status}" == "200" ]; then + echo "OK" + else + echo "FAIL" + exit 1 + fi + - name: Run attack test vhost 2 + run: | + status=$(curl -sSo /dev/null -w %{http_code} -I -X GET -H "Host: modsectest2" "http://localhost/?q=attack") + if [ "${status}" == "403" ]; then + echo "OK" + else + echo "FAIL" + exit 1 + fi + - name: Run non-attack test vhost 2 + run: | + status=$(curl -sSo /dev/null -w %{http_code} -I -X GET -H "Host: modsectest2" "http://localhost/?q=1") + if [ "${status}" == "200" ]; then + echo "OK" + else + echo "FAIL" + exit 1 + fi + - name: Run file consistency check 1 + run: | + curl -sS "http://localhost/data50k.json" --output data50k.json + if [ -f data50k.json ]; then + diff data50k.json /usr/local/nginx/html/data50k.json > /dev/null + if [ $? -eq 0 ]; then + echo "OK" + else + echo "FAIL" + exit 2 + fi + else + echo "FAIL" + exit 1 + fi + - name: Run file consistency check 2 + run: | + curl -sS "http://localhost/plugged.png" --output plugged.png + if [ -f plugged.png ]; then + diff plugged.png /usr/local/nginx/html/plugged.png > /dev/null + if [ $? -eq 0 ]; then + echo "OK" + else + echo "FAIL" + exit 2 + fi + else + echo "FAIL" + exit 1 + fi + - name: Check attack log vhost 2 (modsecurity_use_error_log on(default)) + run: | + if ( grep -q "modsectest2" /usr/local/nginx/logs/error.log ); then + echo "OK" + else + echo "FAIL" + exit 1 + fi + - name: Check attack log vhost 3 (modsecurity_use_error_log off) + run: | + status=$(curl -sSo /dev/null -w %{http_code} -I -X GET -H "Host: modsecurity_use_error_log_off" "http://localhost/?q=attack") + if [ "${status}" == "403" ]; then + if ( grep -q "modsecurity_use_error_log_off" /usr/local/nginx/logs/error.log ); then + echo "FAIL" + exit 1 + else + echo "OK" + fi + else + echo "FAIL" + exit 1 + fi + - name: Start Nginx with redir + run: | + sudo killall nginx + sudo /usr/local/nginx/sbin/nginx -c "${{ github.workspace }}/ModSecurity-nginx/.github/nginx/nginx.conf.redir" + - name: Run attack test vhost 1 + run: | + status=$(curl -sSo /dev/null -w %{http_code} -I -X GET -H "Host: modsectest1" "http://localhost/?q=attack") + if [ "${status}" == "403" ]; then + echo "OK" + else + echo "FAIL" + exit 1 + fi + - name: Run non-attack test vhost 1 (redir config) + run: | + status=$(curl -sSo /dev/null -w %{http_code} -I -X GET -H "Host: modsectest1" "http://localhost/?q=1") + if [ "${status}" == "200" ]; then + echo "OK" + else + echo "FAIL" + exit 1 + fi + - name: Run attack test vhost 2 (redir config) + run: | + status=$(curl -sSo /dev/null -w %{http_code} -I -X GET -H "Host: modsectest2" "http://localhost/?q=attack") + if [ "${status}" == "403" ]; then + echo "OK" + else + echo "FAIL" + exit 1 + fi + - name: Run non-attack test vhost 2 (redir config) + run: | + status=$(curl -sSo /dev/null -w %{http_code} -I -X GET -H "Host: modsectest2" "http://localhost/?q=1") + if [ "${status}" == "200" ]; then + echo "OK" + else + echo "FAIL" + exit 1 + fi + - name: Run file consistency check 1 (redir config) + run: | + curl -sS "http://localhost/data50k.json" --output data50k.json + if [ -f data50k.json ]; then + diff data50k.json /usr/local/nginx/html/data50k.json > /dev/null + if [ $? -eq 0 ]; then + ls -l data50k.json /usr/local/nginx/html/data50k.json + echo "OK" + else + echo "FAIL" + exit 2 + fi + else + echo "FAIL" + exit 1 + fi + - name: Run file consistency check 2 (redir config) + run: | + curl -sS "http://localhost/plugged.png" --output plugged.png + if [ -f plugged.png ]; then + diff plugged.png /usr/local/nginx/html/plugged.png > /dev/null + if [ $? -eq 0 ]; then + ls -l plugged.png /usr/local/nginx/html/plugged.png + echo "OK" + else + echo "FAIL" + exit 2 + fi + else + echo "FAIL" + exit 1 + fi + + + build-windows: + name: windows-nginx-${{ matrix.nginx_version }} + runs-on: windows-2025 + strategy: + fail-fast: false + matrix: + nginx_version: ["1.30.x", "1.29.x", "1.28.x"] + env: + NGINX_VERSION: ${{ matrix.nginx_version }} + defaults: + run: + shell: msys2 {0} + steps: + - name: Set up MSVC + uses: ilammy/msvc-dev-cmd@v1 + - name: Set up msys + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + path-type: inherit + - name: Get ModSecurity-nginx workflow scripts + uses: actions/checkout@v6 + with: + path: ModSecurity-nginx + fetch-depth: 1 + - name: Resolve latest Nginx version + id: nginx_ref + shell: bash + run: bash ModSecurity-nginx/.github/scripts/resolve-nginx-ref.sh + + - name: Get Nginx source + uses: actions/checkout@v6 + with: + repository: nginx/nginx + ref: ${{ steps.nginx_ref.outputs.ref }} + path: nginx + fetch-depth: 0 + + - name: Show Nginx source version + working-directory: nginx + shell: bash + run: | + echo "Requested NGINX_VERSION=${NGINX_VERSION}" + echo "Resolved ref=${{ steps.nginx_ref.outputs.ref }}" + echo "Commit=$(git rev-parse HEAD)" + echo "Describe=$(git describe --tags --always)" + grep '#define NGINX_VERSION' src/core/nginx.h || true + + - name: Get Nginx tests + uses: actions/checkout@v6 + with: + repository: nginx/nginx-tests + path: nginx/test + fetch-depth: 0 + + - name: Show Nginx tests version + working-directory: nginx/test + shell: bash + run: | + echo "Commit=$(git rev-parse HEAD)" + echo "Describe=$(git describe --tags --always)" + git log -1 --oneline + - name: Set up third-party libraries + working-directory: nginx + run: | + mkdir objs + mkdir objs/lib + cd objs/lib + wget -q -O - https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.47/pcre2-10.47.tar.gz | tar -xzf - + wget -q -O - https://www.zlib.net/fossils/zlib-1.3.2.tar.gz | tar -xzf - + wget -q -O - https://www.openssl.org/source/openssl-3.6.2.tar.gz | tar -xzf - + - name: Get libModSecurity source + uses: actions/checkout@v6 + with: + repository: owasp-modsecurity/ModSecurity + submodules: recursive + path: nginx/objs/lib/ModSecurity + fetch-depth: 1 + - name: Setup Conan + shell: cmd + run: | + pip3 install conan --upgrade + conan profile detect + - name: Build libModSecurity + working-directory: nginx/objs/lib/ModSecurity + shell: cmd + run: | + vcbuild.bat + - name: Get ModSecurity-nginx source code + uses: actions/checkout@v6 + with: + path: nginx/objs/lib/ModSecurity-nginx + - name: Copy ModSecurity-nginx tests to nginx/test + working-directory: nginx/test + run: | + cp ../objs/lib/ModSecurity-nginx/tests/* . + - name: Remove /usr/bin/link conflicting with MSVC link.exe + run: | + set -ex + which link + rm /usr/bin/link + - name: Build nginx w/ModSecurity-nginx module + working-directory: nginx + run: | + : # Windows native version of Perl is required by nginx build + export PATH=/c/Strawberry/perl/bin:$PATH + : # Set env variables to point to libModSecurity v3 include & lib directories + export MODSECURITY_INC=objs/lib/ModSecurity/headers + export MODSECURITY_LIB=objs/lib/ModSecurity/build/win32/build/Release + : # Copy libModSecurity.dll to objs dir (to be able to run nginx later) + cp $MODSECURITY_LIB/libModSecurity.dll objs + : # Configure nginx build w/ModSecurity-nginx module + auto/configure \ + --with-cc=cl \ + --with-debug \ + --prefix= \ + --conf-path=conf/nginx.conf \ + --pid-path=logs/nginx.pid \ + --http-log-path=logs/access.log \ + --error-log-path=logs/error.log \ + --sbin-path=nginx.exe \ + --http-client-body-temp-path=temp/client_body_temp \ + --http-proxy-temp-path=temp/proxy_temp \ + --http-fastcgi-temp-path=temp/fastcgi_temp \ + --http-scgi-temp-path=temp/scgi_temp \ + --http-uwsgi-temp-path=temp/uwsgi_temp \ + --with-cc-opt=-DFD_SETSIZE=1024 \ + --with-pcre=objs/lib/pcre2-10.47 \ + --with-zlib=objs/lib/zlib-1.3.2 \ + --with-openssl=objs/lib/openssl-3.6.2 \ + --with-openssl-opt=no-asm \ + --with-http_ssl_module \ + --with-http_v2_module \ + --with-http_auth_request_module \ + --add-module=objs/lib/ModSecurity-nginx + nmake + - name: Run ModSecurity-nginx tests + working-directory: nginx/test + shell: cmd # tests need to run on a "windows" shell + run: | + md temp + set TEMP=temp + set TEST_NGINX_BINARY=..\objs\nginx.exe + prove modsecurity*.t diff --git a/.travis.yml b/.travis.yml index f6e050b5..3cf380db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,12 @@ addons: apt: packages: - libyajl-dev - - libgeoip-dev - liblmdb-dev env: - VER_NGINX=1.21.0 - VER_NGINX=1.20.1 + - VER_NGINX=1.29.8 before_script: - cd .. @@ -31,15 +31,14 @@ before_script: - make - sudo make install - cd .. - - wget http://nginx.org/download/nginx-${VER_NGINX}.tar.gz && tar -xf nginx-${VER_NGINX}.tar.gz - - cd nginx-${VER_NGINX} + - git clone --depth 1 --branch release-${VER_NGINX} https://github.com/nginx/nginx.git + - cd nginx - ./configure --with-http_auth_request_module --with-http_v2_module --add-module=../ModSecurity-nginx - make - sudo make install - cd .. - - wget http://hg.nginx.org/nginx-tests/archive/tip.tar.gz - - tar xvzf tip.tar.gz - - cd nginx-tests-* + - git clone --depth 1 https://github.com/nginx/nginx-tests.git + - cd nginx-tests - cp ../ModSecurity-nginx/tests/* . - export TEST_NGINX_BINARY=/usr/local/nginx/sbin/nginx From 88d033910c1df26a4e5b7cf52bcb05ebd1af5c61 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Mon, 4 May 2026 21:43:40 +0200 Subject: [PATCH 03/40] Fix phase4 build issues and raw rule/status logging order --- src/ngx_http_modsecurity_body_filter.c | 20 ++++++++++++++------ src/ngx_http_modsecurity_module.c | 9 +++++---- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index 1d5f5cf2..d6f84ca1 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -14,6 +14,7 @@ */ #include +#include #ifndef MODSECURITY_DDEBUG #define MODSECURITY_DDEBUG 0 @@ -224,7 +225,7 @@ ngx_http_modsecurity_phase4_in_scope(ngx_http_request_t *r) ct = r->headers_out.content_type; semi = (u_char *)ngx_strlchr(ct.data, ct.data + ct.len, ';'); if (semi != NULL) ct.len = semi - ct.data; - while (ct.len > 0 && ngx_isspace(ct.data[ct.len - 1])) ct.len--; + while (ct.len > 0 && isspace((unsigned char)ct.data[ct.len - 1])) ct.len--; for (i = 0; i < mcf->phase4_content_types->nelts; i++) { ngx_str_t *arr = mcf->phase4_content_types->elts; if (arr[i].len == ct.len && ngx_strncasecmp(arr[i].data, ct.data, ct.len) == 0) return 1; @@ -237,7 +238,7 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit { u_char buf[2048]; u_char *p; - ngx_str_t euri, emethod, ect, elog, erule; + ngx_str_t euri, emethod, ect, elog, erule, raw_log; const char *mode = "safe"; const char *header_sent = r->header_sent ? "true" : "false"; ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); @@ -245,13 +246,20 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit ngx_http_modsecurity_json_escape(r->pool, &r->uri, &euri); ngx_http_modsecurity_json_escape(r->pool, &r->method_name, &emethod); ngx_http_modsecurity_json_escape(r->pool, &r->headers_out.content_type, &ect); - if (ctx) ngx_http_modsecurity_json_escape(r->pool, &ctx->last_intervention_log, &elog); else { elog.len=0; elog.data=(u_char*)""; } + if (ctx) { + raw_log = ctx->last_intervention_log; + ngx_http_modsecurity_extract_rule_id(r->pool, &raw_log, &erule); + ngx_http_modsecurity_json_escape(r->pool, &raw_log, &elog); + } else { + raw_log.len = 0; raw_log.data = (u_char *)""; + elog.len = 0; elog.data=(u_char*)""; + erule.len = 0; erule.data=(u_char*)""; + } if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL) mode = "minimal"; else if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) mode = "strict"; - ngx_http_modsecurity_extract_rule_id(r->pool, &elog, &erule); p = ngx_snprintf(buf, sizeof(buf), - "{\"event\":\"phase4_intervention\",\"uri\":\"%V\",\"method\":\"%V\",\"status\":%ui,\"content_type\":\"%V\",\"header_sent\":%s,\"mode\":\"%s\",\"wanted_action\":\"%s\",\"actual_action\":\"%s\",\"reason\":\"%s\",\"intervention\":\"%V\",\"rule_id\":\"%V\"}\n", - &euri,&emethod,(ngx_uint_t)r->headers_out.status,&ect,header_sent,mode,wanted,actual,reason,&elog,&erule); + "{\"event\":\"phase4_intervention\",\"uri\":\"%V\",\"method\":\"%V\",\"response_status\":%ui,\"waf_status\":%i,\"content_type\":\"%V\",\"header_sent\":%s,\"mode\":\"%s\",\"wanted_action\":\"%s\",\"actual_action\":\"%s\",\"reason\":\"%s\",\"intervention\":\"%V\",\"rule_id\":\"%V\"}\n", + &euri,&emethod,(ngx_uint_t)r->headers_out.status,ctx ? (int) ctx->last_intervention_status : 0,&ect,header_sent,mode,wanted,actual,reason,&elog,&erule); ngx_write_fd(mcf->phase4_log_file->fd, buf, p - buf); return NGX_OK; } diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index 3c6dc123..7d493a88 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -22,6 +22,7 @@ #include "ngx_http_modsecurity_common.h" #include "stdio.h" +#include #include #include @@ -542,7 +543,7 @@ ngx_http_modsecurity_phase4_validate_content_type(u_char *s, size_t len) if (len == 0 || ngx_strchr(s, '*') != NULL) return NGX_ERROR; for (i = 0; i < len; i++) { u_char c = s[i]; - if (!(ngx_isalnum(c) || c == '/' || c == '.' || c == '+' || c == '-' )) return NGX_ERROR; + if (!(isalnum((unsigned char)c) || c == '/' || c == '.' || c == '+' || c == '-' )) return NGX_ERROR; } return (ngx_strchr(s, '/') != NULL) ? NGX_OK : NGX_ERROR; } @@ -592,15 +593,15 @@ ngx_http_modsecurity_phase4_load_content_types_file(ngx_conf_t *cf, ngx_http_mod for (p = buf, line = buf; p <= buf + n; p++) { if (p == buf + n || *p == '\n' || *p == '\r') { *p = '\0'; end = p; - while (line < end && ngx_isspace(*line)) line++; - while (end > line && ngx_isspace(*(end-1))) *(--end) = '\0'; + while (line < end && isspace((unsigned char)*line)) line++; + while (end > line && isspace((unsigned char)*(end-1))) *(--end) = '\0'; if (line[0] && line[0] != '#') { u_char *hash = (u_char *)ngx_strchr(line, '#'); u_char *semi = (u_char *)ngx_strchr(line, ';'); if (hash && (!semi || hash < semi)) *hash = '\0'; if (semi) *semi = '\0'; end = line + ngx_strlen(line); - while (end > line && ngx_isspace(*(end-1))) *(--end) = '\0'; + while (end > line && isspace((unsigned char)*(end-1))) *(--end) = '\0'; ngx_strlow(line, line, end - line); if (ngx_http_modsecurity_phase4_validate_content_type(line, end-line) != NGX_OK) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid content-type entry in modsecurity_phase4_content_types_file \"%V\": \"%s\"", path, line); From 4d1bb873c38a84cf886d021ff231215e7a765825 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Tue, 5 May 2026 12:40:30 +0200 Subject: [PATCH 04/40] Fix phase4 event logging paths and JSON output correctness --- src/ngx_http_modsecurity_body_filter.c | 89 ++++++++++++++++++-------- 1 file changed, 61 insertions(+), 28 deletions(-) diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index d6f84ca1..ff9842ca 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -26,8 +26,10 @@ static ngx_http_output_body_filter_pt ngx_http_next_body_filter; static ngx_int_t ngx_http_modsecurity_phase4_in_scope(ngx_http_request_t *r); static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason); +static ngx_int_t ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf); static void ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst); static void ngx_http_modsecurity_extract_rule_id(ngx_pool_t *pool, ngx_str_t *intervention, ngx_str_t *rule_id); +static ngx_str_t ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in); /* XXX: check behaviour on few body filters installed */ ngx_int_t @@ -158,25 +160,8 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, ret); } else if (ret < 0) { - if (ctx->phase4_headers_checked) { - return ngx_http_next_body_filter(r, in); - } - ctx->phase4_headers_checked = 1; - if (ngx_http_modsecurity_phase4_in_scope(r) == 0 || mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL) { - ngx_http_modsecurity_phase4_log_event(r, mcf, "deny_status", "log_only", ngx_http_modsecurity_phase4_in_scope(r) ? "mode_minimal" : "content_type_not_in_scope"); - ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, - "modsecurity phase4 intervention after headers sent, action=log_only, uri=\"%V\"", &r->uri); - } else if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { - ngx_http_modsecurity_phase4_log_event(r, mcf, "deny_status", "connection_abort", "headers_already_sent"); - ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "modsecurity phase4 intervention after headers sent, action=connection_abort, uri=\"%V\"", &r->uri); - r->connection->error = 1; - return NGX_ERROR; - } else { - ngx_http_modsecurity_phase4_log_event(r, mcf, "deny_status", "log_only", "mode_safe"); - ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, - "modsecurity phase4 intervention after headers sent, action=log_only, uri=\"%V\"", &r->uri); - } + ret = ngx_http_modsecurity_phase4_handle_intervention(r, mcf); + if (ret == NGX_ERROR) return NGX_ERROR; } /* XXX: chain->buf->last_buf || chain->buf->last_in_chain */ @@ -193,14 +178,16 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) XXX: body we can proceed to adjust body size (content-length). see xslt_body_filter() for example */ ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); if (ret > 0) { + if (!ctx->phase4_headers_checked) { + ngx_http_modsecurity_phase4_log_event(r, mcf, "deny", "deny_status", "headers_not_sent"); + ctx->phase4_headers_checked = 1; + } return ret; } else if (ret < 0) { - if (ngx_http_modsecurity_phase4_in_scope(r) == 0 || mcf->phase4_mode != NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { - return ngx_http_next_body_filter(r, in); - } - r->connection->error = 1; - return NGX_ERROR; + ret = ngx_http_modsecurity_phase4_handle_intervention(r, mcf); + if (ret == NGX_ERROR) return NGX_ERROR; + return ngx_http_next_body_filter(r, in); } } @@ -214,6 +201,30 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) return ngx_http_next_body_filter(r, in); } +static ngx_int_t +ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf) +{ + ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); + ngx_int_t in_scope = ngx_http_modsecurity_phase4_in_scope(r); + if (ctx && ctx->phase4_headers_checked) return NGX_OK; + if (ctx) ctx->phase4_headers_checked = 1; + + if (in_scope == 0) { + ngx_http_modsecurity_phase4_log_event(r, mcf, "deny", "log_only", r->headers_out.content_type.len ? "content_type_not_in_scope" : "content_type_missing"); + return NGX_OK; + } + if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { + ngx_http_modsecurity_phase4_log_event(r, mcf, "deny", "connection_abort", "headers_already_sent"); + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "modsecurity phase4 intervention after headers sent, action=connection_abort, uri=\"%V\"", &r->uri); + r->connection->error = 1; + return NGX_ERROR; + } + ngx_http_modsecurity_phase4_log_event(r, mcf, "deny", "log_only", + mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL ? "mode_minimal" : "mode_safe"); + return NGX_OK; +} + static ngx_int_t ngx_http_modsecurity_phase4_in_scope(ngx_http_request_t *r) { @@ -236,7 +247,6 @@ ngx_http_modsecurity_phase4_in_scope(ngx_http_request_t *r) static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason) { - u_char buf[2048]; u_char *p; ngx_str_t euri, emethod, ect, elog, erule, raw_log; const char *mode = "safe"; @@ -245,7 +255,8 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit if (mcf->phase4_log_file == NULL || mcf->phase4_log_file->fd == NGX_INVALID_FILE) return NGX_OK; ngx_http_modsecurity_json_escape(r->pool, &r->uri, &euri); ngx_http_modsecurity_json_escape(r->pool, &r->method_name, &emethod); - ngx_http_modsecurity_json_escape(r->pool, &r->headers_out.content_type, &ect); + ngx_str_t nct = ngx_http_modsecurity_normalize_content_type(r->pool, r->headers_out.content_type); + ngx_http_modsecurity_json_escape(r->pool, &nct, &ect); if (ctx) { raw_log = ctx->last_intervention_log; ngx_http_modsecurity_extract_rule_id(r->pool, &raw_log, &erule); @@ -257,13 +268,35 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit } if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL) mode = "minimal"; else if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) mode = "strict"; - p = ngx_snprintf(buf, sizeof(buf), + size_t need = 256 + euri.len + emethod.len + ect.len + elog.len + erule.len + ngx_strlen(mode) + ngx_strlen(wanted) + ngx_strlen(actual) + ngx_strlen(reason); + u_char *dbuf = ngx_pnalloc(r->pool, need); + if (dbuf == NULL) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, "modsecurity phase4 log allocation failed"); + return NGX_ERROR; + } + p = ngx_snprintf(dbuf, need, "{\"event\":\"phase4_intervention\",\"uri\":\"%V\",\"method\":\"%V\",\"response_status\":%ui,\"waf_status\":%i,\"content_type\":\"%V\",\"header_sent\":%s,\"mode\":\"%s\",\"wanted_action\":\"%s\",\"actual_action\":\"%s\",\"reason\":\"%s\",\"intervention\":\"%V\",\"rule_id\":\"%V\"}\n", &euri,&emethod,(ngx_uint_t)r->headers_out.status,ctx ? (int) ctx->last_intervention_status : 0,&ect,header_sent,mode,wanted,actual,reason,&elog,&erule); - ngx_write_fd(mcf->phase4_log_file->fd, buf, p - buf); + ngx_write_fd(mcf->phase4_log_file->fd, dbuf, p - dbuf); return NGX_OK; } +static ngx_str_t +ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in) +{ + ngx_str_t out; size_t i; + u_char *semi; + out = in; + if (out.data == NULL || out.len == 0) return out; + semi = (u_char *)ngx_strlchr(out.data, out.data + out.len, ';'); + if (semi) out.len = semi - out.data; + while (out.len > 0 && isspace((unsigned char) out.data[out.len - 1])) out.len--; + out.data = ngx_pnalloc(pool, out.len); + if (out.data == NULL) { out.len = 0; return out; } + for (i = 0; i < out.len; i++) out.data[i] = ngx_tolower(in.data[i]); + return out; +} + static void ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst) { From 6962d42d0f4e415567366100cfed138438c6a848 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Tue, 5 May 2026 18:21:09 +0200 Subject: [PATCH 05/40] Fix phase4 log sanitization, deterministic reasons, and proxy phase4 strict test config --- src/ngx_http_modsecurity_body_filter.c | 26 ++++++++++++++++++++++++-- tests/modsecurity-proxy.t | 1 + 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index ff9842ca..6d33de5e 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -30,6 +30,7 @@ static ngx_int_t ngx_http_modsecurity_phase4_handle_intervention(ngx_http_reques static void ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst); static void ngx_http_modsecurity_extract_rule_id(ngx_pool_t *pool, ngx_str_t *intervention, ngx_str_t *rule_id); static ngx_str_t ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in); +static ngx_str_t ngx_http_modsecurity_sanitize_intervention(ngx_pool_t *pool, ngx_str_t in); /* XXX: check behaviour on few body filters installed */ ngx_int_t @@ -248,7 +249,7 @@ static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason) { u_char *p; - ngx_str_t euri, emethod, ect, elog, erule, raw_log; + ngx_str_t euri, emethod, ect, elog, erule, raw_log, slog; const char *mode = "safe"; const char *header_sent = r->header_sent ? "true" : "false"; ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); @@ -260,7 +261,8 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit if (ctx) { raw_log = ctx->last_intervention_log; ngx_http_modsecurity_extract_rule_id(r->pool, &raw_log, &erule); - ngx_http_modsecurity_json_escape(r->pool, &raw_log, &elog); + slog = ngx_http_modsecurity_sanitize_intervention(r->pool, raw_log); + ngx_http_modsecurity_json_escape(r->pool, &slog, &elog); } else { raw_log.len = 0; raw_log.data = (u_char *)""; elog.len = 0; elog.data=(u_char*)""; @@ -297,6 +299,26 @@ ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in) return out; } +static ngx_str_t +ngx_http_modsecurity_sanitize_intervention(ngx_pool_t *pool, ngx_str_t in) +{ + ngx_str_t out = ngx_string("redacted"); + u_char *id, *msg, *op; + size_t len = 0; + id = (u_char *)ngx_strstr(in.data, "id \""); + msg = (u_char *)ngx_strstr(in.data, "msg \""); + op = (u_char *)ngx_strstr(in.data, "Operator"); + if (id == NULL && msg == NULL && op == NULL) { + return out; + } + len = 9 + (id ? 10 : 0) + (msg ? 12 : 0) + (op ? 10 : 0); + out.data = ngx_pnalloc(pool, len); + if (out.data == NULL) return ngx_string("redacted"); + out.len = ngx_snprintf(out.data, len, "id:%s msg:%s op:%s", + id ? "present" : "-", msg ? "present" : "-", op ? "present" : "-") - out.data; + return out; +} + static void ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst) { diff --git a/tests/modsecurity-proxy.t b/tests/modsecurity-proxy.t index a412f5ea..38c27d93 100644 --- a/tests/modsecurity-proxy.t +++ b/tests/modsecurity-proxy.t @@ -84,6 +84,7 @@ http { } location /phase4 { modsecurity on; + modsecurity_phase4_mode strict; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On From 3e355481b2a4de76f390e19a15ba54c9dfd6d4eb Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Tue, 5 May 2026 18:36:34 +0200 Subject: [PATCH 06/40] Fix phase4 C-level string safety issues --- src/ngx_http_modsecurity_body_filter.c | 10 +++++++--- src/ngx_http_modsecurity_module.c | 8 +++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index 6d33de5e..9b2fefd3 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -302,18 +302,22 @@ ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in) static ngx_str_t ngx_http_modsecurity_sanitize_intervention(ngx_pool_t *pool, ngx_str_t in) { - ngx_str_t out = ngx_string("redacted"); + static ngx_str_t redacted = { sizeof("redacted") - 1, (u_char *) "redacted" }; + ngx_str_t out = redacted; u_char *id, *msg, *op; size_t len = 0; + if (in.data == NULL || in.len == 0) { + return redacted; + } id = (u_char *)ngx_strstr(in.data, "id \""); msg = (u_char *)ngx_strstr(in.data, "msg \""); op = (u_char *)ngx_strstr(in.data, "Operator"); if (id == NULL && msg == NULL && op == NULL) { - return out; + return redacted; } len = 9 + (id ? 10 : 0) + (msg ? 12 : 0) + (op ? 10 : 0); out.data = ngx_pnalloc(pool, len); - if (out.data == NULL) return ngx_string("redacted"); + if (out.data == NULL) return redacted; out.len = ngx_snprintf(out.data, len, "id:%s msg:%s op:%s", id ? "present" : "-", msg ? "present" : "-", op ? "present" : "-") - out.data; return out; diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index 7d493a88..1fbbf96c 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -176,11 +176,17 @@ ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_re ctx->last_intervention_log.data = NULL; if (intervention.log != NULL) { size_t l = ngx_strlen(intervention.log); - u_char *cp = ngx_pnalloc(r->pool, l); + u_char *cp = ngx_pnalloc(r->pool, l + 1); if (cp != NULL) { ngx_memcpy(cp, intervention.log, l); + cp[l] = '\0'; ctx->last_intervention_log.data = cp; ctx->last_intervention_log.len = l; + } else { + if (intervention.log != NULL) { + free(intervention.log); + } + return NGX_HTTP_INTERNAL_SERVER_ERROR; } } From e2ede45d540197ed4126856fca72052982cc4032 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 6 May 2026 05:44:57 +0200 Subject: [PATCH 07/40] docs: add bilingual phase4 handling and config examples --- docs/examples/phase4-content-types.conf | 5 + docs/examples/phase4-minimal.conf | 14 ++ docs/examples/phase4-safe.conf | 14 ++ docs/examples/phase4-strict.conf | 14 ++ docs/phase4-handling.de.md | 162 ++++++++++++++++++++++++ docs/phase4-handling.en.md | 162 ++++++++++++++++++++++++ 6 files changed, 371 insertions(+) create mode 100644 docs/examples/phase4-content-types.conf create mode 100644 docs/examples/phase4-minimal.conf create mode 100644 docs/examples/phase4-safe.conf create mode 100644 docs/examples/phase4-strict.conf create mode 100644 docs/phase4-handling.de.md create mode 100644 docs/phase4-handling.en.md diff --git a/docs/examples/phase4-content-types.conf b/docs/examples/phase4-content-types.conf new file mode 100644 index 00000000..4bf8701b --- /dev/null +++ b/docs/examples/phase4-content-types.conf @@ -0,0 +1,5 @@ +# One content type per line; comments are allowed. +text/html +application/json +application/problem+json +text/plain diff --git a/docs/examples/phase4-minimal.conf b/docs/examples/phase4-minimal.conf new file mode 100644 index 00000000..35d78ce8 --- /dev/null +++ b/docs/examples/phase4-minimal.conf @@ -0,0 +1,14 @@ +# Example: minimal mode +location /phase4 { + modsecurity on; + modsecurity_phase4_mode minimal; + modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; + + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940001,phase:4,deny,log,status:403" + '; + + proxy_pass http://backend; +} diff --git a/docs/examples/phase4-safe.conf b/docs/examples/phase4-safe.conf new file mode 100644 index 00000000..851a09ea --- /dev/null +++ b/docs/examples/phase4-safe.conf @@ -0,0 +1,14 @@ +# Example: safe mode +location /phase4 { + modsecurity on; + modsecurity_phase4_mode safe; + modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; + + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940002,phase:4,deny,log,status:403" + '; + + proxy_pass http://backend; +} diff --git a/docs/examples/phase4-strict.conf b/docs/examples/phase4-strict.conf new file mode 100644 index 00000000..80d6c882 --- /dev/null +++ b/docs/examples/phase4-strict.conf @@ -0,0 +1,14 @@ +# Example: strict mode +location /phase4 { + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; + + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940003,phase:4,deny,log,status:403" + '; + + proxy_pass http://backend; +} diff --git a/docs/phase4-handling.de.md b/docs/phase4-handling.de.md new file mode 100644 index 00000000..1cb42cd7 --- /dev/null +++ b/docs/phase4-handling.de.md @@ -0,0 +1,162 @@ +# ModSecurity-nginx: Phase-4-Handling (Deutsch) + +## Einführung: Was ist das `phase:4`-Problem? + +`phase:4`-Regeln laufen während der Verarbeitung des **Response-Bodys**. In nginx kann zu diesem Zeitpunkt der Response-Header bereits an den Client gesendet worden sein. Wenn der Header bereits gesendet wurde, kann nginx den HTTP-Status (z. B. 403) oder Redirect-Header nicht mehr zuverlässig umstellen. + +In diesem Repository ist das explizit durch neue Phase-4-Mechanismen adressiert, inklusive Modi und strukturiertem Logging. + +## Warum „Header bereits gesendet“ problematisch ist + +Wenn eine ModSecurity-Intervention (`deny`, `status`, `redirect`) erst in `phase:4` auftritt, ist ein klassisches Blocken/Redirect nur möglich, solange Header noch nicht gesendet wurden. + +Wenn Header bereits gesendet sind: + +- ein späteres `status:401/403` ist nicht mehr sauber als HTTP-Status umsetzbar, +- ein späteres `redirect` (301/302 + `Location`) ist nicht mehr sauber als Redirect umsetzbar, +- nur degradierte Reaktion ist möglich (loggen, ggf. Verbindung abbrechen). + +## Warum **kein globales Response-Body-Buffering** genutzt wird + +Der implementierte Ansatz vermeidet globales Buffering großer Response-Bodies. Das ist konsistent mit folgenden Zielen: + +- kein pauschaler Speicher-/Latenz-Overhead für alle Responses, +- keine zusätzliche Reordering-Logik zwischen bereits gesendeten Headern und späteren Body-Entscheidungen, +- keine Schein-Garantie, dass `phase:4` immer in saubere HTTP-Block-/Redirect-Semantik überführt werden kann. + +## Neue Directives + +### `modsecurity_phase4_mode` + +Unterstützte Werte: + +- `minimal` +- `safe` +- `strict` + +Ungültige Werte werden in der Konfiguration abgelehnt. + +### `modsecurity_phase4_content_types_file` + +Lädt Content-Types aus einer Datei (eine Zeile pro Typ, `#`-Kommentare erlaubt). Einträge werden validiert; ungültige Einträge führen zu Konfigurationsfehlern. + +Wenn keine Datei gesetzt ist, werden Standardtypen verwendet. + +### `modsecurity_phase4_log` + +Aktiviert ein separates JSON-Line-Log für Phase-4-Interventionen. + +## Modi: Verhalten und Sicherheitsprofil + +## `minimal` + +- Bei `phase:4`-Intervention nach gesendeten Headern: **kein künstliches Blocken**, nur `log_only`. +- Ziel: minimalinvasiv, kein Verbindungsabbruch. + +## `safe` + +- Verhalten bei spät erkannter Intervention ebenfalls `log_only`. +- Ziel: stabiler Betrieb ohne erzwungenen Abbruch. + +## `strict` + +- Bei `phase:4`-Intervention nach gesendeten Headern: `connection_abort`. +- Ziel: striktere Reaktion, wenn keine saubere Status-/Redirect-Änderung mehr möglich ist. + +> Wichtig: `strict` garantiert **keinen** nachträglichen 401/403/301/302-Status; es kann stattdessen zum Verbindungsabbruch kommen. + +## Verhalten nach Header-Status + +### Header **noch nicht** gesendet + +Normale ModSecurity-Interventionspfade bleiben möglich (z. B. `deny_status`), weil nginx den Header noch anpassen kann. + +### Header **bereits** gesendet + +- `minimal`/`safe`: `log_only` +- `strict`: `connection_abort` + +## Erklärung der Actions + +### `connection_abort` + +- Implementierter Fallback im `strict`-Modus bei spät erkannter `phase:4`-Intervention. +- Technisch: Request wird mit Fehlerpfad beendet (keine nachträgliche Header-Umschreibung). + +### `log_only` + +- Intervention wird protokolliert, aber Response-Fluss bleibt (so weit möglich) bestehen. +- Dient Nachvollziehbarkeit ohne erzwungenen Transportabbruch. + +## Content-Type-Scoping (`modsecurity_phase4_content_types_file`) + +Phase-4-Handling wird über Content-Type eingeschränkt. Wenn `Content-Type` fehlt oder nicht in Scope ist, wird statt harter Aktion protokolliert (`log_only`, z. B. Grund `content_type_missing` / `content_type_not_in_scope`). + +Das reduziert unerwartete Effekte auf nicht-zielgerichtete Antworttypen. + +## Logging + +### `modsecurity_phase4_log` + +Schreibt JSON-Zeilen mit Feldern wie: + +- `event` (`phase4_intervention`) +- `uri`, `method` +- `response_status`, `waf_status` +- `content_type` +- `header_sent` (Boolean) +- `mode` +- `wanted_action`, `actual_action` +- `reason` +- `intervention` +- `rule_id` + +### `nginx error.log` + +Zusätzlich wird insbesondere im `strict`-Pfad eine Warnung ins nginx-Error-Log geschrieben, wenn nach gesendeten Headern eine Intervention eintritt. + +## Sicherheitsaspekte (Implementierungsentscheidungen) + +- **Keine Response-Bodies im Phase-4-Log**: reduziert Risiko von Datenabfluss über Logs. +- **Keine `ngx_chain_t`-Rewrites**: vermeidet komplexe, fehleranfällige Manipulation bereits laufender Body-Ketten. +- **Keine künstliche Reordering-Logik**: verhindert inkonsistente Zustände zwischen bereits gesendeten Headern und späteren Blockentscheidungen. +- **`strict` kann Verbindung abbrechen**: bewusster Trade-off für „fail closed“-ähnliche Reaktion ohne falsche HTTP-Status-Versprechen. + +## Beispielkonfigurationen + +Siehe: + +- `docs/examples/phase4-minimal.conf` +- `docs/examples/phase4-safe.conf` +- `docs/examples/phase4-strict.conf` +- `docs/examples/phase4-content-types.conf` + +## JSON-Log-Beispiele + +`log_only` (z. B. `safe`): + +```json +{"event":"phase4_intervention","uri":"/phase4","method":"GET","response_status":200,"waf_status":403,"content_type":"text/html","header_sent":true,"mode":"safe","wanted_action":"deny","actual_action":"log_only","reason":"mode_safe","intervention":"...","rule_id":"910002"} +``` + +`connection_abort` (`strict`): + +```json +{"event":"phase4_intervention","uri":"/phase4","method":"GET","response_status":200,"waf_status":403,"content_type":"text/html","header_sent":true,"mode":"strict","wanted_action":"deny","actual_action":"connection_abort","reason":"headers_already_sent","intervention":"...","rule_id":"910003"} +``` + +> Feldwerte hängen vom konkreten Request/Rule-Match ab. + +## Einschränkungen / bekannte Grenzen + +- `phase:4` kann **nicht** garantieren, dass ein gewünschter Block-/Redirect-Status noch als HTTP-Status beim Client ankommt. +- Bei bereits gesendeten Headern ist nur degradierte Reaktion möglich (`log_only` oder `connection_abort`). +- Content-Type-Scoping ist entscheidend; Out-of-Scope-Responses werden nicht „hart“ behandelt. + +## Hinweise für Betreiber + +1. `phase:4` nicht als alleinigen Mechanismus für harte Zugriffskontrolle planen. +2. Für harte Block-/Redirect-Entscheidungen möglichst frühere Phasen bevorzugen. +3. `modsecurity_phase4_log` aktivieren und regelmäßig auswerten. +4. `modsecurity_phase4_content_types_file` bewusst auf sensible MIME-Typen begrenzen. +5. `strict` nur einsetzen, wenn Verbindungsabbrüche betrieblich akzeptabel sind. diff --git a/docs/phase4-handling.en.md b/docs/phase4-handling.en.md new file mode 100644 index 00000000..600fcbcb --- /dev/null +++ b/docs/phase4-handling.en.md @@ -0,0 +1,162 @@ +# ModSecurity-nginx: Phase 4 Handling (English) + +## Introduction: what is the `phase:4` problem? + +`phase:4` rules run while processing the **response body**. In nginx, response headers may already have been sent to the client at that point. Once headers are sent, nginx cannot reliably switch HTTP status (for example to 403) or inject redirect headers anymore. + +This repository adds explicit Phase 4 handling for that condition, including dedicated modes and structured logging. + +## Why “headers already sent” is a problem + +If a ModSecurity intervention (`deny`, `status`, `redirect`) is triggered in `phase:4`, classic blocking/redirecting only works when headers are still unsent. + +If headers are already sent: + +- late `status:401/403` cannot be cleanly enforced as HTTP status, +- late redirect (`301/302` + `Location`) cannot be cleanly enforced, +- only degraded reactions remain (logging, optionally aborting the connection). + +## Why there is **no global response-body buffering** + +The implementation intentionally avoids global buffering of large response bodies. This aligns with: + +- no blanket memory/latency overhead for all responses, +- no extra reordering logic between already-sent headers and late body-time decisions, +- no false guarantee that `phase:4` can always be translated to clean HTTP block/redirect semantics. + +## New directives + +### `modsecurity_phase4_mode` + +Supported values: + +- `minimal` +- `safe` +- `strict` + +Invalid values are rejected at config parsing time. + +### `modsecurity_phase4_content_types_file` + +Loads content types from a file (one type per line, `#` comments supported). Entries are validated; invalid entries fail configuration loading. + +If not set, built-in default content types are used. + +### `modsecurity_phase4_log` + +Enables a dedicated JSON-lines log for Phase 4 interventions. + +## Modes: behavior and safety profile + +## `minimal` + +- For `phase:4` interventions after headers are sent: no synthetic deny, `log_only`. +- Goal: least intrusive behavior, no forced disconnect. + +## `safe` + +- For late interventions, behavior is also `log_only`. +- Goal: operational stability without forced abort. + +## `strict` + +- For `phase:4` interventions after headers are sent: `connection_abort`. +- Goal: stricter handling when clean status/redirect changes are no longer possible. + +> Important: `strict` does **not** guarantee a retroactive 401/403/301/302; it may terminate the connection instead. + +## Behavior by header state + +### Headers **not sent yet** + +Normal ModSecurity intervention paths remain possible (for example `deny_status`), because nginx can still adjust outgoing headers. + +### Headers **already sent** + +- `minimal`/`safe`: `log_only` +- `strict`: `connection_abort` + +## Action semantics + +### `connection_abort` + +- Implemented fallback in `strict` mode for late `phase:4` interventions. +- Technical effect: request terminates via error path (no post-hoc header rewrite). + +### `log_only` + +- Intervention is recorded, while response flow continues as far as possible. +- Provides traceability without forced transport interruption. + +## Content-Type scoping (`modsecurity_phase4_content_types_file`) + +Phase 4 handling is scoped by content type. If `Content-Type` is missing or not in scope, the module logs a degraded action (`log_only`, with reasons such as `content_type_missing` / `content_type_not_in_scope`) instead of enforcing a hard action. + +This limits side effects on non-targeted response types. + +## Logging + +### `modsecurity_phase4_log` + +Writes JSON lines with fields such as: + +- `event` (`phase4_intervention`) +- `uri`, `method` +- `response_status`, `waf_status` +- `content_type` +- `header_sent` (boolean) +- `mode` +- `wanted_action`, `actual_action` +- `reason` +- `intervention` +- `rule_id` + +### nginx `error.log` + +Additionally, especially on the `strict` path, a warning is emitted to nginx error log when an intervention occurs after headers are sent. + +## Security decisions (implementation) + +- **No response body in Phase 4 log**: reduces leakage risk through logs. +- **No `ngx_chain_t` rewriting**: avoids brittle low-level mutations of already flowing body chains. +- **No synthetic reordering logic**: avoids inconsistent states between sent headers and late decisions. +- **`strict` may abort connections**: deliberate trade-off for stricter fail behavior without false HTTP-status guarantees. + +## Example configurations + +See: + +- `docs/examples/phase4-minimal.conf` +- `docs/examples/phase4-safe.conf` +- `docs/examples/phase4-strict.conf` +- `docs/examples/phase4-content-types.conf` + +## JSON log examples + +`log_only` (for example `safe`): + +```json +{"event":"phase4_intervention","uri":"/phase4","method":"GET","response_status":200,"waf_status":403,"content_type":"text/html","header_sent":true,"mode":"safe","wanted_action":"deny","actual_action":"log_only","reason":"mode_safe","intervention":"...","rule_id":"910002"} +``` + +`connection_abort` (`strict`): + +```json +{"event":"phase4_intervention","uri":"/phase4","method":"GET","response_status":200,"waf_status":403,"content_type":"text/html","header_sent":true,"mode":"strict","wanted_action":"deny","actual_action":"connection_abort","reason":"headers_already_sent","intervention":"...","rule_id":"910003"} +``` + +> Field values depend on the actual request and matched rule. + +## Limitations / known boundaries + +- `phase:4` cannot **guarantee** that the desired block/redirect status still reaches the client as HTTP status. +- When headers are already sent, only degraded handling is possible (`log_only` or `connection_abort`). +- Content-type scoping is critical; out-of-scope responses are not hard-enforced. + +## Operator guidance + +1. Do not treat `phase:4` as the sole hard access-control mechanism. +2. Prefer earlier phases for hard block/redirect decisions. +3. Enable and monitor `modsecurity_phase4_log`. +4. Keep `modsecurity_phase4_content_types_file` narrowly scoped to sensitive MIME types. +5. Use `strict` only if connection aborts are operationally acceptable. From 2e2c79203ce829f236a4cbcc46b66292e0db4aee Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 6 May 2026 08:55:18 +0200 Subject: [PATCH 08/40] Fix json escape alloc safety and phase4 modes log-line assertions --- src/ngx_http_modsecurity_body_filter.c | 9 +++++++-- src/ngx_http_modsecurity_module.c | 11 ++++++++--- tests/modsecurity-phase4-content-types.t | 6 +++++- tests/modsecurity-phase4-invalid-config.t | 2 +- tests/modsecurity-phase4-modes.t | 7 +++++-- tests/modsecurity-proxy.t | 9 +++++++++ 6 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index 9b2fefd3..176cdcc6 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -329,7 +329,7 @@ ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *ds size_t i, extra = 0; u_char *d; if (src == NULL || src->data == NULL) { dst->len=0; dst->data=(u_char*)""; return; } for (i = 0; i < src->len; i++) if (src->data[i] < 0x20 || src->data[i] == '"' || src->data[i] == '\\') extra++; - dst->data = ngx_pnalloc(pool, src->len + extra + 1); if (dst->data == NULL) { dst->len=0; return; } + dst->data = ngx_pnalloc(pool, src->len + extra + 1); if (dst->data == NULL) { dst->len=0; dst->data=(u_char*)""; return; } d = dst->data; for (i = 0; i < src->len; i++) { u_char c = src->data[i]; @@ -354,7 +354,12 @@ ngx_http_modsecurity_extract_rule_id(ngx_pool_t *pool, ngx_str_t *intervention, if (j > i + 4 && j < intervention->len && intervention->data[j] == '"') { rule_id->len = j - (i + 4); rule_id->data = ngx_pnalloc(pool, rule_id->len); - if (rule_id->data != NULL) ngx_memcpy(rule_id->data, intervention->data + i + 4, rule_id->len); + if (rule_id->data == NULL) { + rule_id->len = 0; + rule_id->data = (u_char *)""; + return; + } + ngx_memcpy(rule_id->data, intervention->data + i + 4, rule_id->len); return; } } diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index 1fbbf96c..22561485 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -545,13 +545,18 @@ ngx_conf_set_phase4_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) static ngx_int_t ngx_http_modsecurity_phase4_validate_content_type(u_char *s, size_t len) { - size_t i; + size_t i, slash = (size_t)-1; if (len == 0 || ngx_strchr(s, '*') != NULL) return NGX_ERROR; for (i = 0; i < len; i++) { u_char c = s[i]; - if (!(isalnum((unsigned char)c) || c == '/' || c == '.' || c == '+' || c == '-' )) return NGX_ERROR; + if (c == '/') { + if (slash != (size_t)-1 || i == 0 || i + 1 >= len) return NGX_ERROR; + slash = i; + continue; + } + if (!(isalnum((unsigned char)c) || c == '-' || c == '.' || c == '_' || c == '+')) return NGX_ERROR; } - return (ngx_strchr(s, '/') != NULL) ? NGX_OK : NGX_ERROR; + return (slash != (size_t)-1) ? NGX_OK : NGX_ERROR; } static char * diff --git a/tests/modsecurity-phase4-content-types.t b/tests/modsecurity-phase4-content-types.t index d59e4553..9ce9c35d 100644 --- a/tests/modsecurity-phase4-content-types.t +++ b/tests/modsecurity-phase4-content-types.t @@ -15,6 +15,9 @@ Application/JSON; charset=utf-8 # inline comment TEXT/HTML text/plain application/xml +application/vnd.api+json +application/problem+json +application/x.custom_type CT $t->write_file_expand('nginx.conf', <<'EOC'); @@ -75,7 +78,7 @@ $t->write_file('/unknown', 'HIT UNKNOWN'); $t->write_file('/emptytype', 'HIT EMPTY'); $t->run(); -$t->plan(8); +$t->plan(9); is(http_get('/json'), '', 'json in-scope + strict => abort after headers sent'); like(http_get('/unknown'), qr/HIT UNKNOWN/, 'unknown content-type not in scope => no hard action'); @@ -87,3 +90,4 @@ like($log, qr/"actual_action":"connection_abort"/, 'strict in-scope abort logged like($log, qr/"reason":"content_type_not_in_scope"/, 'out-of-scope reason logged'); like($log, qr/"event":"phase4_intervention"/, 'json lines event present'); unlike($log, qr/HIT JSON|HIT UNKNOWN|HIT EMPTY/, 'no response body data in log'); +like($log, qr/"content_type":"application\/json"/, 'application/json remains valid'); diff --git a/tests/modsecurity-phase4-invalid-config.t b/tests/modsecurity-phase4-invalid-config.t index 775c2be8..176274a5 100644 --- a/tests/modsecurity-phase4-invalid-config.t +++ b/tests/modsecurity-phase4-invalid-config.t @@ -7,7 +7,7 @@ use Test::Nginx; # this test verifies parser rejects invalid entries/wildcards at config load my $t = Test::Nginx->new()->has(qw/http/); -$t->write_file('phase4-invalid.conf', "text/*\napplication/json\ninvalid type\n"); +$t->write_file('phase4-invalid.conf', "text/*\n*/json\ntext/html bad\napplication/problem+json\n"); $t->write_file_expand('nginx.conf', <<'EOC'); %%TEST_GLOBALS%% diff --git a/tests/modsecurity-phase4-modes.t b/tests/modsecurity-phase4-modes.t index 6462facc..859a1f20 100644 --- a/tests/modsecurity-phase4-modes.t +++ b/tests/modsecurity-phase4-modes.t @@ -59,7 +59,7 @@ $t->write_file('/m', 'Hello minimal'); $t->write_file('/s', 'Hello safe'); $t->write_file('/x', 'Hello strict'); $t->run(); -$t->plan(10); +$t->plan(12); like(http_get('/m'), qr/Hello minimal/, 'minimal no fake deny'); like(http_get('/s'), qr/Hello safe/, 'safe no fake deny'); @@ -71,5 +71,8 @@ like($log, qr/"reason":"mode_safe"|"reason":"headers_already_sent"/, 'safe reaso like($log, qr/"actual_action":"connection_abort"/, 'strict action logged'); like($log, qr/"event":"phase4_intervention"/, 'event field present'); like($log, qr/"header_sent":true/, 'json boolean header_sent'); -unlike($log, qr/\n.*\n/s, 'single-line json entries'); unlike($log, qr/Hello minimal|Hello safe|Hello strict/, 'no response body data in phase4 log'); + +my @lines = grep { length $_ } split /\n/, $log; +ok(@lines >= 1, 'phase4 log has one or more json lines'); +ok(!grep { $_ !~ /^\{.*\}$/ } @lines, 'each log line is a single JSON object line'); diff --git a/tests/modsecurity-proxy.t b/tests/modsecurity-proxy.t index 38c27d93..a742029f 100644 --- a/tests/modsecurity-proxy.t +++ b/tests/modsecurity-proxy.t @@ -192,6 +192,15 @@ EOF select undef, undef, undef, 0.1; print $client 'AND-THIS'; + } elsif ($uri =~ m{^/phase4\?what=(redirect301|redirect302|block401|block403)$}) { + print $client <<"EOF"; +HTTP/1.1 200 OK +Content-Type: text/html +Connection: close + +phase4 trigger +EOF + } else { print $client <<"EOF"; From d73f9eab3fc8b7f12d68c25f0f6a0df07ac84a4f Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 6 May 2026 13:16:18 +0200 Subject: [PATCH 09/40] Fix phase4-modes test plan count mismatch --- tests/modsecurity-phase4-modes.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modsecurity-phase4-modes.t b/tests/modsecurity-phase4-modes.t index 859a1f20..6763d177 100644 --- a/tests/modsecurity-phase4-modes.t +++ b/tests/modsecurity-phase4-modes.t @@ -59,7 +59,7 @@ $t->write_file('/m', 'Hello minimal'); $t->write_file('/s', 'Hello safe'); $t->write_file('/x', 'Hello strict'); $t->run(); -$t->plan(12); +$t->plan(11); like(http_get('/m'), qr/Hello minimal/, 'minimal no fake deny'); like(http_get('/s'), qr/Hello safe/, 'safe no fake deny'); From a7bb6e4119c540bee39c28b58d70aee3c1a24339 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 6 May 2026 13:24:23 +0200 Subject: [PATCH 10/40] docs: expand phase4 guidance and generalize production examples --- docs/examples/phase4-content-types.conf | 15 +- docs/examples/phase4-minimal.conf | 43 +++-- docs/examples/phase4-safe.conf | 41 ++-- docs/examples/phase4-strict.conf | 41 ++-- docs/phase4-handling.de.md | 245 ++++++++++++++++-------- docs/phase4-handling.en.md | 245 ++++++++++++++++-------- 6 files changed, 433 insertions(+), 197 deletions(-) diff --git a/docs/examples/phase4-content-types.conf b/docs/examples/phase4-content-types.conf index 4bf8701b..5baa48ba 100644 --- a/docs/examples/phase4-content-types.conf +++ b/docs/examples/phase4-content-types.conf @@ -1,5 +1,16 @@ -# One content type per line; comments are allowed. +# phase4-content-types.conf +# +# Format rules: +# 1) Exactly one MIME type per line. +# 2) Lines starting with # are comments. +# 3) Wildcards are not supported (e.g. text/* is invalid). +# 4) Parameters (e.g. "; charset=utf-8") do not need separate entries. +# The module normalizes response Content-Type and compares the base type. +# 5) Keep this list focused on body types you actually want to scope. + text/html +text/plain application/json application/problem+json -text/plain +application/xml +application/xhtml+xml diff --git a/docs/examples/phase4-minimal.conf b/docs/examples/phase4-minimal.conf index 35d78ce8..6d3250e8 100644 --- a/docs/examples/phase4-minimal.conf +++ b/docs/examples/phase4-minimal.conf @@ -1,14 +1,31 @@ -# Example: minimal mode -location /phase4 { - modsecurity on; - modsecurity_phase4_mode minimal; - modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; - - modsecurity_rules ' - SecRuleEngine On - SecResponseBodyAccess On - SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940001,phase:4,deny,log,status:403" - '; - - proxy_pass http://backend; +# Production example: phase-4 mode "minimal" +# Use when response continuity is more important than forced connection aborts +# for late phase:4 interventions (after headers were already sent). + +http { + upstream app_backend { + server 127.0.0.1:8081; + } + + server { + listen 80; + server_name example.local; + + modsecurity on; + modsecurity_phase4_mode minimal; + modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; + modsecurity_phase4_content_types_file /etc/modsecurity/phase4-content-types.conf; + + # Inline rule sample for demonstration. + # In production, you can keep using modsecurity_rules_file as usual. + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940101,phase:4,deny,log,status:403" + '; + + location / { + proxy_pass http://app_backend; + } + } } diff --git a/docs/examples/phase4-safe.conf b/docs/examples/phase4-safe.conf index 851a09ea..baaa7a08 100644 --- a/docs/examples/phase4-safe.conf +++ b/docs/examples/phase4-safe.conf @@ -1,14 +1,29 @@ -# Example: safe mode -location /phase4 { - modsecurity on; - modsecurity_phase4_mode safe; - modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; - - modsecurity_rules ' - SecRuleEngine On - SecResponseBodyAccess On - SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940002,phase:4,deny,log,status:403" - '; - - proxy_pass http://backend; +# Production example: phase-4 mode "safe" +# Default mode in this module. Recommended baseline if you need phase:4 +# visibility while avoiding forced disconnects on late interventions. + +http { + upstream app_backend { + server 127.0.0.1:8081; + } + + server { + listen 80; + server_name example.local; + + modsecurity on; + modsecurity_phase4_mode safe; + modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; + modsecurity_phase4_content_types_file /etc/modsecurity/phase4-content-types.conf; + + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940102,phase:4,deny,log,status:403" + '; + + location / { + proxy_pass http://app_backend; + } + } } diff --git a/docs/examples/phase4-strict.conf b/docs/examples/phase4-strict.conf index 80d6c882..68fb18ec 100644 --- a/docs/examples/phase4-strict.conf +++ b/docs/examples/phase4-strict.conf @@ -1,14 +1,29 @@ -# Example: strict mode -location /phase4 { - modsecurity on; - modsecurity_phase4_mode strict; - modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; - - modsecurity_rules ' - SecRuleEngine On - SecResponseBodyAccess On - SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940003,phase:4,deny,log,status:403" - '; - - proxy_pass http://backend; +# Production example: phase-4 mode "strict" +# Use only if connection aborts are operationally acceptable, because +# late phase:4 interventions may terminate the connection. + +http { + upstream app_backend { + server 127.0.0.1:8081; + } + + server { + listen 80; + server_name example.local; + + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; + modsecurity_phase4_content_types_file /etc/modsecurity/phase4-content-types.conf; + + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940103,phase:4,deny,log,status:403" + '; + + location / { + proxy_pass http://app_backend; + } + } } diff --git a/docs/phase4-handling.de.md b/docs/phase4-handling.de.md index 1cb42cd7..78799869 100644 --- a/docs/phase4-handling.de.md +++ b/docs/phase4-handling.de.md @@ -1,32 +1,62 @@ # ModSecurity-nginx: Phase-4-Handling (Deutsch) -## Einführung: Was ist das `phase:4`-Problem? +## Zweck dieses Dokuments -`phase:4`-Regeln laufen während der Verarbeitung des **Response-Bodys**. In nginx kann zu diesem Zeitpunkt der Response-Header bereits an den Client gesendet worden sein. Wenn der Header bereits gesendet wurde, kann nginx den HTTP-Status (z. B. 403) oder Redirect-Header nicht mehr zuverlässig umstellen. +Dieses Dokument beschreibt die im aktuellen Code implementierte Phase-4-Behandlung im nginx-Modul, inklusive: -In diesem Repository ist das explizit durch neue Phase-4-Mechanismen adressiert, inklusive Modi und strukturiertem Logging. +- technischer Grenzen bei bereits gesendeten Headern, +- Verhalten der Modi `minimal`, `safe`, `strict`, +- Content-Type-Scoping, +- Logging (`modsecurity_phase4_log`) und Sicherheitsaspekte, +- Abgrenzung zwischen **Produktionskonfiguration** und **Test-/Demo-Verhalten**. -## Warum „Header bereits gesendet“ problematisch ist +Es werden nur Aussagen getroffen, die durch den aktuellen Repository-Stand belegt sind. -Wenn eine ModSecurity-Intervention (`deny`, `status`, `redirect`) erst in `phase:4` auftritt, ist ein klassisches Blocken/Redirect nur möglich, solange Header noch nicht gesendet wurden. +--- -Wenn Header bereits gesendet sind: +## 1) Hintergrund: Request- vs. Response-Phasen -- ein späteres `status:401/403` ist nicht mehr sauber als HTTP-Status umsetzbar, -- ein späteres `redirect` (301/302 + `Location`) ist nicht mehr sauber als Redirect umsetzbar, -- nur degradierte Reaktion ist möglich (loggen, ggf. Verbindung abbrechen). +ModSecurity-Regeln laufen in unterschiedlichen Phasen der Transaktion. -## Warum **kein globales Response-Body-Buffering** genutzt wird +- Frühe Phasen (z. B. Request-Phasen) treffen Entscheidungen, bevor eine Antwort gesendet wird. +- `phase:4` gehört zur **Response-Body-Verarbeitung**. -Der implementierte Ansatz vermeidet globales Buffering großer Response-Bodies. Das ist konsistent mit folgenden Zielen: +Das ist ein zentraler Unterschied: In `phase:4` kann nginx bereits begonnen haben, Header und/oder Body zu senden. -- kein pauschaler Speicher-/Latenz-Overhead für alle Responses, -- keine zusätzliche Reordering-Logik zwischen bereits gesendeten Headern und späteren Body-Entscheidungen, -- keine Schein-Garantie, dass `phase:4` immer in saubere HTTP-Block-/Redirect-Semantik überführt werden kann. +### Warum ist das relevant? -## Neue Directives +Wenn eine Regel in `phase:4` eine Intervention wie `deny` oder `redirect` auslöst, ist ein sauberer Wechsel auf neuen HTTP-Status nur möglich, solange Header noch nicht final gesendet wurden. -### `modsecurity_phase4_mode` +--- + +## 2) Was `phase:4` praktisch bedeutet + +`phase:4`-Regeln prüfen den Antwortinhalt (Response-Body). Das ist nützlich, wenn Sicherheitskriterien erst am Antwortinhalt erkennbar sind. + +Gleichzeitig erzeugt es eine harte technische Grenze: + +- **Vor Header-Versand**: Status/Redirect kann noch sauber gesetzt werden. +- **Nach Header-Versand**: Status/Redirect lässt sich nicht zuverlässig „nachträglich“ korrigieren. + +Daher existiert im Modul ein dediziertes Handling für späte Interventionen. + +--- + +## 3) Warum Header in `phase:4` bereits gesendet sein können + +In nginx werden Antworten als Stream verarbeitet. Je nach Upstream-, Filter- und Buffering-Verhalten kann der Header bereits auf dem Socket sein, bevor die vollständige Body-Prüfung abgeschlossen ist. + +Folge: Ein in `phase:4` erkanntes `status:403` oder `redirect:302` kann nicht garantiert als sauberer HTTP-Status beim Client landen. + +> Keine falsche Garantie: `phase:4` kann nach gesendeten Headern keinen sauberen HTTP-Status mehr garantieren. + +--- + +## 4) Neue Directives im Modul + +## `modsecurity_phase4_mode` + +Konfiguriert das Verhalten bei Phase-4-Interventionen. Unterstützte Werte: @@ -34,129 +64,188 @@ Unterstützte Werte: - `safe` - `strict` -Ungültige Werte werden in der Konfiguration abgelehnt. +Ungültige Werte führen zu Konfigurationsfehlern. + +## `modsecurity_phase4_content_types_file` + +Lädt erlaubte/gescopte Content-Types aus einer Datei. -### `modsecurity_phase4_content_types_file` +- eine Zeile pro Typ, +- Kommentare mit `#`, +- Einträge werden validiert, +- Wildcards (`*`) sind nicht erlaubt. -Lädt Content-Types aus einer Datei (eine Zeile pro Typ, `#`-Kommentare erlaubt). Einträge werden validiert; ungültige Einträge führen zu Konfigurationsfehlern. +Wenn nicht gesetzt, nutzt das Modul Default-Typen. -Wenn keine Datei gesetzt ist, werden Standardtypen verwendet. +## `modsecurity_phase4_log` -### `modsecurity_phase4_log` +Aktiviert dediziertes JSON-Line-Logging für Phase-4-Ereignisse. -Aktiviert ein separates JSON-Line-Log für Phase-4-Interventionen. +--- -## Modi: Verhalten und Sicherheitsprofil +## 5) Modusverhalten (`minimal`, `safe`, `strict`) ## `minimal` -- Bei `phase:4`-Intervention nach gesendeten Headern: **kein künstliches Blocken**, nur `log_only`. -- Ziel: minimalinvasiv, kein Verbindungsabbruch. +Ziel: möglichst wenig Eingriff in laufende Responses. + +Bei Intervention nach gesendeten Headern: + +- Aktion wird auf `log_only` degradiert, +- keine erzwungene Verbindungsbeendigung. + +Sinnvoll, wenn Stabilität und vollständige Antwortauslieferung Vorrang haben. ## `safe` -- Verhalten bei spät erkannter Intervention ebenfalls `log_only`. -- Ziel: stabiler Betrieb ohne erzwungenen Abbruch. +Ziel: konservativer Produktionsstandard (Default-Merge-Verhalten im Modul). + +Bei Intervention nach gesendeten Headern: + +- ebenfalls `log_only`. + +Sinnvoll als Standardmodus, wenn Phase-4-Transparenz gewünscht ist, aber Verbindungsabbrüche vermieden werden sollen. ## `strict` -- Bei `phase:4`-Intervention nach gesendeten Headern: `connection_abort`. -- Ziel: striktere Reaktion, wenn keine saubere Status-/Redirect-Änderung mehr möglich ist. +Ziel: strengere Reaktion, wenn keine saubere Statusänderung mehr möglich ist. + +Bei Intervention nach gesendeten Headern: + +- `connection_abort`. + +### Risiken von `strict` + +- Kann aktive Verbindungen abbrechen. +- Liefert nicht garantiert einen „schönen“ HTTP-Fehlerstatus beim Client. +- Kann je nach Client/Proxy als Transportfehler statt als 4xx/3xx sichtbar werden. -> Wichtig: `strict` garantiert **keinen** nachträglichen 401/403/301/302-Status; es kann stattdessen zum Verbindungsabbruch kommen. +`strict` ist daher nur sinnvoll, wenn diese Nebenwirkungen betrieblich akzeptabel sind. -## Verhalten nach Header-Status +--- -### Header **noch nicht** gesendet +## 6) Verhalten nach Header-Status -Normale ModSecurity-Interventionspfade bleiben möglich (z. B. `deny_status`), weil nginx den Header noch anpassen kann. +## Header noch **nicht** gesendet -### Header **bereits** gesendet +Wenn eine Intervention vor Header-Versand finalisiert wird, bleibt normaler Deny-/Statuspfad möglich (im Code als `deny_status` geloggt). -- `minimal`/`safe`: `log_only` +## Header bereits gesendet + +- `minimal`: `log_only` +- `safe`: `log_only` - `strict`: `connection_abort` -## Erklärung der Actions +Das ist eine bewusste Degradierung, um keine falschen Status-Garantien vorzutäuschen. + +--- + +## 7) Warum **kein globales Response-Body-Buffering** genutzt wird + +Ein globales Buffering aller Responses würde zwar den Eingriffszeitpunkt verschieben, bringt aber technische und betriebliche Kosten: + +- zusätzlicher Speicher- und Latenz-Overhead, +- größere Komplexität für allgemeine Response-Pfade, +- Risiko unbeabsichtigter Nebenwirkungen auf Durchsatz/Stabilität. -### `connection_abort` +Der aktuelle Ansatz versucht stattdessen, late Interventionen transparent und kontrolliert zu behandeln (`log_only` oder `connection_abort`). -- Implementierter Fallback im `strict`-Modus bei spät erkannter `phase:4`-Intervention. -- Technisch: Request wird mit Fehlerpfad beendet (keine nachträgliche Header-Umschreibung). +--- -### `log_only` +## 8) Warum kein `ngx_chain_t`-Reordering/Rewriting genutzt wird -- Intervention wird protokolliert, aber Response-Fluss bleibt (so weit möglich) bestehen. -- Dient Nachvollziehbarkeit ohne erzwungenen Transportabbruch. +Das Modul implementiert **keine** künstliche Reordering-Logik auf bereits laufenden Body-Ketten, um nachträglich „doch noch“ andere Header-/Statussemantik zu erzwingen. -## Content-Type-Scoping (`modsecurity_phase4_content_types_file`) +Begründung: -Phase-4-Handling wird über Content-Type eingeschränkt. Wenn `Content-Type` fehlt oder nicht in Scope ist, wird statt harter Aktion protokolliert (`log_only`, z. B. Grund `content_type_missing` / `content_type_not_in_scope`). +- hohe Komplexität, +- erhöhte Fehleranfälligkeit, +- schwierige Garantien über korrektes Verhalten in allen Filter-/Upstream-Kombinationen. -Das reduziert unerwartete Effekte auf nicht-zielgerichtete Antworttypen. +Die dokumentierte Degradierung ist technisch robuster als scheinbar harte, aber unsichere Nachkorrekturen. -## Logging +--- -### `modsecurity_phase4_log` +## 9) Content-Type-Scoping: Bedeutung und sichere Nutzung -Schreibt JSON-Zeilen mit Feldern wie: +`modsecurity_phase4_content_types_file` begrenzt Phase-4-Sonderbehandlung auf definierte MIME-Typen. + +Wenn `Content-Type` fehlt oder nicht im Scope liegt: + +- Ereignis wird als `log_only` dokumentiert, +- Grund ist typischerweise `content_type_missing` oder `content_type_not_in_scope`. + +### Warum wichtig? + +- reduziert Nebeneffekte auf nicht-zielgerichtete Antworttypen, +- macht Verhalten vorhersagbarer, +- zwingt zu expliziter Auswahl relevanter Content-Typen. + +--- + +## 10) Logging-Format und Sicherheitsgrenzen + +Mit `modsecurity_phase4_log` schreibt das Modul JSON-Zeilen, u. a. mit: - `event` (`phase4_intervention`) - `uri`, `method` - `response_status`, `waf_status` - `content_type` -- `header_sent` (Boolean) +- `header_sent` - `mode` - `wanted_action`, `actual_action` - `reason` - `intervention` - `rule_id` -### `nginx error.log` +Zusätzlich kann im nginx `error.log` ein Hinweis erscheinen (insbesondere im `strict`-Pfad bei Headern, die schon gesendet wurden). -Zusätzlich wird insbesondere im `strict`-Pfad eine Warnung ins nginx-Error-Log geschrieben, wenn nach gesendeten Headern eine Intervention eintritt. +### Sicherheitsgrenze im Logging -## Sicherheitsaspekte (Implementierungsentscheidungen) +Die Tests prüfen explizit, dass keine Response-Body-Inhalte in das Phase-4-Log „durchrutschen“. -- **Keine Response-Bodies im Phase-4-Log**: reduziert Risiko von Datenabfluss über Logs. -- **Keine `ngx_chain_t`-Rewrites**: vermeidet komplexe, fehleranfällige Manipulation bereits laufender Body-Ketten. -- **Keine künstliche Reordering-Logik**: verhindert inkonsistente Zustände zwischen bereits gesendeten Headern und späteren Blockentscheidungen. -- **`strict` kann Verbindung abbrechen**: bewusster Trade-off für „fail closed“-ähnliche Reaktion ohne falsche HTTP-Status-Versprechen. +--- -## Beispielkonfigurationen +## 11) Produktionsbeispiele -Siehe: +Allgemeine (nicht testpfadgebundene) Beispielkonfigurationen: - `docs/examples/phase4-minimal.conf` - `docs/examples/phase4-safe.conf` - `docs/examples/phase4-strict.conf` - `docs/examples/phase4-content-types.conf` -## JSON-Log-Beispiele +Diese Beispiele nutzen `http`/`server`/`location /` und keine test-spezifischen `/phase4`-Pfade. + +--- + +## 12) Test-/Demo-Verhalten (klar getrennt) + +Im Repository existieren Testfälle mit `/phase4`-Pfaden, z. B. in: -`log_only` (z. B. `safe`): +- `tests/modsecurity.t` +- `tests/modsecurity-proxy.t` +- `tests/modsecurity-h2.t` +- `tests/modsecurity-proxy-h2.t` +- `tests/modsecurity-phase4-*.t` -```json -{"event":"phase4_intervention","uri":"/phase4","method":"GET","response_status":200,"waf_status":403,"content_type":"text/html","header_sent":true,"mode":"safe","wanted_action":"deny","actual_action":"log_only","reason":"mode_safe","intervention":"...","rule_id":"910002"} -``` +Diese Pfade sind **Testkontext** und nicht als allgemeine Produktionskonfiguration gedacht. -`connection_abort` (`strict`): +--- -```json -{"event":"phase4_intervention","uri":"/phase4","method":"GET","response_status":200,"waf_status":403,"content_type":"text/html","header_sent":true,"mode":"strict","wanted_action":"deny","actual_action":"connection_abort","reason":"headers_already_sent","intervention":"...","rule_id":"910003"} -``` +## 13) Bekannte Grenzen / keine falschen Versprechen -> Feldwerte hängen vom konkreten Request/Rule-Match ab. +- `phase:4` kann **nicht garantieren**, dass ein gewünschter 301/302/401/403-Status nach Header-Versand noch sauber beim Client ankommt. +- Bei späten Interventionen ist nur degradierte Behandlung möglich (`log_only` oder `connection_abort`). +- `strict` bedeutet nicht „garantierter 403“, sondern kann Verbindungsabbruch bedeuten. -## Einschränkungen / bekannte Grenzen +--- -- `phase:4` kann **nicht** garantieren, dass ein gewünschter Block-/Redirect-Status noch als HTTP-Status beim Client ankommt. -- Bei bereits gesendeten Headern ist nur degradierte Reaktion möglich (`log_only` oder `connection_abort`). -- Content-Type-Scoping ist entscheidend; Out-of-Scope-Responses werden nicht „hart“ behandelt. +## 14) Betriebsleitfaden (kurz) -## Hinweise für Betreiber +1. Harte Block-/Redirect-Entscheidungen möglichst in frühere Phasen legen. +2. `safe` als Ausgangspunkt verwenden, wenn keine klare Notwendigkeit für `strict` besteht. +3. `strict` nur einsetzen, wenn Abbrüche tolerierbar und beobachtbar sind. +4. `modsecurity_phase4_log` aktivieren und auf `actual_action`/`reason` auswerten. +5. Content-Type-Liste eng halten und regelmäßig prüfen. -1. `phase:4` nicht als alleinigen Mechanismus für harte Zugriffskontrolle planen. -2. Für harte Block-/Redirect-Entscheidungen möglichst frühere Phasen bevorzugen. -3. `modsecurity_phase4_log` aktivieren und regelmäßig auswerten. -4. `modsecurity_phase4_content_types_file` bewusst auf sensible MIME-Typen begrenzen. -5. `strict` nur einsetzen, wenn Verbindungsabbrüche betrieblich akzeptabel sind. diff --git a/docs/phase4-handling.en.md b/docs/phase4-handling.en.md index 600fcbcb..7e5e1a69 100644 --- a/docs/phase4-handling.en.md +++ b/docs/phase4-handling.en.md @@ -1,32 +1,62 @@ # ModSecurity-nginx: Phase 4 Handling (English) -## Introduction: what is the `phase:4` problem? +## Purpose of this document -`phase:4` rules run while processing the **response body**. In nginx, response headers may already have been sent to the client at that point. Once headers are sent, nginx cannot reliably switch HTTP status (for example to 403) or inject redirect headers anymore. +This document describes the currently implemented Phase 4 behavior in the nginx module, including: -This repository adds explicit Phase 4 handling for that condition, including dedicated modes and structured logging. +- technical limits once headers are already sent, +- behavior of `minimal`, `safe`, and `strict` modes, +- content-type scoping, +- logging (`modsecurity_phase4_log`) and security boundaries, +- a clear split between **production configuration** and **test/demo behavior**. -## Why “headers already sent” is a problem +Only statements supported by the current repository code and tests are included. -If a ModSecurity intervention (`deny`, `status`, `redirect`) is triggered in `phase:4`, classic blocking/redirecting only works when headers are still unsent. +--- -If headers are already sent: +## 1) Background: request vs response phases -- late `status:401/403` cannot be cleanly enforced as HTTP status, -- late redirect (`301/302` + `Location`) cannot be cleanly enforced, -- only degraded reactions remain (logging, optionally aborting the connection). +ModSecurity rules run in multiple transaction phases. -## Why there is **no global response-body buffering** +- Early phases (for example request phases) make decisions before a response is emitted. +- `phase:4` belongs to **response-body processing**. -The implementation intentionally avoids global buffering of large response bodies. This aligns with: +This difference is critical: during `phase:4`, nginx may already have sent headers and/or part of the body. -- no blanket memory/latency overhead for all responses, -- no extra reordering logic between already-sent headers and late body-time decisions, -- no false guarantee that `phase:4` can always be translated to clean HTTP block/redirect semantics. +### Why this matters -## New directives +If a `phase:4` rule triggers an intervention (`deny`, `status`, `redirect`), a clean status/redirect rewrite is only possible while headers are still unsent. -### `modsecurity_phase4_mode` +--- + +## 2) What `phase:4` means operationally + +`phase:4` rules inspect response body content. This is useful when security signals are visible only in outgoing payload. + +At the same time, this creates a hard technical boundary: + +- **Before headers are sent**: status/redirect can still be applied cleanly. +- **After headers are sent**: status/redirect can no longer be reliably changed. + +Therefore the module includes dedicated late-intervention handling. + +--- + +## 3) Why headers may already be sent in `phase:4` + +nginx processes responses as a stream. Depending on upstream behavior, buffering, and filter timing, headers can already be on the wire before body inspection fully completes. + +Result: a `phase:4` `status:403` or `redirect:302` is not guaranteed to become a clean client-visible HTTP status. + +> No false guarantee: after headers are sent, `phase:4` cannot guarantee a clean HTTP status rewrite. + +--- + +## 4) New directives in this module + +## `modsecurity_phase4_mode` + +Configures phase-4 late-intervention behavior. Supported values: @@ -34,129 +64,188 @@ Supported values: - `safe` - `strict` -Invalid values are rejected at config parsing time. +Invalid values are rejected by configuration parsing. + +## `modsecurity_phase4_content_types_file` + +Loads scoped content types from a file. -### `modsecurity_phase4_content_types_file` +- one MIME type per line, +- `#` comments supported, +- entries are validated, +- wildcards (`*`) are rejected. -Loads content types from a file (one type per line, `#` comments supported). Entries are validated; invalid entries fail configuration loading. +If unset, module defaults are used. -If not set, built-in default content types are used. +## `modsecurity_phase4_log` -### `modsecurity_phase4_log` +Enables dedicated JSON-lines logging for Phase 4 events. -Enables a dedicated JSON-lines log for Phase 4 interventions. +--- -## Modes: behavior and safety profile +## 5) Mode behavior (`minimal`, `safe`, `strict`) ## `minimal` -- For `phase:4` interventions after headers are sent: no synthetic deny, `log_only`. -- Goal: least intrusive behavior, no forced disconnect. +Goal: least intrusive behavior. + +For interventions after headers were sent: + +- action is downgraded to `log_only`, +- no forced connection termination. + +Use when delivery continuity is prioritized. ## `safe` -- For late interventions, behavior is also `log_only`. -- Goal: operational stability without forced abort. +Goal: conservative production baseline (module default merge behavior). + +For late interventions: + +- also downgraded to `log_only`. + +Use as default when you want phase:4 visibility without forced disconnect side effects. ## `strict` -- For `phase:4` interventions after headers are sent: `connection_abort`. -- Goal: stricter handling when clean status/redirect changes are no longer possible. +Goal: stricter fallback once clean status rewrite is no longer possible. + +For interventions after headers were sent: + +- `connection_abort`. + +### Risks of `strict` + +- Active connections may terminate. +- Clients/proxies may observe transport interruption rather than a clean 4xx/3xx response. +- It does **not** mean “guaranteed 403/401/301/302”. -> Important: `strict` does **not** guarantee a retroactive 401/403/301/302; it may terminate the connection instead. +Use only when those trade-offs are acceptable. -## Behavior by header state +--- -### Headers **not sent yet** +## 6) Behavior by header state -Normal ModSecurity intervention paths remain possible (for example `deny_status`), because nginx can still adjust outgoing headers. +## Headers **not sent yet** -### Headers **already sent** +If intervention is finalized before header send, normal deny/status behavior remains possible (logged as `deny_status` in code path). -- `minimal`/`safe`: `log_only` +## Headers **already sent** + +- `minimal`: `log_only` +- `safe`: `log_only` - `strict`: `connection_abort` -## Action semantics +This is an intentional downgrade to avoid false status guarantees. + +--- + +## 7) Why there is **no global response-body buffering** + +Global buffering of all responses could delay decision points, but introduces broad costs: + +- additional memory and latency overhead, +- higher complexity in generic response paths, +- increased risk of throughput/stability side effects. -### `connection_abort` +Current implementation instead makes late interventions explicit and controlled (`log_only` or `connection_abort`). -- Implemented fallback in `strict` mode for late `phase:4` interventions. -- Technical effect: request terminates via error path (no post-hoc header rewrite). +--- -### `log_only` +## 8) Why there is no `ngx_chain_t` reordering/rewriting -- Intervention is recorded, while response flow continues as far as possible. -- Provides traceability without forced transport interruption. +The module does **not** implement synthetic reordering of already flowing body chains to force post-hoc status semantics. -## Content-Type scoping (`modsecurity_phase4_content_types_file`) +Reasoning: -Phase 4 handling is scoped by content type. If `Content-Type` is missing or not in scope, the module logs a degraded action (`log_only`, with reasons such as `content_type_missing` / `content_type_not_in_scope`) instead of enforcing a hard action. +- high implementation complexity, +- higher fragility, +- difficult correctness guarantees across all filter/upstream combinations. -This limits side effects on non-targeted response types. +The documented downgrade model is more robust than pretending hard guarantees. -## Logging +--- -### `modsecurity_phase4_log` +## 9) Content-type scoping: meaning and safe usage -Writes JSON lines with fields such as: +`modsecurity_phase4_content_types_file` limits special phase-4 handling to selected MIME types. + +If `Content-Type` is missing or out of scope: + +- action is logged as `log_only`, +- reason is typically `content_type_missing` or `content_type_not_in_scope`. + +### Why this matters + +- reduces side effects on non-target response types, +- improves predictability, +- enforces explicit operator intent. + +--- + +## 10) Logging format and security boundaries + +With `modsecurity_phase4_log`, the module emits JSON lines including fields such as: - `event` (`phase4_intervention`) - `uri`, `method` - `response_status`, `waf_status` - `content_type` -- `header_sent` (boolean) +- `header_sent` - `mode` - `wanted_action`, `actual_action` - `reason` - `intervention` - `rule_id` -### nginx `error.log` +Additionally, nginx `error.log` may contain warnings (especially on strict late-intervention paths). -Additionally, especially on the `strict` path, a warning is emitted to nginx error log when an intervention occurs after headers are sent. +### Logging boundary -## Security decisions (implementation) +Tests explicitly check that response body payload is not leaked into phase-4 log output. -- **No response body in Phase 4 log**: reduces leakage risk through logs. -- **No `ngx_chain_t` rewriting**: avoids brittle low-level mutations of already flowing body chains. -- **No synthetic reordering logic**: avoids inconsistent states between sent headers and late decisions. -- **`strict` may abort connections**: deliberate trade-off for stricter fail behavior without false HTTP-status guarantees. +--- -## Example configurations +## 11) Production examples -See: +General (non test-path-specific) example configurations: - `docs/examples/phase4-minimal.conf` - `docs/examples/phase4-safe.conf` - `docs/examples/phase4-strict.conf` - `docs/examples/phase4-content-types.conf` -## JSON log examples +These use `http` / `server` / `location /` patterns and avoid `/phase4` as a production example path. + +--- + +## 12) Test/demo behavior (explicitly separate) + +Repository tests include `/phase4` endpoints, for example in: -`log_only` (for example `safe`): +- `tests/modsecurity.t` +- `tests/modsecurity-proxy.t` +- `tests/modsecurity-h2.t` +- `tests/modsecurity-proxy-h2.t` +- `tests/modsecurity-phase4-*.t` -```json -{"event":"phase4_intervention","uri":"/phase4","method":"GET","response_status":200,"waf_status":403,"content_type":"text/html","header_sent":true,"mode":"safe","wanted_action":"deny","actual_action":"log_only","reason":"mode_safe","intervention":"...","rule_id":"910002"} -``` +Those paths are **test context**, not generic production guidance. -`connection_abort` (`strict`): +--- -```json -{"event":"phase4_intervention","uri":"/phase4","method":"GET","response_status":200,"waf_status":403,"content_type":"text/html","header_sent":true,"mode":"strict","wanted_action":"deny","actual_action":"connection_abort","reason":"headers_already_sent","intervention":"...","rule_id":"910003"} -``` +## 13) Known limits / no false promises -> Field values depend on the actual request and matched rule. +- `phase:4` cannot **guarantee** clean 301/302/401/403 delivery once headers are already sent. +- For late interventions, only downgraded handling is possible (`log_only` or `connection_abort`). +- `strict` is not “guaranteed block status”; it can mean connection termination. -## Limitations / known boundaries +--- -- `phase:4` cannot **guarantee** that the desired block/redirect status still reaches the client as HTTP status. -- When headers are already sent, only degraded handling is possible (`log_only` or `connection_abort`). -- Content-type scoping is critical; out-of-scope responses are not hard-enforced. +## 14) Operator checklist -## Operator guidance +1. Place hard block/redirect decisions in earlier phases whenever possible. +2. Start with `safe` unless you have a clear need for `strict`. +3. Use `strict` only when connection abort side effects are acceptable. +4. Enable `modsecurity_phase4_log` and monitor `actual_action` + `reason` fields. +5. Keep content-type scope narrow and review it regularly. -1. Do not treat `phase:4` as the sole hard access-control mechanism. -2. Prefer earlier phases for hard block/redirect decisions. -3. Enable and monitor `modsecurity_phase4_log`. -4. Keep `modsecurity_phase4_content_types_file` narrowly scoped to sensitive MIME types. -5. Use `strict` only if connection aborts are operationally acceptable. From 4775f328e8e6c96de227f8ad336a837ea69d0007 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 6 May 2026 13:26:47 +0200 Subject: [PATCH 11/40] Fix phase4-modes JSON line assertion for CRLF log lines --- tests/modsecurity-phase4-modes.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modsecurity-phase4-modes.t b/tests/modsecurity-phase4-modes.t index 6763d177..26484db7 100644 --- a/tests/modsecurity-phase4-modes.t +++ b/tests/modsecurity-phase4-modes.t @@ -75,4 +75,4 @@ unlike($log, qr/Hello minimal|Hello safe|Hello strict/, 'no response body data i my @lines = grep { length $_ } split /\n/, $log; ok(@lines >= 1, 'phase4 log has one or more json lines'); -ok(!grep { $_ !~ /^\{.*\}$/ } @lines, 'each log line is a single JSON object line'); +ok(!grep { $_ !~ /^\{.*\}\r?$/ } @lines, 'each log line is a single JSON object line'); From 1f37b84c921569968db88aa5fced219196db4c1a Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 6 May 2026 19:12:41 +0200 Subject: [PATCH 12/40] Add diagnostics for failing phase4 modes JSON-line assertion --- tests/modsecurity-phase4-modes.t | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/modsecurity-phase4-modes.t b/tests/modsecurity-phase4-modes.t index 26484db7..ff9798e1 100644 --- a/tests/modsecurity-phase4-modes.t +++ b/tests/modsecurity-phase4-modes.t @@ -75,4 +75,9 @@ unlike($log, qr/Hello minimal|Hello safe|Hello strict/, 'no response body data i my @lines = grep { length $_ } split /\n/, $log; ok(@lines >= 1, 'phase4 log has one or more json lines'); -ok(!grep { $_ !~ /^\{.*\}\r?$/ } @lines, 'each log line is a single JSON object line'); +my @bad_lines = grep { $_ !~ /^\{.*\}\r?$/ } @lines; +note('phase4.log lines=' . scalar(@lines)); +if (@bad_lines) { + diag('non-json-lines: ' . join(' || ', map { my $x = $_; $x =~ s/\r/\\r/g; $x } @bad_lines)); +} +ok(scalar(@bad_lines) == 0, 'each log line is a single JSON object line'); From 0e8720e8f293645055f3d10dae061a15719d23e2 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 7 May 2026 07:50:01 +0200 Subject: [PATCH 13/40] Fix MSVC pointer type warning in ngx_file_info call --- src/ngx_http_modsecurity_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index 22561485..8a8b0467 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -585,7 +585,7 @@ ngx_http_modsecurity_phase4_load_content_types_file(ngx_conf_t *cf, ngx_http_mod ngx_file_info_t fi; u_char *buf, *p, *line, *end; ssize_t n; - if (ngx_file_info((char *)path->data, &fi) == NGX_FILE_ERROR) { + if (ngx_file_info(path->data, &fi) == NGX_FILE_ERROR) { ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno, "modsecurity_phase4_content_types_file \"%V\" stat() failed", path); return NGX_CONF_ERROR; } From 2a31a5b37be0f6600e4390e1ec92817efc875082 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 13 May 2026 22:52:30 +0200 Subject: [PATCH 14/40] Refactor phase4 code to address Sonar maintainability issues --- src/ngx_http_modsecurity_body_filter.c | 27 +++++-- src/ngx_http_modsecurity_module.c | 105 +++++++++++++++++++------ 2 files changed, 101 insertions(+), 31 deletions(-) diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index 176cdcc6..50529c96 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -185,11 +185,12 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) } return ret; } - else if (ret < 0) { + if (ret < 0) { ret = ngx_http_modsecurity_phase4_handle_intervention(r, mcf); - if (ret == NGX_ERROR) return NGX_ERROR; + if (ret == NGX_ERROR) { + return NGX_ERROR; + } return ngx_http_next_body_filter(r, in); - } } } @@ -286,7 +287,8 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit static ngx_str_t ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in) { - ngx_str_t out; size_t i; + ngx_str_t out; + size_t i; u_char *semi; out = in; if (out.data == NULL || out.len == 0) return out; @@ -326,10 +328,21 @@ ngx_http_modsecurity_sanitize_intervention(ngx_pool_t *pool, ngx_str_t in) static void ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst) { - size_t i, extra = 0; u_char *d; + size_t i; + size_t extra = 0; + u_char *d; if (src == NULL || src->data == NULL) { dst->len=0; dst->data=(u_char*)""; return; } - for (i = 0; i < src->len; i++) if (src->data[i] < 0x20 || src->data[i] == '"' || src->data[i] == '\\') extra++; - dst->data = ngx_pnalloc(pool, src->len + extra + 1); if (dst->data == NULL) { dst->len=0; dst->data=(u_char*)""; return; } + for (i = 0; i < src->len; i++) { + if (src->data[i] < 0x20 || src->data[i] == '"' || src->data[i] == '\\') { + extra++; + } + } + dst->data = ngx_pnalloc(pool, src->len + extra + 1); + if (dst->data == NULL) { + dst->len = 0; + dst->data = (u_char *)""; + return; + } d = dst->data; for (i = 0; i < src->len; i++) { u_char c = src->data[i]; diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index 8a8b0467..39c09d95 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -545,7 +545,8 @@ ngx_conf_set_phase4_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) static ngx_int_t ngx_http_modsecurity_phase4_validate_content_type(u_char *s, size_t len) { - size_t i, slash = (size_t)-1; + size_t i; + size_t slash = (size_t)-1; if (len == 0 || ngx_strchr(s, '*') != NULL) return NGX_ERROR; for (i = 0; i < len; i++) { u_char c = s[i]; @@ -578,12 +579,69 @@ ngx_http_modsecurity_phase4_set_default_content_types(ngx_conf_t *cf, ngx_http_m return NGX_CONF_OK; } +static ngx_int_t +ngx_http_modsecurity_phase4_push_content_type(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf, + u_char *line, u_char *end, ngx_str_t *path) +{ + ngx_str_t *ct; + + ngx_strlow(line, line, end - line); + if (ngx_http_modsecurity_phase4_validate_content_type(line, end - line) != NGX_OK) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid content-type entry in modsecurity_phase4_content_types_file \"%V\": \"%s\"", path, line); + return NGX_ERROR; + } + + ct = ngx_array_push(mcf->phase4_content_types); + if (ct == NULL) { + return NGX_ERROR; + } + ct->len = end - line; + ct->data = ngx_pnalloc(cf->pool, ct->len); + if (ct->data == NULL) { + return NGX_ERROR; + } + ngx_memcpy(ct->data, line, ct->len); + + return NGX_OK; +} + +static void +ngx_http_modsecurity_phase4_trim_line(u_char **line, u_char **end) +{ + while (*line < *end && isspace((unsigned char)**line)) { + (*line)++; + } + + while (*end > *line && isspace((unsigned char)*((*end) - 1))) { + (*end)--; + } + + **end = '\0'; +} + +static void +ngx_http_modsecurity_phase4_strip_inline_comment(u_char *line) +{ + u_char *hash = (u_char *) ngx_strchr(line, '#'); + u_char *semi = (u_char *) ngx_strchr(line, ';'); + + if (hash && (!semi || hash < semi)) { + *hash = '\0'; + } + if (semi) { + *semi = '\0'; + } +} + static char * ngx_http_modsecurity_phase4_load_content_types_file(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf, ngx_str_t *path) { ngx_file_t file; ngx_file_info_t fi; - u_char *buf, *p, *line, *end; + u_char *buf; + u_char *p; + u_char *line; + u_char *end; ssize_t n; if (ngx_file_info(path->data, &fi) == NGX_FILE_ERROR) { ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno, "modsecurity_phase4_content_types_file \"%V\" stat() failed", path); @@ -592,7 +650,8 @@ ngx_http_modsecurity_phase4_load_content_types_file(ngx_conf_t *cf, ngx_http_mod buf = ngx_pnalloc(cf->pool, ngx_file_size(&fi) + 1); if (buf == NULL) return NGX_CONF_ERROR; ngx_memzero(&file, sizeof(file)); - file.name = *path; file.log = cf->log; + file.name = *path; + file.log = cf->log; file.fd = ngx_open_file(path->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); if (file.fd == NGX_INVALID_FILE) return NGX_CONF_ERROR; n = ngx_read_file(&file, buf, ngx_file_size(&fi), 0); @@ -603,27 +662,25 @@ ngx_http_modsecurity_phase4_load_content_types_file(ngx_conf_t *cf, ngx_http_mod if (mcf->phase4_content_types == NULL) return NGX_CONF_ERROR; for (p = buf, line = buf; p <= buf + n; p++) { if (p == buf + n || *p == '\n' || *p == '\r') { - *p = '\0'; end = p; - while (line < end && isspace((unsigned char)*line)) line++; - while (end > line && isspace((unsigned char)*(end-1))) *(--end) = '\0'; - if (line[0] && line[0] != '#') { - u_char *hash = (u_char *)ngx_strchr(line, '#'); - u_char *semi = (u_char *)ngx_strchr(line, ';'); - if (hash && (!semi || hash < semi)) *hash = '\0'; - if (semi) *semi = '\0'; - end = line + ngx_strlen(line); - while (end > line && isspace((unsigned char)*(end-1))) *(--end) = '\0'; - ngx_strlow(line, line, end - line); - if (ngx_http_modsecurity_phase4_validate_content_type(line, end-line) != NGX_OK) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid content-type entry in modsecurity_phase4_content_types_file \"%V\": \"%s\"", path, line); - return NGX_CONF_ERROR; - } - ngx_str_t *ct = ngx_array_push(mcf->phase4_content_types); - if (ct == NULL) return NGX_CONF_ERROR; - ct->len = end - line; - ct->data = ngx_pnalloc(cf->pool, ct->len); - if (ct->data == NULL) return NGX_CONF_ERROR; - ngx_memcpy(ct->data, line, ct->len); + *p = '\0'; + end = p; + + ngx_http_modsecurity_phase4_trim_line(&line, &end); + if (line[0] == '\0' || line[0] == '#') { + line = p + 1; + continue; + } + + ngx_http_modsecurity_phase4_strip_inline_comment(line); + end = line + ngx_strlen(line); + ngx_http_modsecurity_phase4_trim_line(&line, &end); + if (line[0] == '\0') { + line = p + 1; + continue; + } + + if (ngx_http_modsecurity_phase4_push_content_type(cf, mcf, line, end, path) != NGX_OK) { + return NGX_CONF_ERROR; } line = p + 1; } From 0d3ad622a001cf243dcf472a67ffd58ebc8a753f Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 13 May 2026 22:58:36 +0200 Subject: [PATCH 15/40] Flatten processed-chain branch in body filter --- src/ngx_http_modsecurity_body_filter.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index 50529c96..7c847dd5 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -168,7 +168,11 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) /* XXX: chain->buf->last_buf || chain->buf->last_in_chain */ is_request_processed = chain->buf->last_buf; - if (is_request_processed) { + if (!is_request_processed) { + continue; + } + + { ngx_pool_t *old_pool; old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); From 06f99afa29c5f57624b4b12700511edeff64efa9 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 13 May 2026 23:15:05 +0200 Subject: [PATCH 16/40] Fix remaining SonarQube issues in body filter --- src/ngx_http_modsecurity_body_filter.c | 38 ++++++++++++++++---------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index 7c847dd5..779211b4 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -254,7 +254,13 @@ static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason) { u_char *p; - ngx_str_t euri, emethod, ect, elog, erule, raw_log, slog; + ngx_str_t euri; + ngx_str_t emethod; + ngx_str_t ect; + ngx_str_t elog; + ngx_str_t erule; + ngx_str_t raw_log; + ngx_str_t slog; const char *mode = "safe"; const char *header_sent = r->header_sent ? "true" : "false"; ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); @@ -365,20 +371,22 @@ ngx_http_modsecurity_extract_rule_id(ngx_pool_t *pool, ngx_str_t *intervention, rule_id->len = 0; if (intervention == NULL || intervention->data == NULL) return; for (i = 0; i + 4 < intervention->len; i++) { - if (ngx_strncasecmp(intervention->data + i, (u_char *)"id \"", 4) == 0) { - size_t j = i + 4; - while (j < intervention->len && intervention->data[j] >= '0' && intervention->data[j] <= '9') j++; - if (j > i + 4 && j < intervention->len && intervention->data[j] == '"') { - rule_id->len = j - (i + 4); - rule_id->data = ngx_pnalloc(pool, rule_id->len); - if (rule_id->data == NULL) { - rule_id->len = 0; - rule_id->data = (u_char *)""; - return; - } - ngx_memcpy(rule_id->data, intervention->data + i + 4, rule_id->len); - return; - } + size_t j; + + if (ngx_strncasecmp(intervention->data + i, (u_char *)"id \"", 4) != 0) continue; + + j = i + 4; + while (j < intervention->len && intervention->data[j] >= '0' && intervention->data[j] <= '9') j++; + if (j <= i + 4 || j >= intervention->len || intervention->data[j] != '"') continue; + + rule_id->len = j - (i + 4); + rule_id->data = ngx_pnalloc(pool, rule_id->len); + if (rule_id->data == NULL) { + rule_id->len = 0; + rule_id->data = (u_char *)""; + return; } + ngx_memcpy(rule_id->data, intervention->data + i + 4, rule_id->len); + return; } } From 505d4af49fbff35769d6eb1c2419a1c57e09f818 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 14 May 2026 18:48:51 +0200 Subject: [PATCH 17/40] Address Copilot review feedback for phase4 handling --- src/ngx_http_modsecurity_body_filter.c | 17 ++++++++++---- src/ngx_http_modsecurity_common.h | 1 - src/ngx_http_modsecurity_module.c | 27 +++++++++-------------- tests/modsecurity-phase4-content-types.t | 2 +- tests/modsecurity-phase4-invalid-config.t | 5 +++-- tests/modsecurity-phase4-modes.t | 2 +- tests/modsecurity-proxy-h2.t | 9 ++++++++ 7 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index 779211b4..ece49a5c 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -212,21 +212,25 @@ ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, ngx_http_ { ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); ngx_int_t in_scope = ngx_http_modsecurity_phase4_in_scope(r); + const char *wanted = "deny"; + if (ctx && ctx->last_intervention_status >= 300 && ctx->last_intervention_status < 400) { + wanted = "redirect"; + } if (ctx && ctx->phase4_headers_checked) return NGX_OK; if (ctx) ctx->phase4_headers_checked = 1; if (in_scope == 0) { - ngx_http_modsecurity_phase4_log_event(r, mcf, "deny", "log_only", r->headers_out.content_type.len ? "content_type_not_in_scope" : "content_type_missing"); + ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, "log_only", r->headers_out.content_type.len ? "content_type_not_in_scope" : "content_type_missing"); return NGX_OK; } if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { - ngx_http_modsecurity_phase4_log_event(r, mcf, "deny", "connection_abort", "headers_already_sent"); + ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, "connection_abort", "headers_already_sent"); ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "modsecurity phase4 intervention after headers sent, action=connection_abort, uri=\"%V\"", &r->uri); r->connection->error = 1; return NGX_ERROR; } - ngx_http_modsecurity_phase4_log_event(r, mcf, "deny", "log_only", + ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, "log_only", mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL ? "mode_minimal" : "mode_safe"); return NGX_OK; } @@ -290,7 +294,12 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit p = ngx_snprintf(dbuf, need, "{\"event\":\"phase4_intervention\",\"uri\":\"%V\",\"method\":\"%V\",\"response_status\":%ui,\"waf_status\":%i,\"content_type\":\"%V\",\"header_sent\":%s,\"mode\":\"%s\",\"wanted_action\":\"%s\",\"actual_action\":\"%s\",\"reason\":\"%s\",\"intervention\":\"%V\",\"rule_id\":\"%V\"}\n", &euri,&emethod,(ngx_uint_t)r->headers_out.status,ctx ? (int) ctx->last_intervention_status : 0,&ect,header_sent,mode,wanted,actual,reason,&elog,&erule); - ngx_write_fd(mcf->phase4_log_file->fd, dbuf, p - dbuf); + ssize_t n = ngx_write_fd(mcf->phase4_log_file->fd, dbuf, p - dbuf); + if (n < 0 || (size_t) n != (size_t) (p - dbuf)) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, ngx_errno, + "modsecurity phase4 log write failed"); + return NGX_ERROR; + } return NGX_OK; } diff --git a/src/ngx_http_modsecurity_common.h b/src/ngx_http_modsecurity_common.h index b9a11f39..de9941cb 100644 --- a/src/ngx_http_modsecurity_common.h +++ b/src/ngx_http_modsecurity_common.h @@ -105,7 +105,6 @@ typedef struct { unsigned intervention_triggered:1; unsigned request_body_processed:1; unsigned phase4_headers_checked:1; - ngx_str_t phase4_content_type; ngx_str_t last_intervention_log; ngx_int_t last_intervention_status; } ngx_http_modsecurity_ctx_t; diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index 39c09d95..339ea8da 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -44,11 +44,6 @@ static char *ngx_http_modsecurity_phase4_set_default_content_types(ngx_conf_t *c static char *ngx_http_modsecurity_phase4_load_content_types_file(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf, ngx_str_t *path); static ngx_int_t ngx_http_modsecurity_phase4_validate_content_type(u_char *s, size_t len); -#define NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL 0 -#define NGX_HTTP_MODSEC_PHASE4_MODE_SAFE 1 -#define NGX_HTTP_MODSEC_PHASE4_MODE_STRICT 2 - - /* * PCRE malloc/free workaround, based on * https://github.com/openresty/lua-nginx-module/blob/master/src/ngx_http_lua_pcrefix.c @@ -174,7 +169,12 @@ ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_re ctx->last_intervention_status = intervention.status; ctx->last_intervention_log.len = 0; ctx->last_intervention_log.data = NULL; - if (intervention.log != NULL) { + mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + if (mcf == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + if (mcf->phase4_log_file != NULL && intervention.log != NULL) { size_t l = ngx_strlen(intervention.log); u_char *cp = ngx_pnalloc(r->pool, l + 1); if (cp != NULL) { @@ -182,19 +182,9 @@ ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_re cp[l] = '\0'; ctx->last_intervention_log.data = cp; ctx->last_intervention_log.len = l; - } else { - if (intervention.log != NULL) { - free(intervention.log); - } - return NGX_HTTP_INTERNAL_SERVER_ERROR; } } - mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); - if (mcf == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - // logging to nginx error log can be disable by setting `modsecurity_use_error_log` to off if (mcf->use_error_log) { log = intervention.log; @@ -653,7 +643,10 @@ ngx_http_modsecurity_phase4_load_content_types_file(ngx_conf_t *cf, ngx_http_mod file.name = *path; file.log = cf->log; file.fd = ngx_open_file(path->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); - if (file.fd == NGX_INVALID_FILE) return NGX_CONF_ERROR; + if (file.fd == NGX_INVALID_FILE) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno, "modsecurity_phase4_content_types_file \"%V\" open() failed", path); + return NGX_CONF_ERROR; + } n = ngx_read_file(&file, buf, ngx_file_size(&fi), 0); ngx_close_file(file.fd); if (n < 0) return NGX_CONF_ERROR; diff --git a/tests/modsecurity-phase4-content-types.t b/tests/modsecurity-phase4-content-types.t index 9ce9c35d..412e9fea 100644 --- a/tests/modsecurity-phase4-content-types.t +++ b/tests/modsecurity-phase4-content-types.t @@ -85,7 +85,7 @@ like(http_get('/unknown'), qr/HIT UNKNOWN/, 'unknown content-type not in scope = like(http_get('/emptytype'), qr/HIT EMPTY/, 'empty content-type => no hard action'); my $log = $t->read_file('phase4-content-types.log'); -like($log, qr/"content_type":"application\/json"|"content_type":"application\/json"/, 'json content type logged'); +like($log, qr/"content_type":"application\/json"/, 'json content type logged'); like($log, qr/"actual_action":"connection_abort"/, 'strict in-scope abort logged'); like($log, qr/"reason":"content_type_not_in_scope"/, 'out-of-scope reason logged'); like($log, qr/"event":"phase4_intervention"/, 'json lines event present'); diff --git a/tests/modsecurity-phase4-invalid-config.t b/tests/modsecurity-phase4-invalid-config.t index 176274a5..061cde6b 100644 --- a/tests/modsecurity-phase4-invalid-config.t +++ b/tests/modsecurity-phase4-invalid-config.t @@ -27,6 +27,7 @@ http { } EOC -# Expected: nginx config test/start should fail with clear config error -ok(1, 'manual verification test: start should fail for wildcard/invalid content-type entries'); +my $failed = eval { $t->run(); 1 }; +ok(!$failed, 'nginx startup fails for invalid phase4 content-type entries'); +like($@, qr/invalid content-type entry in modsecurity_phase4_content_types_file/, 'error points to invalid content-type entry'); done_testing(); diff --git a/tests/modsecurity-phase4-modes.t b/tests/modsecurity-phase4-modes.t index ff9798e1..99c2988c 100644 --- a/tests/modsecurity-phase4-modes.t +++ b/tests/modsecurity-phase4-modes.t @@ -67,7 +67,7 @@ is(http_get('/x'), '', 'strict abort after headers sent'); my $log = $t->read_file('phase4.log'); like($log, qr/"actual_action":"log_only"/, 'log_only present'); -like($log, qr/"reason":"mode_safe"|"reason":"headers_already_sent"/, 'safe reason present'); +like($log, qr/"reason":"mode_safe"/, 'safe reason present'); like($log, qr/"actual_action":"connection_abort"/, 'strict action logged'); like($log, qr/"event":"phase4_intervention"/, 'event field present'); like($log, qr/"header_sent":true/, 'json boolean header_sent'); diff --git a/tests/modsecurity-proxy-h2.t b/tests/modsecurity-proxy-h2.t index da0c9d53..df0e43ba 100644 --- a/tests/modsecurity-proxy-h2.t +++ b/tests/modsecurity-proxy-h2.t @@ -271,6 +271,15 @@ EOF select undef, undef, undef, 0.1; print $client 'AND-THIS'; + } elsif ($uri =~ m{^/phase4\?what=(redirect301|redirect302|block401|block403)$}) { + print $client <<"EOF"; +HTTP/1.1 200 OK +Content-Type: text/html +Connection: close + +phase4 trigger +EOF + } else { print $client <<"EOF"; From cfc7e226ab27bf287e054b34b32043cb97d3f00a Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 14 May 2026 19:04:23 +0200 Subject: [PATCH 18/40] Fix sanity-check redeclaration and narrow phase4 log capture --- src/ngx_http_modsecurity_body_filter.c | 4 ++-- src/ngx_http_modsecurity_module.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index ece49a5c..0b7223e8 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -47,8 +47,8 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) { ngx_chain_t *chain = in; ngx_http_modsecurity_ctx_t *ctx = NULL; -#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) ngx_http_modsecurity_conf_t *mcf; +#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) ngx_list_part_t *part = &r->headers_out.headers.part; ngx_table_elt_t *data = part->elts; ngx_uint_t i = 0; @@ -149,7 +149,7 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) #endif int is_request_processed = 0; - ngx_http_modsecurity_conf_t *mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); for (; chain != NULL; chain = chain->next) { u_char *data = chain->buf->pos; diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index 339ea8da..e397dfd1 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -174,7 +174,7 @@ ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_re return NGX_HTTP_INTERNAL_SERVER_ERROR; } - if (mcf->phase4_log_file != NULL && intervention.log != NULL) { + if (mcf->phase4_log_file != NULL && r->header_sent && intervention.log != NULL) { size_t l = ngx_strlen(intervention.log); u_char *cp = ngx_pnalloc(r->pool, l + 1); if (cp != NULL) { From 00519756540d1dbb9bd21ddac3307463363a9cb2 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 14 May 2026 19:04:28 +0200 Subject: [PATCH 19/40] Rewrite phase4 invalid-config test to use nginx -t output --- tests/modsecurity-phase4-invalid-config.t | 30 ++++++++++++----------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/modsecurity-phase4-invalid-config.t b/tests/modsecurity-phase4-invalid-config.t index 061cde6b..e37956d6 100644 --- a/tests/modsecurity-phase4-invalid-config.t +++ b/tests/modsecurity-phase4-invalid-config.t @@ -1,33 +1,35 @@ #!/usr/bin/perl use warnings; use strict; -use Test::More; +use Test::More tests => 1; BEGIN { use FindBin; chdir($FindBin::Bin); } use lib 'lib'; -use Test::Nginx; +use Test::Nginx::Socket -Base; -# this test verifies parser rejects invalid entries/wildcards at config load -my $t = Test::Nginx->new()->has(qw/http/); -$t->write_file('phase4-invalid.conf', "text/*\n*/json\ntext/html bad\napplication/problem+json\n"); +my $t = Test::Nginx::Socket->new()->has(qw/http/); +$t->write_file('phase4-invalid.conf', "text/*\n"); -$t->write_file_expand('nginx.conf', <<'EOC'); +$t->write_file_expand('nginx.conf', <<'EOF'); %%TEST_GLOBALS%% -daemon off; + events {} + http { %%TEST_GLOBALS_HTTP%% + server { - listen 127.0.0.1:8080; + listen 127.0.0.1:%%PORT%%; server_name localhost; location / { modsecurity on; modsecurity_phase4_content_types_file %%TESTDIR%%/phase4-invalid.conf; - return 200 "ok"; + return 200 "ok\n"; } } } -EOC +EOF + +my $cmd = "$t->{_testdir}/../nginx -p $t->{_testdir}/ -c nginx.conf -t 2>&1"; +my $out = `$cmd`; -my $failed = eval { $t->run(); 1 }; -ok(!$failed, 'nginx startup fails for invalid phase4 content-type entries'); -like($@, qr/invalid content-type entry in modsecurity_phase4_content_types_file/, 'error points to invalid content-type entry'); -done_testing(); +like($out, qr/invalid content-type entry in modsecurity_phase4_content_types_file/, + 'error points to invalid content-type entry'); From 329f269977b6f914d819751b659976011aeb02bc Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 14 May 2026 19:18:27 +0200 Subject: [PATCH 20/40] Use repository Test::Nginx harness in phase4 invalid-config test --- tests/modsecurity-phase4-invalid-config.t | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/modsecurity-phase4-invalid-config.t b/tests/modsecurity-phase4-invalid-config.t index e37956d6..c62f53b4 100644 --- a/tests/modsecurity-phase4-invalid-config.t +++ b/tests/modsecurity-phase4-invalid-config.t @@ -3,9 +3,9 @@ use warnings; use strict; use Test::More tests => 1; BEGIN { use FindBin; chdir($FindBin::Bin); } use lib 'lib'; -use Test::Nginx::Socket -Base; +use Test::Nginx; -my $t = Test::Nginx::Socket->new()->has(qw/http/); +my $t = Test::Nginx->new()->has(qw/http/); $t->write_file('phase4-invalid.conf', "text/*\n"); $t->write_file_expand('nginx.conf', <<'EOF'); @@ -28,7 +28,8 @@ http { } EOF -my $cmd = "$t->{_testdir}/../nginx -p $t->{_testdir}/ -c nginx.conf -t 2>&1"; +my $cmd = $ENV{TEST_NGINX_BINARY} || ($t->testdir() . '/../nginx'); +$cmd .= " -p " . $t->testdir() . "/ -c nginx.conf -t 2>&1"; my $out = `$cmd`; like($out, qr/invalid content-type entry in modsecurity_phase4_content_types_file/, From 8b766e44038642390593c6a0d2a6d13b64247fc9 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 14 May 2026 19:26:58 +0200 Subject: [PATCH 21/40] Stabilize phase4 invalid-config test nginx -t setup --- tests/modsecurity-phase4-invalid-config.t | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/modsecurity-phase4-invalid-config.t b/tests/modsecurity-phase4-invalid-config.t index c62f53b4..cc695551 100644 --- a/tests/modsecurity-phase4-invalid-config.t +++ b/tests/modsecurity-phase4-invalid-config.t @@ -7,6 +7,8 @@ use Test::Nginx; my $t = Test::Nginx->new()->has(qw/http/); $t->write_file('phase4-invalid.conf', "text/*\n"); +mkdir($t->testdir() . '/logs') unless -d $t->testdir() . '/logs'; +$t->write_file('logs/error.log', ''); $t->write_file_expand('nginx.conf', <<'EOF'); %%TEST_GLOBALS%% @@ -17,7 +19,7 @@ http { %%TEST_GLOBALS_HTTP%% server { - listen 127.0.0.1:%%PORT%%; + listen 127.0.0.1:19849; server_name localhost; location / { modsecurity on; From 20d7aca643db680455a3baea9e33bb0e1ae739cb Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 14 May 2026 20:17:29 +0200 Subject: [PATCH 22/40] docs: document phase4 nginx directives in README --- README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a93e5df0..ed1aa335 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Further information about nginx third-party add-ons support are available [here] # Usage ModSecurity for nginx extends your nginx configuration directives. -It adds four new directives and they are: +It adds eight directives and they are: modsecurity ----------- @@ -175,6 +175,31 @@ using the same unique identificator. String can contain variables. + +modsecurity_phase4_mode +------------------------ +**syntax:** *modsecurity_phase4_mode minimal | safe | strict* + +**context:** *http, server, location* + +Controls how phase 4 interventions are handled when response headers were already sent. + +modsecurity_phase4_content_types_file +-------------------------------------- +**syntax:** *modsecurity_phase4_content_types_file <path>* + +**context:** *http, server, location* + +Loads the list of response content types that are in scope for phase 4 handling from a file. + +modsecurity_phase4_log +---------------------- +**syntax:** *modsecurity_phase4_log <path>* + +**context:** *http, server, location* + +Sets the file used for phase 4 JSON event logging. + modsecurity_use_error_log ----------- **syntax:** *modsecurity_use_error_log on | off* From 6f3dd0f73a64b9bb78ea7300453310d05e6eea94 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:44:52 +0200 Subject: [PATCH 23/40] Port current connector Phase 4 behavior to NGINX (#14) Adapt response inspection limits, buffer handling and interventions from ModSecurity-conector@b0f3bdab429717b5b0311c30c5b4d1153c672ac0. Use off/safe/strict with engine-owned MIME policy; keep request contexts isolated and preserve standalone JSONL events. Verified 33 helper cases and the 311-check native Linux suite with GCC/Clang for NGINX 1.28.x, 1.29.x and 1.30.x. Existing Windows CI setup failures are documented in the PR. --- .github/workflows/test.yml | 18 +- README.md | 47 +- docs/examples/phase4-content-types.conf | 16 - docs/examples/phase4-engine-mime.conf | 11 + docs/examples/phase4-minimal.conf | 31 -- docs/examples/phase4-off.conf | 33 ++ docs/examples/phase4-safe.conf | 15 +- docs/examples/phase4-strict.conf | 15 +- docs/phase4-handling.de.md | 422 +++++++--------- docs/phase4-handling.en.md | 406 +++++++--------- src/ngx_http_modsecurity_body_filter.c | 528 +++++++++++++------- src/ngx_http_modsecurity_common.h | 17 +- src/ngx_http_modsecurity_header_filter.c | 37 +- src/ngx_http_modsecurity_module.c | 406 +++++----------- tests/README.md | 46 +- tests/modsecurity-phase4-content-types.t | 102 ++-- tests/modsecurity-phase4-invalid-config.t | 38 +- tests/modsecurity-phase4-modes.t | 111 ++--- tests/modsecurity-phase4-regression.t | 50 +- tests/modsecurity-phase4-subrequest.t | 83 ++++ tests/test_phase4_runtime.py | 560 ++++++++++++++++++++++ 21 files changed, 1806 insertions(+), 1186 deletions(-) delete mode 100644 docs/examples/phase4-content-types.conf create mode 100644 docs/examples/phase4-engine-mime.conf delete mode 100644 docs/examples/phase4-minimal.conf create mode 100644 docs/examples/phase4-off.conf create mode 100644 tests/modsecurity-phase4-subrequest.t create mode 100644 tests/test_phase4_runtime.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4fa41c1d..fdd50b1e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,6 +8,20 @@ on: permissions: {} jobs: + phase4-unit-tests: + permissions: + contents: read + runs-on: ubuntu-22.04 + strategy: + matrix: + compiler: [gcc, clang] + env: + CC: "/usr/bin/${{ matrix.compiler }}" + steps: + - uses: actions/checkout@v4 + - name: Run standalone Phase4 helper regressions + run: python3 tests/test_phase4_runtime.py -v + build-linux: permissions: contents: read @@ -58,7 +72,7 @@ jobs: fetch-depth: 1 - name: Copy ModSecurity-nginx tests to nginx/test run: | - cp ModSecurity-nginx/tests/* nginx/test + cp ModSecurity-nginx/tests/*.t ModSecurity-nginx/tests/*.pl nginx/test - name: Build nginx with ModSecurity-nginx module working-directory: nginx run: | @@ -294,7 +308,7 @@ jobs: - name: Copy ModSecurity-nginx tests to nginx/test working-directory: nginx/test run: | - cp ../objs/lib/ModSecurity-nginx/tests/* . + cp ../objs/lib/ModSecurity-nginx/tests/*.t ../objs/lib/ModSecurity-nginx/tests/*.pl . - name: Remove /usr/bin/link conflicting with MSVC link.exe run: | set -ex diff --git a/README.md b/README.md index ed1aa335..ea3fefce 100644 --- a/README.md +++ b/README.md @@ -178,19 +178,37 @@ String can contain variables. modsecurity_phase4_mode ------------------------ -**syntax:** *modsecurity_phase4_mode minimal | safe | strict* +**syntax:** *modsecurity_phase4_mode off | safe | strict* **context:** *http, server, location* -Controls how phase 4 interventions are handled when response headers were already sent. +**default:** *off* + +Controls the connector's additional phase 4 intervention policy. `safe` logs a +late intervention as `log_only`; `strict` aborts the connection once headers +are sent. Before headers are sent, both modes can apply the intervention's HTTP +status. Neither mode guarantees a clean 403 after headers have been sent. -modsecurity_phase4_content_types_file --------------------------------------- -**syntax:** *modsecurity_phase4_content_types_file <path>* +`off` uses the native intervention path and disables the connector's additional +phase 4 policy and body budget. It does not disable ModSecurity response-body +inspection; late native interventions can still fail the response. Configure +inspection with `SecResponseBodyAccess`, `SecResponseBodyMimeType`, and +`SecResponseBodyMimeTypesClear` in ModSecurity rules. + +modsecurity_phase4_body_limit +----------------------------- +**syntax:** *modsecurity_phase4_body_limit <size>* **context:** *http, server, location* -Loads the list of response content types that are in scope for phase 4 handling from a file. +**default:** *1m (1 MiB)* + +Sets a positive cumulative response-body budget for `safe` and `strict`, in +bytes or nginx size notation such as `256k` or `2m`. The connector counts bytes +seen, including file-backed buffers, and rejects a chunk that would exceed the +budget before forwarding that chunk. This is independent of the engine's +response-body limit and does not introduce global response buffering. +`off` ignores this budget but retains overflow checks. modsecurity_phase4_log ---------------------- @@ -200,6 +218,22 @@ modsecurity_phase4_log Sets the file used for phase 4 JSON event logging. +### Phase 4 migration + +This branch adopts the phase 4 behavior from +[ModSecurity-conector at b0f3bdab429717b5b0311c30c5b4d1153c672ac0](https://github.com/Easton97-Jens/ModSecurity-conector/tree/b0f3bdab429717b5b0311c30c5b4d1153c672ac0/connectors/nginx/src), +adapted for this standalone nginx module. It does not require that repository's +multi-connector runtime. + +This changes configuration compatibility: `minimal` is rejected, the default +mode changes from `safe` to `off`, and +`modsecurity_phase4_content_types_file` is removed. Set `safe` or `strict` +explicitly when migrating, and move MIME selection into ModSecurity rules. +See [English](docs/phase4-handling.en.md) or +[Deutsch](docs/phase4-handling.de.md) for the behavior, limits, logging schema, +and examples. The documentation describes the source implementation; test +results from the source repository are not validation of this migration. + modsecurity_use_error_log ----------- **syntax:** *modsecurity_use_error_log on | off* @@ -296,4 +330,3 @@ feel free to open GitHub issues requesting for new features. Before opening a ne Having our packages in distros on time is something we highly desire. Let us know if there is anything we can do to facilitate your work as a packager. - diff --git a/docs/examples/phase4-content-types.conf b/docs/examples/phase4-content-types.conf deleted file mode 100644 index 5baa48ba..00000000 --- a/docs/examples/phase4-content-types.conf +++ /dev/null @@ -1,16 +0,0 @@ -# phase4-content-types.conf -# -# Format rules: -# 1) Exactly one MIME type per line. -# 2) Lines starting with # are comments. -# 3) Wildcards are not supported (e.g. text/* is invalid). -# 4) Parameters (e.g. "; charset=utf-8") do not need separate entries. -# The module normalizes response Content-Type and compares the base type. -# 5) Keep this list focused on body types you actually want to scope. - -text/html -text/plain -application/json -application/problem+json -application/xml -application/xhtml+xml diff --git a/docs/examples/phase4-engine-mime.conf b/docs/examples/phase4-engine-mime.conf new file mode 100644 index 00000000..1597e41e --- /dev/null +++ b/docs/examples/phase4-engine-mime.conf @@ -0,0 +1,11 @@ +# ModSecurity rules file: engine response-body MIME additions +# Load with modsecurity_rules_file alongside your other rules. +# This is not an nginx include and not a connector MIME allowlist. +# To replace the engine MIME list, load a separate reset BEFORE this file: +# modsecurity_rules 'SecResponseBodyMimeTypesClear'; +# modsecurity_rules_file /etc/modsecurity/phase4-engine-mime.conf; +# Keep the reset in a separate rule load from the MIME additions. +# Adapt this list to the response types your application needs to inspect. + +SecResponseBodyAccess On +SecResponseBodyMimeType text/html text/plain application/json application/problem+json application/xml application/xhtml+xml diff --git a/docs/examples/phase4-minimal.conf b/docs/examples/phase4-minimal.conf deleted file mode 100644 index 6d3250e8..00000000 --- a/docs/examples/phase4-minimal.conf +++ /dev/null @@ -1,31 +0,0 @@ -# Production example: phase-4 mode "minimal" -# Use when response continuity is more important than forced connection aborts -# for late phase:4 interventions (after headers were already sent). - -http { - upstream app_backend { - server 127.0.0.1:8081; - } - - server { - listen 80; - server_name example.local; - - modsecurity on; - modsecurity_phase4_mode minimal; - modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; - modsecurity_phase4_content_types_file /etc/modsecurity/phase4-content-types.conf; - - # Inline rule sample for demonstration. - # In production, you can keep using modsecurity_rules_file as usual. - modsecurity_rules ' - SecRuleEngine On - SecResponseBodyAccess On - SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940101,phase:4,deny,log,status:403" - '; - - location / { - proxy_pass http://app_backend; - } - } -} diff --git a/docs/examples/phase4-off.conf b/docs/examples/phase4-off.conf new file mode 100644 index 00000000..81d2f699 --- /dev/null +++ b/docs/examples/phase4-off.conf @@ -0,0 +1,33 @@ +# nginx example: phase-4 mode "off" (the default) +# Uses native intervention handling. This does not disable ModSecurity or +# response-body inspection; late native interventions can fail the response. +# The connector's additional policy and body budget are disabled. + +http { + upstream app_backend { + server 127.0.0.1:8081; + } + + server { + listen 80; + server_name example.local; + + modsecurity on; + modsecurity_phase4_mode off; + + # Illustrative rules; adapt to your application before deployment. + # MIME selection is configured in the engine, not in the connector. + # Reset the MIME list in a separate engine rule load. + modsecurity_rules 'SecResponseBodyMimeTypesClear'; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyMimeType text/html text/plain application/json + SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940101,phase:4,deny,log,status:403" + '; + + location / { + proxy_pass http://app_backend; + } + } +} \ No newline at end of file diff --git a/docs/examples/phase4-safe.conf b/docs/examples/phase4-safe.conf index baaa7a08..c07bf09b 100644 --- a/docs/examples/phase4-safe.conf +++ b/docs/examples/phase4-safe.conf @@ -1,6 +1,6 @@ -# Production example: phase-4 mode "safe" -# Default mode in this module. Recommended baseline if you need phase:4 -# visibility while avoiding forced disconnects on late interventions. +# nginx example: phase-4 mode "safe" (must be selected explicitly) +# Late interventions are logged as log_only. Engine, I/O, and body-budget +# failures still stop the affected response. http { upstream app_backend { @@ -13,12 +13,17 @@ http { modsecurity on; modsecurity_phase4_mode safe; + modsecurity_phase4_body_limit 1m; modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; - modsecurity_phase4_content_types_file /etc/modsecurity/phase4-content-types.conf; + # Illustrative rules; adapt to your application before deployment. + # MIME selection is configured in the engine, not in the connector. + # Reset the MIME list in a separate engine rule load. + modsecurity_rules 'SecResponseBodyMimeTypesClear'; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On + SecResponseBodyMimeType text/html text/plain application/json SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940102,phase:4,deny,log,status:403" '; @@ -26,4 +31,4 @@ http { proxy_pass http://app_backend; } } -} +} \ No newline at end of file diff --git a/docs/examples/phase4-strict.conf b/docs/examples/phase4-strict.conf index 68fb18ec..fba1d726 100644 --- a/docs/examples/phase4-strict.conf +++ b/docs/examples/phase4-strict.conf @@ -1,6 +1,6 @@ -# Production example: phase-4 mode "strict" -# Use only if connection aborts are operationally acceptable, because -# late phase:4 interventions may terminate the connection. +# nginx example: phase-4 mode "strict" +# Late interventions terminate the response. A clean HTTP 403 after headers +# have been sent is not guaranteed; clients may see a transport failure. http { upstream app_backend { @@ -13,12 +13,17 @@ http { modsecurity on; modsecurity_phase4_mode strict; + modsecurity_phase4_body_limit 1m; modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; - modsecurity_phase4_content_types_file /etc/modsecurity/phase4-content-types.conf; + # Illustrative rules; adapt to your application before deployment. + # MIME selection is configured in the engine, not in the connector. + # Reset the MIME list in a separate engine rule load. + modsecurity_rules 'SecResponseBodyMimeTypesClear'; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On + SecResponseBodyMimeType text/html text/plain application/json SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940103,phase:4,deny,log,status:403" '; @@ -26,4 +31,4 @@ http { proxy_pass http://app_backend; } } -} +} \ No newline at end of file diff --git a/docs/phase4-handling.de.md b/docs/phase4-handling.de.md index 78799869..4ee7fac4 100644 --- a/docs/phase4-handling.de.md +++ b/docs/phase4-handling.de.md @@ -1,251 +1,175 @@ # ModSecurity-nginx: Phase-4-Handling (Deutsch) -## Zweck dieses Dokuments - -Dieses Dokument beschreibt die im aktuellen Code implementierte Phase-4-Behandlung im nginx-Modul, inklusive: - -- technischer Grenzen bei bereits gesendeten Headern, -- Verhalten der Modi `minimal`, `safe`, `strict`, -- Content-Type-Scoping, -- Logging (`modsecurity_phase4_log`) und Sicherheitsaspekte, -- Abgrenzung zwischen **Produktionskonfiguration** und **Test-/Demo-Verhalten**. - -Es werden nur Aussagen getroffen, die durch den aktuellen Repository-Stand belegt sind. - ---- - -## 1) Hintergrund: Request- vs. Response-Phasen - -ModSecurity-Regeln laufen in unterschiedlichen Phasen der Transaktion. - -- Frühe Phasen (z. B. Request-Phasen) treffen Entscheidungen, bevor eine Antwort gesendet wird. -- `phase:4` gehört zur **Response-Body-Verarbeitung**. - -Das ist ein zentraler Unterschied: In `phase:4` kann nginx bereits begonnen haben, Header und/oder Body zu senden. - -### Warum ist das relevant? - -Wenn eine Regel in `phase:4` eine Intervention wie `deny` oder `redirect` auslöst, ist ein sauberer Wechsel auf neuen HTTP-Status nur möglich, solange Header noch nicht final gesendet wurden. - ---- - -## 2) Was `phase:4` praktisch bedeutet - -`phase:4`-Regeln prüfen den Antwortinhalt (Response-Body). Das ist nützlich, wenn Sicherheitskriterien erst am Antwortinhalt erkennbar sind. - -Gleichzeitig erzeugt es eine harte technische Grenze: - -- **Vor Header-Versand**: Status/Redirect kann noch sauber gesetzt werden. -- **Nach Header-Versand**: Status/Redirect lässt sich nicht zuverlässig „nachträglich“ korrigieren. - -Daher existiert im Modul ein dediziertes Handling für späte Interventionen. - ---- - -## 3) Warum Header in `phase:4` bereits gesendet sein können - -In nginx werden Antworten als Stream verarbeitet. Je nach Upstream-, Filter- und Buffering-Verhalten kann der Header bereits auf dem Socket sein, bevor die vollständige Body-Prüfung abgeschlossen ist. - -Folge: Ein in `phase:4` erkanntes `status:403` oder `redirect:302` kann nicht garantiert als sauberer HTTP-Status beim Client landen. - -> Keine falsche Garantie: `phase:4` kann nach gesendeten Headern keinen sauberen HTTP-Status mehr garantieren. - ---- - -## 4) Neue Directives im Modul - -## `modsecurity_phase4_mode` - -Konfiguriert das Verhalten bei Phase-4-Interventionen. - -Unterstützte Werte: - -- `minimal` -- `safe` -- `strict` - -Ungültige Werte führen zu Konfigurationsfehlern. - -## `modsecurity_phase4_content_types_file` - -Lädt erlaubte/gescopte Content-Types aus einer Datei. - -- eine Zeile pro Typ, -- Kommentare mit `#`, -- Einträge werden validiert, -- Wildcards (`*`) sind nicht erlaubt. - -Wenn nicht gesetzt, nutzt das Modul Default-Typen. - -## `modsecurity_phase4_log` - -Aktiviert dediziertes JSON-Line-Logging für Phase-4-Ereignisse. - ---- - -## 5) Modusverhalten (`minimal`, `safe`, `strict`) - -## `minimal` - -Ziel: möglichst wenig Eingriff in laufende Responses. - -Bei Intervention nach gesendeten Headern: - -- Aktion wird auf `log_only` degradiert, -- keine erzwungene Verbindungsbeendigung. - -Sinnvoll, wenn Stabilität und vollständige Antwortauslieferung Vorrang haben. - -## `safe` - -Ziel: konservativer Produktionsstandard (Default-Merge-Verhalten im Modul). - -Bei Intervention nach gesendeten Headern: - -- ebenfalls `log_only`. - -Sinnvoll als Standardmodus, wenn Phase-4-Transparenz gewünscht ist, aber Verbindungsabbrüche vermieden werden sollen. - -## `strict` - -Ziel: strengere Reaktion, wenn keine saubere Statusänderung mehr möglich ist. - -Bei Intervention nach gesendeten Headern: - -- `connection_abort`. - -### Risiken von `strict` - -- Kann aktive Verbindungen abbrechen. -- Liefert nicht garantiert einen „schönen“ HTTP-Fehlerstatus beim Client. -- Kann je nach Client/Proxy als Transportfehler statt als 4xx/3xx sichtbar werden. - -`strict` ist daher nur sinnvoll, wenn diese Nebenwirkungen betrieblich akzeptabel sind. - ---- - -## 6) Verhalten nach Header-Status - -## Header noch **nicht** gesendet - -Wenn eine Intervention vor Header-Versand finalisiert wird, bleibt normaler Deny-/Statuspfad möglich (im Code als `deny_status` geloggt). - -## Header bereits gesendet - -- `minimal`: `log_only` -- `safe`: `log_only` -- `strict`: `connection_abort` - -Das ist eine bewusste Degradierung, um keine falschen Status-Garantien vorzutäuschen. - ---- - -## 7) Warum **kein globales Response-Body-Buffering** genutzt wird - -Ein globales Buffering aller Responses würde zwar den Eingriffszeitpunkt verschieben, bringt aber technische und betriebliche Kosten: - -- zusätzlicher Speicher- und Latenz-Overhead, -- größere Komplexität für allgemeine Response-Pfade, -- Risiko unbeabsichtigter Nebenwirkungen auf Durchsatz/Stabilität. - -Der aktuelle Ansatz versucht stattdessen, late Interventionen transparent und kontrolliert zu behandeln (`log_only` oder `connection_abort`). - ---- - -## 8) Warum kein `ngx_chain_t`-Reordering/Rewriting genutzt wird - -Das Modul implementiert **keine** künstliche Reordering-Logik auf bereits laufenden Body-Ketten, um nachträglich „doch noch“ andere Header-/Statussemantik zu erzwingen. - -Begründung: - -- hohe Komplexität, -- erhöhte Fehleranfälligkeit, -- schwierige Garantien über korrektes Verhalten in allen Filter-/Upstream-Kombinationen. - -Die dokumentierte Degradierung ist technisch robuster als scheinbar harte, aber unsichere Nachkorrekturen. - ---- - -## 9) Content-Type-Scoping: Bedeutung und sichere Nutzung - -`modsecurity_phase4_content_types_file` begrenzt Phase-4-Sonderbehandlung auf definierte MIME-Typen. - -Wenn `Content-Type` fehlt oder nicht im Scope liegt: - -- Ereignis wird als `log_only` dokumentiert, -- Grund ist typischerweise `content_type_missing` oder `content_type_not_in_scope`. - -### Warum wichtig? - -- reduziert Nebeneffekte auf nicht-zielgerichtete Antworttypen, -- macht Verhalten vorhersagbarer, -- zwingt zu expliziter Auswahl relevanter Content-Typen. - ---- - -## 10) Logging-Format und Sicherheitsgrenzen - -Mit `modsecurity_phase4_log` schreibt das Modul JSON-Zeilen, u. a. mit: - -- `event` (`phase4_intervention`) -- `uri`, `method` -- `response_status`, `waf_status` -- `content_type` -- `header_sent` -- `mode` -- `wanted_action`, `actual_action` -- `reason` -- `intervention` -- `rule_id` - -Zusätzlich kann im nginx `error.log` ein Hinweis erscheinen (insbesondere im `strict`-Pfad bei Headern, die schon gesendet wurden). - -### Sicherheitsgrenze im Logging - -Die Tests prüfen explizit, dass keine Response-Body-Inhalte in das Phase-4-Log „durchrutschen“. - ---- - -## 11) Produktionsbeispiele - -Allgemeine (nicht testpfadgebundene) Beispielkonfigurationen: - -- `docs/examples/phase4-minimal.conf` -- `docs/examples/phase4-safe.conf` -- `docs/examples/phase4-strict.conf` -- `docs/examples/phase4-content-types.conf` - -Diese Beispiele nutzen `http`/`server`/`location /` und keine test-spezifischen `/phase4`-Pfade. - ---- - -## 12) Test-/Demo-Verhalten (klar getrennt) - -Im Repository existieren Testfälle mit `/phase4`-Pfaden, z. B. in: - -- `tests/modsecurity.t` -- `tests/modsecurity-proxy.t` -- `tests/modsecurity-h2.t` -- `tests/modsecurity-proxy-h2.t` -- `tests/modsecurity-phase4-*.t` - -Diese Pfade sind **Testkontext** und nicht als allgemeine Produktionskonfiguration gedacht. - ---- - -## 13) Bekannte Grenzen / keine falschen Versprechen - -- `phase:4` kann **nicht garantieren**, dass ein gewünschter 301/302/401/403-Status nach Header-Versand noch sauber beim Client ankommt. -- Bei späten Interventionen ist nur degradierte Behandlung möglich (`log_only` oder `connection_abort`). -- `strict` bedeutet nicht „garantierter 403“, sondern kann Verbindungsabbruch bedeuten. - ---- - -## 14) Betriebsleitfaden (kurz) - -1. Harte Block-/Redirect-Entscheidungen möglichst in frühere Phasen legen. -2. `safe` als Ausgangspunkt verwenden, wenn keine klare Notwendigkeit für `strict` besteht. -3. `strict` nur einsetzen, wenn Abbrüche tolerierbar und beobachtbar sind. -4. `modsecurity_phase4_log` aktivieren und auf `actual_action`/`reason` auswerten. -5. Content-Type-Liste eng halten und regelmäßig prüfen. - +## 1) Geltungsbereich und Herkunft + +Phase 4 prüft den Response-Body. nginx kann bereits Header oder Body-Bytes +gesendet haben, wenn eine Phase-4-Regel einen Deny-Status oder Redirect +anfordert. Eine späte Intervention kann den bereits beim Client angekommenen +HTTP-Status nicht zuverlässig ersetzen. + +Dieser Branch übernimmt die relevante nginx-Implementierung aus +[Easton97-Jens/ModSecurity-conector, Commit b0f3bdab429717b5b0311c30c5b4d1153c672ac0](https://github.com/Easton97-Jens/ModSecurity-conector/tree/b0f3bdab429717b5b0311c30c5b4d1153c672ac0/connectors/nginx/src). +Sie wird an dieses eigenständige Modul angepasst; das Phase-4-JSON-Lines-Schema +dieses Branches bleibt erhalten. Die Multi-Connector-Laufzeit und +Laufzeitnachweise des Quellrepositories werden nicht übernommen. + +Dieses Dokument beschreibt das Verhalten im Quellcode. Es behauptet nicht, +dass Laufzeit-, Protokoll- oder Integrationstests für diese Migration bestanden +wurden. + +## 2) Konfiguration und inkompatible Änderungen + +Alle drei Connector-Direktiven gelten in `http`-, `server`- und +`location`-Kontexten und werden vom umgebenden Kontext geerbt. + +| Direktive | Werte | Standard | +| --- | --- | --- | +| `modsecurity_phase4_mode` | `off`, `safe`, `strict` | `off` | +| `modsecurity_phase4_body_limit` | Positive Byte-Anzahl oder nginx-Größe, z. B. `256k`, `2m` | `1m` (1 MiB) | +| `modsecurity_phase4_log` | Dateipfad für Phase-4-JSON-Lines-Ereignisse | Kein dediziertes Log | + +Bei der Migration der bisherigen `master-phase4`-Konfiguration sind diese +Änderungen erforderlich: + +- `minimal` ist nicht mehr gültig und kein Alias für `off`. +- Der Standardmodus wechselt von `safe` auf `off`. Für die zusätzliche + Connector-Behandlung muss `safe` oder `strict` explizit gesetzt werden. +- `modsecurity_phase4_content_types_file` entfällt und verursacht einen + nginx-Konfigurationsfehler. Die MIME-Auswahl gehört jetzt in ModSecurity-Regeln. +- Das positive Connector-Body-Budget ist neu. Bei `safe` oder `strict` muss + ein passender Wert für die Antworten gewählt werden; null ist ungültig. + +Die migrierte Konfiguration vor dem nginx-Reload mit `nginx -t` in der +Zielumgebung prüfen. + +## 3) Modusverhalten und Header-Zeitpunkt + +`off` deaktiviert die zusätzliche Phase-4-Interventionsbehandlung und das +Body-Budget des Connectors. Es **deaktiviert weder ModSecurity** noch die +Response-Body-Prüfung oder Phase-4-Regeln. Die native Interventionsbehandlung +bleibt aktiv. Tritt eine native Intervention nach dem Header-Versand auf, +kann die Finalisierung einen Transportfehler verursachen; `off` garantiert +keine vollständige Auslieferung jeder Antwort. + +Für die zusätzliche Behandlung in `safe` und `strict` gilt: + +| Header-Zustand | `safe` | `strict` | +| --- | --- | --- | +| Header noch nicht gesendet | Status/Redirect über den normalen Interventionspfad anwenden; als `deny_status` geloggt | Gleiches Verhalten | +| Header bereits gesendet | Intervention als `log_only` protokollieren und fortfahren | `connection_abort` protokollieren und die Antwort abbrechen | + +`safe` degradiert nur späte **Interventionen**. Fehler der Engine-API, +ungültige Buffer, fehlgeschlagene Dateizugriffe, Speicherallokationsfehler, +Zählerüberläufe und überschrittene Body-Budgets bleiben Fehler. Sie werden +nicht in erfolgreiche Weiterleitung umgewandelt. + +`strict` kann eine abgeschnittene Antwort oder einen Transportfehler beim +Client/Proxy auslösen. Nach dem Header-Versand garantiert es keinen sauberen +403-, 401-, 301- oder 302-Status. Bereits weitergeleitete Body-Bytes lassen +sich nicht zurückholen. + +## 4) Response-Body-Budget und Streaming + +`modsecurity_phase4_body_limit` ist in `safe` und `strict` ein +Connector-Budget für die kumulativ gesehenen Response-Bytes, einschließlich +dateibasierter Buffer. Es ist unabhängig von `SecResponseBodyLimit` der +Engine und zählt nicht nur die Bytes, die die Engine zur Prüfung speichert. + +Eine Antwort darf das Budget exakt erreichen. Ein Chunk, der es überschreiten +würde, wird **vor seiner Weiterleitung** abgewiesen. Frühere Chunks können +bereits beim Client angekommen sein; ein ersetzender HTTP-Fehlerstatus ist +daher nicht garantiert. In `off` wird das konfigurierte Connector-Budget +ignoriert, die kumulative Zählung weist Überläufe über `SIZE_MAX` aber weiterhin +ab. + +Der Connector puffert Antworten nicht global und ordnet nginx-Body-Ketten nicht +um. Speicherbuffer werden direkt geprüft. Rein dateibasierte Buffer werden +über einen auf 32 KiB begrenzten Arbeitsbuffer gelesen, ohne die ganze Datei +in den Speicher zu laden oder die ausgehende Originalkette zu ersetzen. Echte +Verarbeitungs- oder I/O-Fehler stoppen die betroffene Antwort. + +Der Response-Body wird pro Transaktion genau einmal finalisiert: Beim +Hauptrequest wird `last_buf` ausgewertet, beim Subrequest `last_in_chain`. +Wiederholte Filteraufrufe nach der Finalisierung führen Phase 4 nicht erneut +aus. + +## 5) MIME-Auswahl durch ModSecurity + +Die Engine entscheidet über die Response-Body-Prüfung anhand von +`SecResponseBodyAccess`, `SecResponseBodyMimeType` und +`SecResponseBodyMimeTypesClear`. Es gibt keine Connector-Content-Type-Liste +und keine MIME-basierte Degradierung durch den Connector. + +Beispielsweise die MIME-Liste in einem Regel-Ladevorgang zurücksetzen und +die ausgewählten Typen in einem getrennten Ladevorgang hinzufügen: + +```nginx +modsecurity_rules 'SecResponseBodyMimeTypesClear'; +modsecurity_rules ' + SecResponseBodyAccess On + SecResponseBodyMimeType text/html text/plain application/json +'; +``` + +Das Zurücksetzen erfolgt getrennt, weil die +[Merge-Implementierung von libModSecurity](https://raw.githubusercontent.com/owasp-modsecurity/ModSecurity/v3/master/headers/modsecurity/rules_set_properties.h) +MIME-Werte entfernen kann, die im selben Regel-Ladevorgang hinzugefügt werden. + +Das eigenständige [Engine-MIME-Beispiel](examples/phase4-engine-mime.conf) +enthält MIME-Ergänzungen und aktiviert die Response-Body-Prüfung. Es ist eine +ModSecurity-Regeldatei, kein nginx-Include. Um die MIME-Liste der Engine durch +diese Datei zu ersetzen, zuerst das Zurücksetzen und dann die Datei zusätzlich +zu den anderen Regeln laden: + +```nginx +modsecurity_rules 'SecResponseBodyMimeTypesClear'; +modsecurity_rules_file /etc/modsecurity/phase4-engine-mime.conf; +``` + +Die vollständigen nginx-Beispiele unten konfigurieren die MIME-Auswahl inline +mit derselben Reihenfolge getrennter Ladevorgänge. + +Die Engine-Auswahl deaktiviert das unabhängige Connector-Body-Budget in +`safe` oder `strict` nicht. + +## 6) Logging-Format und Sicherheitsgrenze + +`modsecurity_phase4_log` erhält das bisherige JSON-Lines-Schema der +Interventionsereignisse: + +- `event` (`phase4_intervention`), `uri`, `method`; +- `response_status`, `waf_status`, `content_type`, `header_sent`, `mode`; +- `wanted_action`, `actual_action`, `reason`; +- `intervention`, `rule_id`. + +Die Werte von `actual_action` bleiben `deny_status`, `log_only` und +`connection_abort`; `mode` nutzt die aktuellen Modusnamen. Die +Interventionsnachricht wird maskiert, statt aus der Engine kopiert zu werden, +und das Ereignis enthält keine hinzugefügten Response-Body-Inhalte. Das +nginx-Error-Log kann weitere Diagnosen enthalten. + +`off` nutzt die native Interventionsbehandlung und erzeugt diese dedizierten +Policy-Ereignisse nicht. Ein dediziertes Interventionsereignis ist keine +vollständige Erfassung aller Engine-, Buffer- oder I/O-Fehler. + +## 7) Konfigurationsbeispiele und Verifikation + +- [off](examples/phase4-off.conf): native Interventionsbehandlung bei + eingeschalteter Engine-Prüfung. +- [safe](examples/phase4-safe.conf): explizite späte `log_only`-Behandlung und + 1 MiB Connector-Budget. +- [strict](examples/phase4-strict.conf): später Verbindungsabbruch und + 1 MiB Connector-Budget. +- [Engine-MIME-Auswahl](examples/phase4-engine-mime.conf): + ModSecurity-Konfiguration für den Response-Body. + +Die nginx-Beispiele verwenden `location /` und einen Beispiel-Upstream unter +`127.0.0.1:8081`. Listen-Adresse, Upstream, Logpfad und Regeln an die +Zielumgebung anpassen. Die Regel mit `sensitive-marker` dient zur Illustration. + +Repository-Testendpunkte wie `/phase4` sind Testfixtures, keine erforderlichen +Produktionspfade. Laufzeitprüfungen sollten späte Deny-/Redirect-Interventionen, +HTTP/1.1 und HTTP/2, dateibasierte Antworten, Subrequests, wiederholte +Finalisierung, Budgetgrenzen und Fehlerpfade abdecken. Ergebnisse aus einem +anderen Repository oder Build belegen diese Eigenschaften des eingesetzten +Moduls nicht. diff --git a/docs/phase4-handling.en.md b/docs/phase4-handling.en.md index 7e5e1a69..41f44318 100644 --- a/docs/phase4-handling.en.md +++ b/docs/phase4-handling.en.md @@ -1,251 +1,165 @@ # ModSecurity-nginx: Phase 4 Handling (English) -## Purpose of this document +## 1) Scope and provenance + +Phase 4 inspects the response body. nginx can already have sent headers or body +bytes when a phase 4 rule requests a deny status or redirect. A late +intervention cannot reliably replace the HTTP status already sent to the +client. + +This branch adopts the relevant nginx implementation from +[Easton97-Jens/ModSecurity-conector, commit b0f3bdab429717b5b0311c30c5b4d1153c672ac0](https://github.com/Easton97-Jens/ModSecurity-conector/tree/b0f3bdab429717b5b0311c30c5b4d1153c672ac0/connectors/nginx/src). +It is adapted to this standalone module and preserves this branch's phase 4 +JSON-lines schema. The source repository's multi-connector runtime and run +evidence are not imported. + +This document describes source behavior. It does not claim that runtime, +protocol, or integration tests have passed for this migration. + +## 2) Configuration and breaking changes + +All three connector directives below are valid in `http`, `server`, and +`location` contexts and inherit from the enclosing context. + +| Directive | Values | Default | +| --- | --- | --- | +| `modsecurity_phase4_mode` | `off`, `safe`, `strict` | `off` | +| `modsecurity_phase4_body_limit` | Positive byte count or nginx size, e.g. `256k`, `2m` | `1m` (1 MiB) | +| `modsecurity_phase4_log` | File path for phase 4 JSON-lines events | No dedicated log | -This document describes the currently implemented Phase 4 behavior in the nginx module, including: +Migration from the previous `master-phase4` configuration requires these +changes: -- technical limits once headers are already sent, -- behavior of `minimal`, `safe`, and `strict` modes, -- content-type scoping, -- logging (`modsecurity_phase4_log`) and security boundaries, -- a clear split between **production configuration** and **test/demo behavior**. - -Only statements supported by the current repository code and tests are included. - ---- - -## 1) Background: request vs response phases - -ModSecurity rules run in multiple transaction phases. - -- Early phases (for example request phases) make decisions before a response is emitted. -- `phase:4` belongs to **response-body processing**. - -This difference is critical: during `phase:4`, nginx may already have sent headers and/or part of the body. - -### Why this matters - -If a `phase:4` rule triggers an intervention (`deny`, `status`, `redirect`), a clean status/redirect rewrite is only possible while headers are still unsent. - ---- - -## 2) What `phase:4` means operationally - -`phase:4` rules inspect response body content. This is useful when security signals are visible only in outgoing payload. - -At the same time, this creates a hard technical boundary: - -- **Before headers are sent**: status/redirect can still be applied cleanly. -- **After headers are sent**: status/redirect can no longer be reliably changed. - -Therefore the module includes dedicated late-intervention handling. - ---- - -## 3) Why headers may already be sent in `phase:4` - -nginx processes responses as a stream. Depending on upstream behavior, buffering, and filter timing, headers can already be on the wire before body inspection fully completes. - -Result: a `phase:4` `status:403` or `redirect:302` is not guaranteed to become a clean client-visible HTTP status. - -> No false guarantee: after headers are sent, `phase:4` cannot guarantee a clean HTTP status rewrite. - ---- - -## 4) New directives in this module - -## `modsecurity_phase4_mode` - -Configures phase-4 late-intervention behavior. - -Supported values: - -- `minimal` -- `safe` -- `strict` - -Invalid values are rejected by configuration parsing. - -## `modsecurity_phase4_content_types_file` - -Loads scoped content types from a file. - -- one MIME type per line, -- `#` comments supported, -- entries are validated, -- wildcards (`*`) are rejected. - -If unset, module defaults are used. - -## `modsecurity_phase4_log` - -Enables dedicated JSON-lines logging for Phase 4 events. - ---- - -## 5) Mode behavior (`minimal`, `safe`, `strict`) - -## `minimal` - -Goal: least intrusive behavior. - -For interventions after headers were sent: - -- action is downgraded to `log_only`, -- no forced connection termination. - -Use when delivery continuity is prioritized. - -## `safe` - -Goal: conservative production baseline (module default merge behavior). - -For late interventions: - -- also downgraded to `log_only`. - -Use as default when you want phase:4 visibility without forced disconnect side effects. - -## `strict` - -Goal: stricter fallback once clean status rewrite is no longer possible. - -For interventions after headers were sent: - -- `connection_abort`. - -### Risks of `strict` - -- Active connections may terminate. -- Clients/proxies may observe transport interruption rather than a clean 4xx/3xx response. -- It does **not** mean “guaranteed 403/401/301/302”. - -Use only when those trade-offs are acceptable. - ---- - -## 6) Behavior by header state - -## Headers **not sent yet** - -If intervention is finalized before header send, normal deny/status behavior remains possible (logged as `deny_status` in code path). - -## Headers **already sent** - -- `minimal`: `log_only` -- `safe`: `log_only` -- `strict`: `connection_abort` - -This is an intentional downgrade to avoid false status guarantees. - ---- - -## 7) Why there is **no global response-body buffering** - -Global buffering of all responses could delay decision points, but introduces broad costs: - -- additional memory and latency overhead, -- higher complexity in generic response paths, -- increased risk of throughput/stability side effects. - -Current implementation instead makes late interventions explicit and controlled (`log_only` or `connection_abort`). - ---- - -## 8) Why there is no `ngx_chain_t` reordering/rewriting - -The module does **not** implement synthetic reordering of already flowing body chains to force post-hoc status semantics. - -Reasoning: - -- high implementation complexity, -- higher fragility, -- difficult correctness guarantees across all filter/upstream combinations. - -The documented downgrade model is more robust than pretending hard guarantees. - ---- - -## 9) Content-type scoping: meaning and safe usage - -`modsecurity_phase4_content_types_file` limits special phase-4 handling to selected MIME types. - -If `Content-Type` is missing or out of scope: - -- action is logged as `log_only`, -- reason is typically `content_type_missing` or `content_type_not_in_scope`. - -### Why this matters - -- reduces side effects on non-target response types, -- improves predictability, -- enforces explicit operator intent. - ---- - -## 10) Logging format and security boundaries - -With `modsecurity_phase4_log`, the module emits JSON lines including fields such as: - -- `event` (`phase4_intervention`) -- `uri`, `method` -- `response_status`, `waf_status` -- `content_type` -- `header_sent` -- `mode` -- `wanted_action`, `actual_action` -- `reason` -- `intervention` -- `rule_id` - -Additionally, nginx `error.log` may contain warnings (especially on strict late-intervention paths). - -### Logging boundary - -Tests explicitly check that response body payload is not leaked into phase-4 log output. - ---- - -## 11) Production examples - -General (non test-path-specific) example configurations: - -- `docs/examples/phase4-minimal.conf` -- `docs/examples/phase4-safe.conf` -- `docs/examples/phase4-strict.conf` -- `docs/examples/phase4-content-types.conf` - -These use `http` / `server` / `location /` patterns and avoid `/phase4` as a production example path. - ---- - -## 12) Test/demo behavior (explicitly separate) - -Repository tests include `/phase4` endpoints, for example in: - -- `tests/modsecurity.t` -- `tests/modsecurity-proxy.t` -- `tests/modsecurity-h2.t` -- `tests/modsecurity-proxy-h2.t` -- `tests/modsecurity-phase4-*.t` - -Those paths are **test context**, not generic production guidance. - ---- - -## 13) Known limits / no false promises - -- `phase:4` cannot **guarantee** clean 301/302/401/403 delivery once headers are already sent. -- For late interventions, only downgraded handling is possible (`log_only` or `connection_abort`). -- `strict` is not “guaranteed block status”; it can mean connection termination. - ---- - -## 14) Operator checklist - -1. Place hard block/redirect decisions in earlier phases whenever possible. -2. Start with `safe` unless you have a clear need for `strict`. -3. Use `strict` only when connection abort side effects are acceptable. -4. Enable `modsecurity_phase4_log` and monitor `actual_action` + `reason` fields. -5. Keep content-type scope narrow and review it regularly. +- `minimal` is no longer valid and is not an alias for `off`. +- The default mode changes from `safe` to `off`. Set `safe` or `strict` + explicitly to enable the additional connector policy. +- `modsecurity_phase4_content_types_file` is removed and causes an nginx + configuration error. Move MIME selection into ModSecurity rules. +- The positive connector body budget is new. Choose a value suitable for your + responses when enabling `safe` or `strict`; zero is invalid. + +Validate the migrated configuration with `nginx -t` in the deployment +environment before reloading nginx. + +## 3) Mode behavior and header timing +`off` disables the connector's additional phase 4 intervention policy and body +budget. It **does not disable ModSecurity**, response-body inspection, or phase +4 rules. Native intervention handling remains active. If a native intervention +occurs after headers have been committed, response finalization can fail the +transport; `off` is not a promise to deliver every response. + +For the additional policy in `safe` and `strict`: + +| Header state | `safe` | `strict` | +| --- | --- | --- | +| Headers not sent | Apply the intervention's status/redirect through the normal path; logged as `deny_status` | Same | +| Headers already sent | Log the intervention as `log_only` and continue | Log `connection_abort` and terminate the response | + +`safe` only downgrades late **interventions**. An engine API failure, invalid +buffer, failed file read, allocation failure, counter overflow, or exceeded +body budget remains a failure. It is not converted into successful forwarding. + +`strict` can produce a truncated response or a client/proxy transport error. +It cannot guarantee a clean 403, 401, 301, or 302 after headers have been sent. +Previously forwarded body bytes cannot be recalled. + +## 4) Response-body budget and streaming + +`modsecurity_phase4_body_limit` is a connector budget for the cumulative +response bytes it sees in `safe` and `strict`, including file-backed buffers. +It is separate from the engine's `SecResponseBodyLimit` and is not restricted +to the bytes the engine retains for inspection. + +A response may reach the budget exactly. A chunk that would cross it is +rejected **before that chunk is forwarded**. Earlier chunks may already have +reached the client, so this does not guarantee a replacement HTTP error +status. In `off`, the configured connector budget is ignored, but cumulative +accounting still rejects overflow beyond `SIZE_MAX`. + +The connector does not globally buffer responses or reorder nginx body +chains. Memory buffers are inspected directly. File-only buffers are read +through a bounded 32 KiB scratch buffer, without loading the entire file into +memory or replacing the original outgoing chain. Genuine processing or I/O +failures stop the affected response. + +The response body is finalized once per transaction: a main request uses +`last_buf`; a subrequest uses `last_in_chain`. Repeated filter calls after +finalization do not run phase 4 again. + +## 5) MIME selection belongs to ModSecurity + +The engine decides response-body inspection using `SecResponseBodyAccess`, +`SecResponseBodyMimeType`, and `SecResponseBodyMimeTypesClear`. There is no +connector content-type allowlist or connector MIME-based downgrade. + +For example, reset the MIME list in one rule load, then add the selected types +in a separate load: + +```nginx +modsecurity_rules 'SecResponseBodyMimeTypesClear'; +modsecurity_rules ' + SecResponseBodyAccess On + SecResponseBodyMimeType text/html text/plain application/json +'; +``` + +The reset is separate because libModSecurity's +[merge implementation](https://raw.githubusercontent.com/owasp-modsecurity/ModSecurity/v3/master/headers/modsecurity/rules_set_properties.h) +can clear MIME values added in the same rule load. + +The standalone [engine MIME example](examples/phase4-engine-mime.conf) contains +MIME additions and enables response-body access. It is a ModSecurity rules +file, not an nginx include. To replace the engine's MIME list with this file, +load the reset first, then the file, alongside your other rules: + +```nginx +modsecurity_rules 'SecResponseBodyMimeTypesClear'; +modsecurity_rules_file /etc/modsecurity/phase4-engine-mime.conf; +``` + +The complete nginx examples below configure MIME selection inline using the +same separate-load sequence. + +Engine selection does not disable the independent connector body budget in +`safe` or `strict`. + +## 6) Logging format and security boundary + +`modsecurity_phase4_log` preserves the existing JSON-lines intervention schema: + +- `event` (`phase4_intervention`), `uri`, `method`; +- `response_status`, `waf_status`, `content_type`, `header_sent`, `mode`; +- `wanted_action`, `actual_action`, `reason`; +- `intervention`, `rule_id`. + +The `actual_action` values remain `deny_status`, `log_only`, and +`connection_abort`; `mode` uses the current mode names. The intervention +message is redacted rather than copied from the engine, and response-body +payload is not added to this event. nginx's error log may also contain +diagnostics. + +`off` uses native intervention handling and does not produce these dedicated +policy events. A dedicated intervention event is not a complete inventory of +all engine, buffer, or I/O errors. + +## 7) Configuration examples and verification + +- [off](examples/phase4-off.conf): native intervention handling with engine + inspection enabled. +- [safe](examples/phase4-safe.conf): explicit late `log_only` policy and a + 1 MiB connector budget. +- [strict](examples/phase4-strict.conf): late connection termination and a + 1 MiB connector budget. +- [engine MIME selection](examples/phase4-engine-mime.conf): ModSecurity + response-body configuration. + +The nginx examples use `location /` and an example upstream at +`127.0.0.1:8081`. Adapt the listen address, upstream, log path, and rules to +your environment. The `sensitive-marker` rule is illustrative. + +Repository test endpoints such as `/phase4` are test fixtures, not required +production paths. Runtime checks should cover late deny/redirect behavior, +HTTP/1.1 and HTTP/2, file-backed responses, subrequests, repeated finalization, +budget boundaries, and failure paths. Results from another repository or build +do not establish those behaviors for the deployed module. \ No newline at end of file diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index 0b7223e8..b23d0f14 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -15,6 +15,7 @@ #include #include +#include #ifndef MODSECURITY_DDEBUG #define MODSECURITY_DDEBUG 0 @@ -23,8 +24,10 @@ #include "ngx_http_modsecurity_common.h" +/* Bounded, reusable materialization for file-only response buffers. */ +#define NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK 32768U + static ngx_http_output_body_filter_pt ngx_http_next_body_filter; -static ngx_int_t ngx_http_modsecurity_phase4_in_scope(ngx_http_request_t *r); static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason); static ngx_int_t ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf); static void ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst); @@ -42,218 +45,401 @@ ngx_http_modsecurity_body_filter_init(void) return NGX_OK; } -ngx_int_t -ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) +static ngx_int_t +ngx_http_modsecurity_response_body_failure(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) { - ngx_chain_t *chain = in; - ngx_http_modsecurity_ctx_t *ctx = NULL; - ngx_http_modsecurity_conf_t *mcf; -#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) - ngx_list_part_t *part = &r->headers_out.headers.part; - ngx_table_elt_t *data = part->elts; - ngx_uint_t i = 0; -#endif + ctx->intervention_triggered = 1; + if (r->header_sent) { + r->connection->error = 1; + return NGX_ERROR; + } + return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, + NGX_HTTP_INTERNAL_SERVER_ERROR); +} - if (in == NULL) { - return ngx_http_next_body_filter(r, in); +/* This is an inspection budget. SIZE_MAX in off is an accounting ceiling, + * never an allocation size; native engine and host limits remain active. */ +static ngx_int_t +ngx_http_modsecurity_plan_limited_response_body( + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf, + size_t len, size_t *allowed) +{ + size_t limit; + + if (allowed == NULL) { + return NGX_ERROR; } + *allowed = 0U; + if (ctx == NULL || mcf == NULL) { + return NGX_ERROR; + } + if (len == 0U) { + return NGX_OK; + } + ctx->response_body_seen = 1; + if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_OFF) { + limit = SIZE_MAX; + } else if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_SAFE || + mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { + limit = mcf->phase4_body_limit; + } else { + return NGX_ERROR; + } + if (limit == 0U || + ctx->response_body_bytes_inspected > ctx->response_body_bytes_seen || + ctx->response_body_bytes_inspected > limit) { + ctx->response_body_truncated = 1; + return NGX_ERROR; + } + if (len > SIZE_MAX - ctx->response_body_bytes_seen) { + ctx->response_body_bytes_seen = SIZE_MAX; + ctx->response_body_truncated = 1; + return NGX_ERROR; + } + ctx->response_body_bytes_seen += len; + if (ctx->response_body_bytes_seen > limit) { + ctx->response_body_truncated = 1; + return NGX_ERROR; + } + *allowed = len; + return NGX_OK; +} - ctx = ngx_http_modsecurity_get_module_ctx(r); +static ngx_int_t +ngx_http_modsecurity_append_response_body_chunk( + ngx_http_modsecurity_ctx_t *ctx, u_char *data, size_t bytes) +{ + if (bytes == 0U) { + return NGX_OK; + } + if (data == NULL || ctx->response_body_bytes_inspected > SIZE_MAX - bytes) { + return NGX_ERROR; + } + if (msc_append_response_body(ctx->modsec_transaction, data, bytes) < 0) { + return NGX_ERROR; + } + ctx->response_body_bytes_inspected += bytes; + return NGX_OK; +} - dd("body filter, recovering ctx: %p", ctx); +static ngx_int_t +ngx_http_modsecurity_append_limited_response_body( + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf, + u_char *data, size_t len) +{ + size_t allowed; - if (ctx == NULL) { - return ngx_http_next_body_filter(r, in); + if (ngx_http_modsecurity_plan_limited_response_body(ctx, mcf, len, + &allowed) != NGX_OK) { + return NGX_ERROR; } + return ngx_http_modsecurity_append_response_body_chunk(ctx, data, allowed); +} - if (ctx->intervention_triggered) { - return ngx_http_next_body_filter(r, in); +static ngx_int_t +ngx_http_modsecurity_append_file_response_body(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf, + ngx_buf_t *buffer) +{ + uintmax_t file_length; + size_t allowed; + size_t remaining; + size_t chunk; + off_t file_offset; + ssize_t read_count; + + if (buffer == NULL || buffer->file_pos < 0 || + buffer->file_last < buffer->file_pos) { + return NGX_ERROR; + } + file_length = (uintmax_t)buffer->file_last - (uintmax_t)buffer->file_pos; + if (file_length > (uintmax_t)SIZE_MAX || + ngx_http_modsecurity_plan_limited_response_body(ctx, mcf, + (size_t)file_length, &allowed) != NGX_OK) { + return NGX_ERROR; + } + if (allowed == 0U) { + return NGX_OK; + } + if (buffer->file == NULL) { + return NGX_ERROR; + } + if (ctx->phase4_file_scratch == NULL) { + ctx->phase4_file_scratch = ngx_pnalloc(r->pool, + NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK); + if (ctx->phase4_file_scratch == NULL) { + return NGX_ERROR; + } + } + file_offset = buffer->file_pos; + remaining = allowed; + while (remaining > 0U) { + chunk = remaining > NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK + ? NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK : remaining; + read_count = ngx_read_file(buffer->file, ctx->phase4_file_scratch, + chunk, file_offset); + if (read_count < 0 || (size_t)read_count != chunk) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, + read_count < 0 ? ngx_errno : 0, + "ModSecurity: file-backed response body read is short or failed"); + return NGX_ERROR; + } + if (ngx_http_modsecurity_append_response_body_chunk(ctx, + ctx->phase4_file_scratch, chunk) != NGX_OK) { + return NGX_ERROR; + } + file_offset += (off_t)chunk; + remaining -= chunk; } + return NGX_OK; +} -#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) - mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); - if (mcf != NULL && mcf->sanity_checks_enabled != NGX_CONF_UNSET) - { -#if 0 - dd("dumping stored ctx headers"); - for (i = 0; i < ctx->sanity_headers_out->nelts; i++) - { - ngx_http_modsecurity_header_t *vals = ctx->sanity_headers_out->elts; - ngx_str_t *s2 = &vals[i].name, *s3 = &vals[i].value; - dd(" dump[%d]: name = '%.*s', value = '%.*s'", (int)i, - (int)s2->len, (char*)s2->data, - (int)s3->len, (char*)s3->data); +static ngx_int_t +ngx_http_modsecurity_append_response_body_buffer(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf, + ngx_buf_t *buffer) +{ + if (buffer == NULL) { + return NGX_ERROR; + } + /* Buffers backed by both memory and a file describe the same bytes. */ + if (ngx_buf_in_memory(buffer)) { + if (buffer->pos == NULL || buffer->last == NULL || + buffer->last < buffer->pos) { + return NGX_ERROR; } -#endif - /* - * Identify if there is a header that was not inspected by ModSecurity. - */ - int worth_to_fail = 0; - - for (i = 0; ; i++) - { - int found = 0; - ngx_uint_t j = 0; - ngx_table_elt_t *s1; - ngx_http_modsecurity_header_t *vals; - - if (i >= part->nelts) - { - if (part->next == NULL) { - break; - } - - part = part->next; - data = part->elts; - i = 0; - } + return ngx_http_modsecurity_append_limited_response_body(ctx, mcf, + buffer->pos, (size_t)(buffer->last - buffer->pos)); + } + if (buffer->in_file) { + return ngx_http_modsecurity_append_file_response_body(r, ctx, mcf, + buffer); + } + return NGX_OK; +} - vals = ctx->sanity_headers_out->elts; - s1 = &data[i]; - - /* - * Headers that were inspected by ModSecurity. - */ - while (j < ctx->sanity_headers_out->nelts) - { - ngx_str_t *s2 = &vals[j].name; - ngx_str_t *s3 = &vals[j].value; - - if (s1->key.len == s2->len && ngx_strncmp(s1->key.data, s2->data, s1->key.len) == 0) - { - if (s1->value.len == s3->len && ngx_strncmp(s1->value.data, s3->data, s1->value.len) == 0) - { - found = 1; - break; - } - } - j++; +static ngx_int_t +ngx_http_modsecurity_process_response_intervention(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf) +{ + int ret; + + ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, + 0); + if (ret == 0) { + return NGX_OK; + } + if (ctx->intervention_failed || (ret < 0 && !ctx->intervention_late)) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_OFF) { + ctx->intervention_triggered = 1; + /* Retain the native intervention failure path in off. */ + return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, + ret < 0 ? NGX_HTTP_INTERNAL_SERVER_ERROR : ret); + } + ret = ngx_http_modsecurity_phase4_handle_intervention(r, mcf); + if (ctx->intervention_triggered || ret != NGX_OK) { + return ret; + } + return NGX_OK; +} + +static ngx_int_t +ngx_http_modsecurity_process_final_response_body(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf) +{ + ngx_pool_t *old_pool; + int ret; + + if (ctx->phase4_processed) { + return NGX_OK; + } + ctx->phase4_processed = 1; + old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); + ret = msc_process_response_body(ctx->modsec_transaction); + ngx_http_modsecurity_pcre_malloc_done(old_pool); + if (ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: response body phase processing failed"); + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + return ngx_http_modsecurity_process_response_intervention(r, ctx, mcf); +} + +#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) +static ngx_int_t +ngx_http_modsecurity_response_header_sanity(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf) +{ + ngx_list_part_t *part = &r->headers_out.headers.part; + ngx_table_elt_t *data = part->elts; + ngx_http_modsecurity_header_t *headers; + ngx_uint_t i = 0; + ngx_uint_t j; + + if (mcf->sanity_checks_enabled == NGX_CONF_UNSET) { + return NGX_OK; + } + if (ctx->sanity_headers_out == NULL) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + headers = ctx->sanity_headers_out->elts; + for (;;) { + while (i >= part->nelts) { + if (part->next == NULL) { + return NGX_OK; } - if (!found) { - dd("header: `%.*s' with value: `%.*s' was not inspected by ModSecurity", - (int) s1->key.len, - (const char *) s1->key.data, - (int) s1->value.len, - (const char *) s1->value.data); - worth_to_fail++; + part = part->next; + data = part->elts; + i = 0; + } + for (j = 0; j < ctx->sanity_headers_out->nelts; j++) { + if (data[i].key.len == headers[j].name.len && + ngx_strncmp(data[i].key.data, headers[j].name.data, + data[i].key.len) == 0 && + data[i].value.len == headers[j].value.len && + ngx_strncmp(data[i].value.data, headers[j].value.data, + data[i].value.len) == 0) { + break; } } - - if (worth_to_fail) - { - dd("%d header(s) were not inspected by ModSecurity, so exiting", worth_to_fail); - return ngx_http_filter_finalize_request(r, - &ngx_http_modsecurity_module, NGX_HTTP_INTERNAL_SERVER_ERROR); + if (j == ctx->sanity_headers_out->nelts) { + return ngx_http_modsecurity_response_body_failure(r, ctx); } + i++; } +} #endif - int is_request_processed = 0; - mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); - for (; chain != NULL; chain = chain->next) - { - u_char *data = chain->buf->pos; - int ret; - - msc_append_response_body(ctx->modsec_transaction, data, chain->buf->last - data); - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); - if (ret > 0) { - return ngx_http_filter_finalize_request(r, - &ngx_http_modsecurity_module, ret); - } else if (ret < 0) { - ret = ngx_http_modsecurity_phase4_handle_intervention(r, mcf); - if (ret == NGX_ERROR) return NGX_ERROR; +static void +ngx_http_modsecurity_discard_replaced_response_body(ngx_chain_t *in) +{ + ngx_chain_t *chain; + + for (chain = in; chain != NULL; chain = chain->next) { + if (chain->buf == NULL) { + continue; } + chain->buf->pos = chain->buf->last; + chain->buf->in_file = 0; + chain->buf->file_last = chain->buf->file_pos; + } +} -/* XXX: chain->buf->last_buf || chain->buf->last_in_chain */ - is_request_processed = chain->buf->last_buf; +ngx_int_t +ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + ngx_chain_t *chain; + ngx_http_modsecurity_ctx_t *ctx; + ngx_http_modsecurity_conf_t *mcf; + ngx_int_t ret; - if (!is_request_processed) { + if (in == NULL) { + return ngx_http_next_body_filter(r, in); + } + ctx = ngx_http_modsecurity_get_module_ctx(r); + if (ctx == NULL) { + return ngx_http_next_body_filter(r, in); + } + if (ctx->response_replaced) { + ngx_http_modsecurity_discard_replaced_response_body(in); + return ngx_http_next_body_filter(r, in); + } + if (ctx->intervention_triggered || ctx->phase4_processed) { + return ngx_http_next_body_filter(r, in); + } + mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + if (mcf == NULL) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } +#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) + ret = ngx_http_modsecurity_response_header_sanity(r, ctx, mcf); + if (ret != NGX_OK) { + return ret; + } +#endif + for (chain = in; chain != NULL; chain = chain->next) { + ret = ngx_http_modsecurity_append_response_body_buffer(r, ctx, mcf, + chain->buf); + if (ret != NGX_OK) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: response body inspection append failed"); + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + ret = ngx_http_modsecurity_process_response_intervention(r, ctx, mcf); + if (ret != NGX_OK || ctx->intervention_triggered) { + return ret; + } + /* A main response may contain last_in_chain before the real EOS. + * Subrequests finish with last_in_chain instead of last_buf. */ + if (!(chain->buf->last_buf || + (r != r->main && chain->buf->last_in_chain))) { continue; } - - { - ngx_pool_t *old_pool; - - old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_response_body(ctx->modsec_transaction); - ngx_http_modsecurity_pcre_malloc_done(old_pool); - -/* XXX: I don't get how body from modsec being transferred to nginx's buffer. If so - after adjusting of nginx's - XXX: body we can proceed to adjust body size (content-length). see xslt_body_filter() for example */ - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); - if (ret > 0) { - if (!ctx->phase4_headers_checked) { - ngx_http_modsecurity_phase4_log_event(r, mcf, "deny", "deny_status", "headers_not_sent"); - ctx->phase4_headers_checked = 1; - } - return ret; - } - if (ret < 0) { - ret = ngx_http_modsecurity_phase4_handle_intervention(r, mcf); - if (ret == NGX_ERROR) { - return NGX_ERROR; - } - return ngx_http_next_body_filter(r, in); - } + ret = ngx_http_modsecurity_process_final_response_body(r, ctx, mcf); + if (ret != NGX_OK || ctx->intervention_triggered) { + return ret; } + /* Subsequent flush/terminal links still belong to NGINX; never + * append bytes after the transaction has completed. */ + break; } - if (!is_request_processed) - { - dd("buffer was not fully loaded! ctx: %p", ctx); - } - -/* XXX: xflt_filter() -- return NGX_OK here */ return ngx_http_next_body_filter(r, in); } static ngx_int_t -ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf) +ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, + ngx_http_modsecurity_conf_t *mcf) { ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); - ngx_int_t in_scope = ngx_http_modsecurity_phase4_in_scope(r); const char *wanted = "deny"; - if (ctx && ctx->last_intervention_status >= 300 && ctx->last_intervention_status < 400) { + ngx_int_t log_result; + ngx_int_t status; + + if (mcf == NULL || ctx == NULL) { + return NGX_ERROR; + } + if (ctx->last_intervention_status >= 300 && + ctx->last_intervention_status < 400) { wanted = "redirect"; } - if (ctx && ctx->phase4_headers_checked) return NGX_OK; - if (ctx) ctx->phase4_headers_checked = 1; - - if (in_scope == 0) { - ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, "log_only", r->headers_out.content_type.len ? "content_type_not_in_scope" : "content_type_missing"); + if (ctx->phase4_headers_checked) { return NGX_OK; } + ctx->phase4_headers_checked = 1; + if (!r->header_sent) { + status = ctx->last_intervention_status >= 300 + ? ctx->last_intervention_status : NGX_HTTP_FORBIDDEN; + log_result = ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, + "deny_status", + "response_not_committed"); + ctx->intervention_triggered = 1; + if (log_result != NGX_OK) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, + status); + } if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { - ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, "connection_abort", "headers_already_sent"); + ctx->phase4_strict_abort = 1; + ctx->intervention_triggered = 1; + r->connection->error = 1; ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "modsecurity phase4 intervention after headers sent, action=connection_abort, uri=\"%V\"", &r->uri); - r->connection->error = 1; + (void)ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, + "connection_abort", "response_committed_strict"); return NGX_ERROR; } - ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, "log_only", - mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL ? "mode_minimal" : "mode_safe"); + log_result = ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, + "log_only", "response_committed_safe"); + if (log_result != NGX_OK) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } return NGX_OK; } -static ngx_int_t -ngx_http_modsecurity_phase4_in_scope(ngx_http_request_t *r) -{ - ngx_http_modsecurity_conf_t *mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); - ngx_uint_t i; - ngx_str_t ct; - u_char *semi; - if (r->headers_out.content_type.len == 0 || mcf->phase4_content_types == NULL) return 0; - ct = r->headers_out.content_type; - semi = (u_char *)ngx_strlchr(ct.data, ct.data + ct.len, ';'); - if (semi != NULL) ct.len = semi - ct.data; - while (ct.len > 0 && isspace((unsigned char)ct.data[ct.len - 1])) ct.len--; - for (i = 0; i < mcf->phase4_content_types->nelts; i++) { - ngx_str_t *arr = mcf->phase4_content_types->elts; - if (arr[i].len == ct.len && ngx_strncasecmp(arr[i].data, ct.data, ct.len) == 0) return 1; - } - return 0; -} - static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason) { @@ -265,10 +451,10 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit ngx_str_t erule; ngx_str_t raw_log; ngx_str_t slog; - const char *mode = "safe"; + const char *mode = "off"; const char *header_sent = r->header_sent ? "true" : "false"; ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); - if (mcf->phase4_log_file == NULL || mcf->phase4_log_file->fd == NGX_INVALID_FILE) return NGX_OK; + if (mcf == NULL || mcf->phase4_log_file == NULL || mcf->phase4_log_file->fd == NGX_INVALID_FILE) return NGX_OK; ngx_http_modsecurity_json_escape(r->pool, &r->uri, &euri); ngx_http_modsecurity_json_escape(r->pool, &r->method_name, &emethod); ngx_str_t nct = ngx_http_modsecurity_normalize_content_type(r->pool, r->headers_out.content_type); @@ -283,7 +469,7 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit elog.len = 0; elog.data=(u_char*)""; erule.len = 0; erule.data=(u_char*)""; } - if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL) mode = "minimal"; + if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_SAFE) mode = "safe"; else if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) mode = "strict"; size_t need = 256 + euri.len + emethod.len + ect.len + elog.len + erule.len + ngx_strlen(mode) + ngx_strlen(wanted) + ngx_strlen(actual) + ngx_strlen(reason); u_char *dbuf = ngx_pnalloc(r->pool, need); diff --git a/src/ngx_http_modsecurity_common.h b/src/ngx_http_modsecurity_common.h index de9941cb..30f5fca8 100644 --- a/src/ngx_http_modsecurity_common.h +++ b/src/ngx_http_modsecurity_common.h @@ -71,7 +71,8 @@ #define MODSECURITY_NGINX_WHOAMI "ModSecurity-nginx v" \ MODSECURITY_NGINX_VERSION -#define NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL 0 +#define NGX_HTTP_MODSEC_PHASE4_MODE_OFF 0 +#define NGX_HTTP_MODSEC_PHASE4_BODY_LIMIT_DEFAULT 1048576U #define NGX_HTTP_MODSEC_PHASE4_MODE_SAFE 1 #define NGX_HTTP_MODSEC_PHASE4_MODE_STRICT 2 @@ -105,6 +106,17 @@ typedef struct { unsigned intervention_triggered:1; unsigned request_body_processed:1; unsigned phase4_headers_checked:1; + unsigned phase4_processed:1; + unsigned phase4_strict_abort:1; + unsigned response_body_seen:1; + unsigned response_body_truncated:1; + unsigned intervention_late:1; + unsigned intervention_failed:1; + unsigned response_replaced:1; + unsigned intervention_redirect_location_installed:1; + size_t response_body_bytes_seen; + size_t response_body_bytes_inspected; + u_char *phase4_file_scratch; ngx_str_t last_intervention_log; ngx_int_t last_intervention_status; } ngx_http_modsecurity_ctx_t; @@ -132,8 +144,7 @@ typedef struct { ngx_http_complex_value_t *transaction_id; ngx_uint_t phase4_mode; - ngx_array_t *phase4_content_types; - ngx_str_t phase4_content_types_file; + size_t phase4_body_limit; ngx_open_file_t *phase4_log_file; ngx_str_t phase4_log_path; } ngx_http_modsecurity_conf_t; diff --git a/src/ngx_http_modsecurity_header_filter.c b/src/ngx_http_modsecurity_header_filter.c index 03b8764d..6189ead1 100644 --- a/src/ngx_http_modsecurity_header_filter.c +++ b/src/ngx_http_modsecurity_header_filter.c @@ -526,13 +526,44 @@ ngx_http_modsecurity_header_filter(ngx_http_request_t *r) #endif old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_response_headers(ctx->modsec_transaction, status, http_response_ver); + ret = msc_process_response_headers(ctx->modsec_transaction, status, + http_response_ver); ngx_http_modsecurity_pcre_malloc_done(old_pool); + if (ret != 1) { + ctx->intervention_triggered = 1; + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: response header phase processing failed"); + return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, + NGX_HTTP_INTERNAL_SERVER_ERROR); + } ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); - if (r->error_page) { - return ngx_http_next_header_filter(r); + if (ret < 0) { + ctx->intervention_triggered = 1; + r->connection->error = 1; + return NGX_ERROR; } if (ret > 0) { + /* An error_page response must honor its own response intervention. + * Mark the decision before finalization re-enters the filter. */ + ctx->intervention_triggered = 1; + if (ctx->intervention_redirect_location_installed) { + ctx->response_replaced = 1; + ngx_http_clear_content_length(r); + ngx_http_clear_last_modified(r); + ngx_http_clear_etag(r); + ngx_http_clear_accept_ranges(r); + ngx_str_null(&r->headers_out.content_type); + r->headers_out.content_type_len = 0; + if (r->headers_out.content_encoding != NULL) { + r->headers_out.content_encoding->hash = 0; + r->headers_out.content_encoding = NULL; + } + r->headers_out.status = ret; + ngx_str_null(&r->headers_out.status_line); + r->headers_out.content_length_n = 0; + r->header_only = 1; + return ngx_http_next_header_filter(r); + } return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, ret); } diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index e397dfd1..94d56b66 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -38,11 +38,8 @@ static char *ngx_http_modsecurity_merge_conf(ngx_conf_t *cf, void *parent, void static void ngx_http_modsecurity_cleanup_instance(void *data); static void ngx_http_modsecurity_cleanup_rules(void *data); static char *ngx_conf_set_phase4_mode(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); -static char *ngx_conf_set_phase4_content_types_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char *ngx_conf_set_phase4_body_limit(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_conf_set_phase4_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); -static char *ngx_http_modsecurity_phase4_set_default_content_types(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf); -static char *ngx_http_modsecurity_phase4_load_content_types_file(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf, ngx_str_t *path); -static ngx_int_t ngx_http_modsecurity_phase4_validate_content_type(u_char *s, size_t len); /* * PCRE malloc/free workaround, based on @@ -143,129 +140,118 @@ ngx_inline char *ngx_str_to_char(ngx_str_t a, ngx_pool_t *p) int -ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_request_t *r, ngx_int_t early_log) +ngx_http_modsecurity_process_intervention (Transaction *transaction, + ngx_http_request_t *r, ngx_int_t early_log) { - char *log = NULL; ModSecurityIntervention intervention; - intervention.status = 200; - intervention.url = NULL; - intervention.log = NULL; - intervention.disruptive = 0; - ngx_http_modsecurity_ctx_t *ctx = NULL; - ngx_http_modsecurity_conf_t *mcf; - - dd("processing intervention"); + ngx_http_modsecurity_ctx_t *ctx; + ngx_http_modsecurity_conf_t *mcf; + ngx_table_elt_t *location; + u_char *url; + size_t len; + size_t i; + int result; ctx = ngx_http_modsecurity_get_module_ctx(r); - if (ctx == NULL) - { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + if (ctx != NULL) { + ctx->intervention_late = 0; + ctx->intervention_failed = 0; + ctx->intervention_redirect_location_installed = 0; } - + if (ctx == NULL || mcf == NULL) { + if (ctx != NULL) ctx->intervention_failed = 1; + return r->header_sent ? NGX_ERROR : NGX_HTTP_INTERNAL_SERVER_ERROR; + } + ngx_memzero(&intervention, sizeof(intervention)); + intervention.status = 200; if (msc_intervention(transaction, &intervention) == 0) { - dd("nothing to do"); return 0; } - ctx->last_intervention_status = intervention.status; - ctx->last_intervention_log.len = 0; - ctx->last_intervention_log.data = NULL; - mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); - if (mcf == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - if (mcf->phase4_log_file != NULL && r->header_sent && intervention.log != NULL) { - size_t l = ngx_strlen(intervention.log); - u_char *cp = ngx_pnalloc(r->pool, l + 1); - if (cp != NULL) { - ngx_memcpy(cp, intervention.log, l); - cp[l] = '\0'; - ctx->last_intervention_log.data = cp; - ctx->last_intervention_log.len = l; + result = r->header_sent ? NGX_ERROR : NGX_HTTP_INTERNAL_SERVER_ERROR; + ctx->intervention_failed = 1; + ctx->last_intervention_status = intervention.status; + ngx_str_null(&ctx->last_intervention_log); + if (mcf->phase4_log_file != NULL && intervention.log != NULL) { + len = ngx_strlen(intervention.log); + ctx->last_intervention_log.data = ngx_pnalloc(r->pool, len + 1); + if (ctx->last_intervention_log.data == NULL) { + goto done; } + ngx_memcpy(ctx->last_intervention_log.data, intervention.log, len + 1); + ctx->last_intervention_log.len = len; } - - // logging to nginx error log can be disable by setting `modsecurity_use_error_log` to off if (mcf->use_error_log) { - log = intervention.log; - if (intervention.log == NULL) { - log = "(no log message was specified)"; - } - ngx_log_error(NGX_LOG_ERR, (ngx_log_t *)r->connection->log, 0, "%s", log); - } - - if (intervention.log != NULL) { - free(intervention.log); + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "%s", + intervention.log != NULL ? intervention.log : + "(no log message was specified)"); } - if (intervention.url != NULL) - { - dd("intervention -- redirecting to: %s with status code: %d", intervention.url, intervention.status); - - if (r->header_sent) - { - dd("Headers are already sent. Cannot perform the redirection at this point."); - return -1; + if (intervention.url != NULL) { + if (r->header_sent) { + ctx->intervention_late = 1; + ctx->intervention_failed = 0; + goto done; } - - /** - * Not sure if it sane to do this indepent of the phase - * but, here we go... - * - * This code cames from: http/ngx_http_special_response.c - * function: ngx_http_send_error_page - * src/http/ngx_http_core_module.c - * From src/http/ngx_http_core_module.c (line 1910) i learnt - * that location->hash should be set to 1. - * - */ - ngx_http_clear_location(r); - ngx_str_t a = ngx_string(""); - - a.data = (unsigned char *)intervention.url; - a.len = strlen(intervention.url); - - ngx_table_elt_t *location = NULL; + len = ngx_strlen(intervention.url); + for (i = 0; i < len; i++) { + if (intervention.url[i] == '\r' || intervention.url[i] == '\n') { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: intervention redirect URL contains CR or LF"); + result = NGX_HTTP_BAD_REQUEST; + goto done; + } + } + url = ngx_pnalloc(r->pool, len); + if (url == NULL) { + goto done; + } + ngx_memcpy(url, intervention.url, len); location = ngx_list_push(&r->headers_out.headers); + if (location == NULL) { + goto done; + } + ngx_http_clear_location(r); ngx_str_set(&location->key, "Location"); - location->value = a; + location->value.data = url; + location->value.len = len; + location->hash = 1; r->headers_out.location = location; - r->headers_out.location->hash = 1; - #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) - ngx_http_modsecurity_store_ctx_header(r, &location->key, &location->value); + if (ngx_http_modsecurity_store_ctx_header(r, &location->key, + &location->value) != NGX_OK) { + goto done; + } #endif - - return intervention.status; - } - - if (intervention.status != 200) - { - /** - * FIXME: this will bring proper response code to audit log in case - * when e.g. error_page redirect was triggered, but there still won't be another - * required pieces like response headers etc. - * - */ + ctx->intervention_redirect_location_installed = 1; + ctx->intervention_failed = 0; + result = intervention.status; + } else if (intervention.status != 200) { msc_update_status_code(ctx->modsec_transaction, intervention.status); - if (early_log) { - dd("intervention -- calling log handler manually with code: %d", intervention.status); ngx_http_modsecurity_log_handler(r); ctx->logged = 1; - } - - if (r->header_sent) - { - dd("Headers are already sent. Cannot perform the redirection at this point."); - return -1; } - dd("intervention -- returning code: %d", intervention.status); - return intervention.status; + if (r->header_sent) { + ctx->intervention_late = 1; + ctx->intervention_failed = 0; + goto done; + } + ctx->intervention_failed = 0; + result = intervention.status; + } else { + ctx->intervention_failed = 0; + result = 0; } - return 0; -} +done: + /* libModSecurity owns these allocations until the intervention is read. + * NGINX headers and metadata retain only request-pool copies. */ + if (intervention.log != NULL) free(intervention.log); + if (intervention.url != NULL) free(intervention.url); + return result; +} void ngx_http_modsecurity_cleanup(void *data) @@ -301,6 +287,7 @@ ngx_http_modsecurity_create_ctx(ngx_http_request_t *r) dd("failed to allocate memory for the context."); return NULL; } + ctx->r = r; mmcf = ngx_http_get_module_main_conf(r, ngx_http_modsecurity_module); mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); @@ -344,22 +331,25 @@ ngx_inline ngx_http_modsecurity_ctx_t * ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r) { ngx_http_modsecurity_ctx_t *ctx; + ngx_http_modsecurity_ctx_t *candidate; + ngx_pool_cleanup_t *cln; + ctx = ngx_http_get_module_ctx(r, ngx_http_modsecurity_module); - if (ctx == NULL) { - /* - * refer /src/http/modules/ngx_http_realip_module.c - * if module context was reset, the original address - * can still be found in the cleanup handler - */ - ngx_pool_cleanup_t *cln; - for (cln = r->pool->cleanup; cln; cln = cln->next) { - if (cln->handler == ngx_http_modsecurity_cleanup) { - ctx = cln->data; - break; - } + if (ctx != NULL && ctx->r == r) { + return ctx; + } + /* Internal redirects reset module contexts but retain the same request. + * Subrequests share its pool, so cleanup recovery must match the owner. */ + for (cln = r->pool->cleanup; cln; cln = cln->next) { + if (cln->handler != ngx_http_modsecurity_cleanup) { + continue; + } + candidate = cln->data; + if (candidate != NULL && candidate->r == r) { + return candidate; } } - return ctx; + return NULL; } char * @@ -503,20 +493,32 @@ ngx_conf_set_phase4_mode(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_modsecurity_conf_t *mcf = conf; ngx_str_t *value = cf->args->elts; - if (ngx_strcmp(value[1].data, "minimal") == 0) mcf->phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_MINIMAL; + (void) cmd; + if (mcf->phase4_mode != NGX_CONF_UNSET_UINT) return "is duplicate"; + if (ngx_strcmp(value[1].data, "off") == 0) mcf->phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_OFF; else if (ngx_strcmp(value[1].data, "safe") == 0) mcf->phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_SAFE; else if (ngx_strcmp(value[1].data, "strict") == 0) mcf->phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_STRICT; - else return "invalid value for modsecurity_phase4_mode (expected minimal|safe|strict)"; + else return "invalid value for modsecurity_phase4_mode (expected off|safe|strict)"; return NGX_CONF_OK; } static char * -ngx_conf_set_phase4_content_types_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +ngx_conf_set_phase4_body_limit(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_modsecurity_conf_t *mcf = conf; ngx_str_t *value = cf->args->elts; - mcf->phase4_content_types_file = value[1]; - return ngx_http_modsecurity_phase4_load_content_types_file(cf, mcf, &value[1]); + ssize_t limit; + + (void) cmd; + if (mcf->phase4_body_limit != NGX_CONF_UNSET_SIZE) { + return "is duplicate"; + } + limit = ngx_parse_size(&value[1]); + if (limit <= 0) { + return "invalid value for modsecurity_phase4_body_limit"; + } + mcf->phase4_body_limit = (size_t) limit; + return NGX_CONF_OK; } static char * @@ -532,156 +534,6 @@ ngx_conf_set_phase4_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_OK; } -static ngx_int_t -ngx_http_modsecurity_phase4_validate_content_type(u_char *s, size_t len) -{ - size_t i; - size_t slash = (size_t)-1; - if (len == 0 || ngx_strchr(s, '*') != NULL) return NGX_ERROR; - for (i = 0; i < len; i++) { - u_char c = s[i]; - if (c == '/') { - if (slash != (size_t)-1 || i == 0 || i + 1 >= len) return NGX_ERROR; - slash = i; - continue; - } - if (!(isalnum((unsigned char)c) || c == '-' || c == '.' || c == '_' || c == '+')) return NGX_ERROR; - } - return (slash != (size_t)-1) ? NGX_OK : NGX_ERROR; -} - -static char * -ngx_http_modsecurity_phase4_set_default_content_types(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf) -{ - static const char *defs[] = {"text/html","text/plain","application/json","application/xml","text/xml","application/xhtml+xml"}; - ngx_uint_t i; - if (mcf->phase4_content_types != NULL) return NGX_CONF_OK; - mcf->phase4_content_types = ngx_array_create(cf->pool, 6, sizeof(ngx_str_t)); - if (mcf->phase4_content_types == NULL) return NGX_CONF_ERROR; - for (i = 0; i < 6; i++) { - ngx_str_t *ct = ngx_array_push(mcf->phase4_content_types); - if (ct == NULL) return NGX_CONF_ERROR; - ct->len = ngx_strlen(defs[i]); - ct->data = ngx_pnalloc(cf->pool, ct->len); - if (ct->data == NULL) return NGX_CONF_ERROR; - ngx_memcpy(ct->data, defs[i], ct->len); - } - return NGX_CONF_OK; -} - -static ngx_int_t -ngx_http_modsecurity_phase4_push_content_type(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf, - u_char *line, u_char *end, ngx_str_t *path) -{ - ngx_str_t *ct; - - ngx_strlow(line, line, end - line); - if (ngx_http_modsecurity_phase4_validate_content_type(line, end - line) != NGX_OK) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid content-type entry in modsecurity_phase4_content_types_file \"%V\": \"%s\"", path, line); - return NGX_ERROR; - } - - ct = ngx_array_push(mcf->phase4_content_types); - if (ct == NULL) { - return NGX_ERROR; - } - ct->len = end - line; - ct->data = ngx_pnalloc(cf->pool, ct->len); - if (ct->data == NULL) { - return NGX_ERROR; - } - ngx_memcpy(ct->data, line, ct->len); - - return NGX_OK; -} - -static void -ngx_http_modsecurity_phase4_trim_line(u_char **line, u_char **end) -{ - while (*line < *end && isspace((unsigned char)**line)) { - (*line)++; - } - - while (*end > *line && isspace((unsigned char)*((*end) - 1))) { - (*end)--; - } - - **end = '\0'; -} - -static void -ngx_http_modsecurity_phase4_strip_inline_comment(u_char *line) -{ - u_char *hash = (u_char *) ngx_strchr(line, '#'); - u_char *semi = (u_char *) ngx_strchr(line, ';'); - - if (hash && (!semi || hash < semi)) { - *hash = '\0'; - } - if (semi) { - *semi = '\0'; - } -} - -static char * -ngx_http_modsecurity_phase4_load_content_types_file(ngx_conf_t *cf, ngx_http_modsecurity_conf_t *mcf, ngx_str_t *path) -{ - ngx_file_t file; - ngx_file_info_t fi; - u_char *buf; - u_char *p; - u_char *line; - u_char *end; - ssize_t n; - if (ngx_file_info(path->data, &fi) == NGX_FILE_ERROR) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno, "modsecurity_phase4_content_types_file \"%V\" stat() failed", path); - return NGX_CONF_ERROR; - } - buf = ngx_pnalloc(cf->pool, ngx_file_size(&fi) + 1); - if (buf == NULL) return NGX_CONF_ERROR; - ngx_memzero(&file, sizeof(file)); - file.name = *path; - file.log = cf->log; - file.fd = ngx_open_file(path->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); - if (file.fd == NGX_INVALID_FILE) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno, "modsecurity_phase4_content_types_file \"%V\" open() failed", path); - return NGX_CONF_ERROR; - } - n = ngx_read_file(&file, buf, ngx_file_size(&fi), 0); - ngx_close_file(file.fd); - if (n < 0) return NGX_CONF_ERROR; - buf[n] = '\0'; - mcf->phase4_content_types = ngx_array_create(cf->pool, 8, sizeof(ngx_str_t)); - if (mcf->phase4_content_types == NULL) return NGX_CONF_ERROR; - for (p = buf, line = buf; p <= buf + n; p++) { - if (p == buf + n || *p == '\n' || *p == '\r') { - *p = '\0'; - end = p; - - ngx_http_modsecurity_phase4_trim_line(&line, &end); - if (line[0] == '\0' || line[0] == '#') { - line = p + 1; - continue; - } - - ngx_http_modsecurity_phase4_strip_inline_comment(line); - end = line + ngx_strlen(line); - ngx_http_modsecurity_phase4_trim_line(&line, &end); - if (line[0] == '\0') { - line = p + 1; - continue; - } - - if (ngx_http_modsecurity_phase4_push_content_type(cf, mcf, line, end, path) != NGX_OK) { - return NGX_CONF_ERROR; - } - line = p + 1; - } - } - return NGX_CONF_OK; -} - - static ngx_command_t ngx_http_modsecurity_commands[] = { { ngx_string("modsecurity"), @@ -732,9 +584,9 @@ static ngx_command_t ngx_http_modsecurity_commands[] = { NULL }, { - ngx_string("modsecurity_phase4_content_types_file"), + ngx_string("modsecurity_phase4_body_limit"), NGX_HTTP_LOC_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, - ngx_conf_set_phase4_content_types_file, + ngx_conf_set_phase4_body_limit, NGX_HTTP_LOC_CONF_OFFSET, 0, NULL @@ -951,10 +803,8 @@ ngx_http_modsecurity_create_conf(ngx_conf_t *cf) conf->transaction_id = NGX_CONF_UNSET_PTR; conf->use_error_log = NGX_CONF_UNSET; conf->phase4_mode = NGX_CONF_UNSET_UINT; - conf->phase4_content_types = NULL; - conf->phase4_content_types_file.len = 0; - conf->phase4_content_types_file.data = NULL; - conf->phase4_log_file = NULL; + conf->phase4_body_limit = NGX_CONF_UNSET_SIZE; + conf->phase4_log_file = NGX_CONF_UNSET_PTR; conf->phase4_log_path.len = 0; conf->phase4_log_path.data = NULL; #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) @@ -997,9 +847,10 @@ ngx_http_modsecurity_merge_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(c->enable, p->enable, 0); ngx_conf_merge_ptr_value(c->transaction_id, p->transaction_id, NULL); ngx_conf_merge_value(c->use_error_log, p->use_error_log, 1); - ngx_conf_merge_uint_value(c->phase4_mode, p->phase4_mode, NGX_HTTP_MODSEC_PHASE4_MODE_SAFE); + ngx_conf_merge_uint_value(c->phase4_mode, p->phase4_mode, NGX_HTTP_MODSEC_PHASE4_MODE_OFF); + ngx_conf_merge_size_value(c->phase4_body_limit, p->phase4_body_limit, + NGX_HTTP_MODSEC_PHASE4_BODY_LIMIT_DEFAULT); ngx_conf_merge_ptr_value(c->phase4_log_file, p->phase4_log_file, NULL); - ngx_conf_merge_ptr_value(c->phase4_content_types, p->phase4_content_types, NULL); #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) ngx_conf_merge_value(c->sanity_checks_enabled, p->sanity_checks_enabled, 0); #endif @@ -1020,9 +871,6 @@ ngx_http_modsecurity_merge_conf(ngx_conf_t *cf, void *parent, void *child) dd("NEW CHILD RULES"); msc_rules_dump(c->rules_set); #endif - if (c->phase4_content_types == NULL) { - return ngx_http_modsecurity_phase4_set_default_content_types(cf, c); - } return NGX_CONF_OK; } diff --git a/tests/README.md b/tests/README.md index 3c9bdb18..bb863e97 100644 --- a/tests/README.md +++ b/tests/README.md @@ -7,4 +7,48 @@ For more information about those tests, read the subsection "Testing your patch" on the project's README file. For more about nginx tests, check their repository: -http://hg.nginx.org/nginx-tests/ \ No newline at end of file +http://hg.nginx.org/nginx-tests/ + +## Standalone Phase4 helper regressions + +Run from the repository root with Python 3 and GCC or Clang: + +```sh +CC=gcc python3 tests/test_phase4_runtime.py -v +CC=clang python3 tests/test_phase4_runtime.py -v +``` + +The suite compiles the production body-filter functions and request-context +lookup with small NGINX and libModSecurity doubles. It checks mode-aware byte +limits, overflow, memory/file buffers, bounded file reads and allocation/read +errors, finalization at EOS, native failures, late interventions, downstream +`NGX_AGAIN`, request ownership, and internal redirect recovery. It needs no +Common connector library. These are helper behavior tests; they do not replace +the native HTTP integration tests. + +## Native Phase4 integration tests + +Copy `tests/*.t` and `tests/*.pl` into an nginx-tests checkout after building +NGINX with this connector and libModSecurity. From that checkout, run: + +```sh +TEST_NGINX_BINARY=/absolute/path/to/nginx prove modsecurity-phase4-*.t +TEST_NGINX_BINARY=/absolute/path/to/nginx prove modsecurity*.t +``` + +The Phase4 suites cover `off` (the default), `safe`, and `strict`, JSON event +logging, engine-owned `SecResponseBodyMimeType` selection, complete response +delivery above the optional connector budget in `off`, and budget rejection +in `safe`/`strict`. They also reject the removed `minimal` mode and connector +MIME directive. Late deny/redirect assertions check transport interruption, +because headers have already been committed; they do not promise a clean 403. + +A separate GitHub workflow job runs the standalone suite immediately with +both Linux compilers, independently of the libModSecurity build. The native +Perl suites run in the Linux and Windows build jobs. + +`modsecurity-phase4-subrequest.t` checks that a disabled, header-only +`auth_request` response cannot consume the main transaction's response headers +or MIME selection. The main response must still run its Phase3 header rule and +Phase4 body rule. Direct subrequest EOS ownership is exercised separately by +the compiled helper suite; the auth fixture does not claim to emit a body EOS. diff --git a/tests/modsecurity-phase4-content-types.t b/tests/modsecurity-phase4-content-types.t index 412e9fea..5500d136 100644 --- a/tests/modsecurity-phase4-content-types.t +++ b/tests/modsecurity-phase4-content-types.t @@ -1,25 +1,13 @@ #!/usr/bin/perl use warnings; use strict; use Test::More; +use JSON::PP qw(decode_json); BEGIN { use FindBin; chdir($FindBin::Bin); } use lib 'lib'; use Test::Nginx; -# content-type parsing and scope behavior +# libModSecurity owns response MIME selection; the connector has no MIME list. my $t = Test::Nginx->new()->has(qw/http/); - -$t->write_file('phase4-content-types.conf', <<'CT'); -# comments and whitespace - -Application/JSON; charset=utf-8 # inline comment -TEXT/HTML -text/plain -application/xml -application/vnd.api+json -application/problem+json -application/x.custom_type -CT - $t->write_file_expand('nginx.conf', <<'EOC'); %%TEST_GLOBALS%% daemon off; @@ -29,65 +17,43 @@ http { server { listen 127.0.0.1:8080; server_name localhost; - - location /json { - modsecurity on; - modsecurity_phase4_mode strict; - modsecurity_phase4_log %%TESTDIR%%/phase4-content-types.log; - modsecurity_phase4_content_types_file %%TESTDIR%%/phase4-content-types.conf; - default_type application/json; - modsecurity_rules ' - SecRuleEngine On - SecResponseBodyAccess On - SecRule RESPONSE_BODY "@rx HIT" "id:920001,phase:4,deny,log,status:403,msg:\"ct\"" - '; - } - - location /unknown { - modsecurity on; - modsecurity_phase4_mode strict; - modsecurity_phase4_log %%TESTDIR%%/phase4-content-types.log; - modsecurity_phase4_content_types_file %%TESTDIR%%/phase4-content-types.conf; - default_type image/png; - modsecurity_rules ' - SecRuleEngine On - SecResponseBodyAccess On - SecRule RESPONSE_BODY "@rx HIT" "id:920002,phase:4,deny,log,status:403,msg:\"ct\"" - '; - } - - location /emptytype { - modsecurity on; - modsecurity_phase4_mode strict; - modsecurity_phase4_log %%TESTDIR%%/phase4-content-types.log; - modsecurity_phase4_content_types_file %%TESTDIR%%/phase4-content-types.conf; - types { } - default_type ""; - modsecurity_rules ' - SecRuleEngine On - SecResponseBodyAccess On - SecRule RESPONSE_BODY "@rx HIT" "id:920003,phase:4,deny,log,status:403,msg:\"ct\"" - '; - } + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log %%TESTDIR%%/phase4-content-types.log; + # Separate loads exercise the engine's sequential clear/add merge. + modsecurity_rules 'SecResponseBodyMimeType text/plain'; + modsecurity_rules 'SecResponseBodyMimeTypesClear'; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyMimeType application/json + SecRule RESPONSE_BODY "@rx HIT" "id:920001,phase:4,deny,log,status:403" + '; + location /json { default_type application/json; } + location /unknown { default_type image/png; } + location /plain { default_type text/plain; } } } EOC $t->write_file('/json', 'HIT JSON'); -$t->write_file('/unknown', 'HIT UNKNOWN'); -$t->write_file('/emptytype', 'HIT EMPTY'); - +$t->write_file('/unknown', 'HIT PNG'); +$t->write_file('/plain', 'HIT PLAIN'); $t->run(); -$t->plan(9); - -is(http_get('/json'), '', 'json in-scope + strict => abort after headers sent'); -like(http_get('/unknown'), qr/HIT UNKNOWN/, 'unknown content-type not in scope => no hard action'); -like(http_get('/emptytype'), qr/HIT EMPTY/, 'empty content-type => no hard action'); +$t->plan(8); +is(http_get('/json'), '', 'engine-selected JSON triggers strict late abort'); +like(http_get('/unknown'), qr/HIT PNG/, 'engine-excluded image body continues'); +like(http_get('/plain'), qr/HIT PLAIN/, 'engine MIME clear/add override excludes inherited text/plain'); my $log = $t->read_file('phase4-content-types.log'); -like($log, qr/"content_type":"application\/json"/, 'json content type logged'); -like($log, qr/"actual_action":"connection_abort"/, 'strict in-scope abort logged'); -like($log, qr/"reason":"content_type_not_in_scope"/, 'out-of-scope reason logged'); -like($log, qr/"event":"phase4_intervention"/, 'json lines event present'); -unlike($log, qr/HIT JSON|HIT UNKNOWN|HIT EMPTY/, 'no response body data in log'); -like($log, qr/"content_type":"application\/json"/, 'application/json remains valid'); +my @events; +my $valid_json = eval { + @events = map { decode_json($_) } grep { length $_ } split /\n/, $log; + 1; +}; +ok($valid_json, 'MIME test event is valid JSON'); +diag($@) unless $valid_json; +is(scalar @events, 1, 'only engine-inspected matching body causes an event'); +is($events[0]->{content_type}, 'application/json', 'engine-selected content type logged'); +is($events[0]->{actual_action}, 'connection_abort', 'JSON intervention abort logged'); +unlike($log, qr/HIT JSON|HIT PNG|HIT PLAIN/, 'no response body is copied to Phase4 log'); diff --git a/tests/modsecurity-phase4-invalid-config.t b/tests/modsecurity-phase4-invalid-config.t index cc695551..1c2c7969 100644 --- a/tests/modsecurity-phase4-invalid-config.t +++ b/tests/modsecurity-phase4-invalid-config.t @@ -1,38 +1,38 @@ #!/usr/bin/perl use warnings; use strict; -use Test::More tests => 1; +use Test::More; BEGIN { use FindBin; chdir($FindBin::Bin); } use lib 'lib'; use Test::Nginx; -my $t = Test::Nginx->new()->has(qw/http/); -$t->write_file('phase4-invalid.conf', "text/*\n"); -mkdir($t->testdir() . '/logs') unless -d $t->testdir() . '/logs'; -$t->write_file('logs/error.log', ''); +my $t = Test::Nginx->new()->has(qw/http/)->plan(3); +# Test::Nginx always reads this file in its two cleanup assertions, even +# when nginx -t exits during configuration parsing before opening its log. +$t->write_file('error.log', ''); +my $binary = $ENV{TEST_NGINX_BINARY} || ($t->testdir() . '/../nginx'); -$t->write_file_expand('nginx.conf', <<'EOF'); +for my $case ( + ['modsecurity_phase4_mode minimal;', qr/invalid value.*minimal|invalid.*phase4.*mode/, 'removed minimal mode is rejected'], + ['modsecurity_phase4_content_types_file removed.conf;', qr/unknown directive.*modsecurity_phase4_content_types_file/, 'removed connector MIME directive is rejected'], + ['modsecurity_phase4_body_limit 0;', qr/(?:must be|invalid|greater than).*0|invalid.*body.*limit|body.*limit.*(?:positive|zero)/, 'zero connector body limit is rejected'], +) { + my ($directive, $expected, $label) = @$case; + $t->write_file_expand('nginx.conf', <<"EOF"); %%TEST_GLOBALS%% - events {} - http { %%TEST_GLOBALS_HTTP%% - server { listen 127.0.0.1:19849; - server_name localhost; location / { modsecurity on; - modsecurity_phase4_content_types_file %%TESTDIR%%/phase4-invalid.conf; - return 200 "ok\n"; + $directive + return 200 "ok\\n"; } } } EOF - -my $cmd = $ENV{TEST_NGINX_BINARY} || ($t->testdir() . '/../nginx'); -$cmd .= " -p " . $t->testdir() . "/ -c nginx.conf -t 2>&1"; -my $out = `$cmd`; - -like($out, qr/invalid content-type entry in modsecurity_phase4_content_types_file/, - 'error points to invalid content-type entry'); + my $prefix = $t->testdir(); + my $out = `"$binary" -p "$prefix/" -c nginx.conf -e error.log -t 2>&1`; + like($out, $expected, $label); +} diff --git a/tests/modsecurity-phase4-modes.t b/tests/modsecurity-phase4-modes.t index 99c2988c..dfce5a8d 100644 --- a/tests/modsecurity-phase4-modes.t +++ b/tests/modsecurity-phase4-modes.t @@ -1,13 +1,14 @@ #!/usr/bin/perl use warnings; use strict; use Test::More; +use JSON::PP qw(decode_json); BEGIN { use FindBin; chdir($FindBin::Bin); } use lib 'lib'; use Test::Nginx; -my $t = Test::Nginx->new()->has(qw/http/); -$t->write_file('phase4-content-types.conf', "# comment\n\nApplication/JSON; charset=utf-8 # inline\ntext/html\n"); - +# Native off/default late-deny finalization intentionally causes the nginx +# header-already-sent alert. Audit those alerts explicitly below. +my $t = Test::Nginx->new()->has(qw/http/)->todo_alerts(); $t->write_file_expand('nginx.conf', <<'EOC'); %%TEST_GLOBALS%% daemon off; @@ -17,67 +18,61 @@ http { server { listen 127.0.0.1:8080; server_name localhost; - - location /m { - modsecurity on; - modsecurity_phase4_mode minimal; - modsecurity_phase4_log %%TESTDIR%%/phase4.log; - modsecurity_phase4_content_types_file %%TESTDIR%%/phase4-content-types.conf; - modsecurity_rules ' - SecRuleEngine On - SecResponseBodyAccess On - SecRule RESPONSE_BODY "@rx Hello" "id:910001,phase:4,deny,log,status:403,msg:\"x\\nq\"" - '; - } - - location /s { - modsecurity on; - modsecurity_phase4_mode safe; - modsecurity_phase4_log %%TESTDIR%%/phase4.log; - modsecurity_rules ' - SecRuleEngine On - SecResponseBodyAccess On - SecRule RESPONSE_BODY "@rx Hello" "id:910002,phase:4,deny,log,status:403" - '; - } - - location /x { - modsecurity on; - modsecurity_phase4_mode strict; - modsecurity_phase4_log %%TESTDIR%%/phase4.log; - modsecurity_rules ' - SecRuleEngine On - SecResponseBodyAccess On - SecRule RESPONSE_BODY "@rx Hello" "id:910003,phase:4,deny,log,status:403" - '; - } + default_type text/plain; + modsecurity on; + modsecurity_phase4_log %%TESTDIR%%/phase4.log; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyMimeType text/plain + SecRule RESPONSE_BODY "@rx Hello" "id:910001,phase:4,deny,log,status:403" + '; + location /off { modsecurity_phase4_mode off; } + location /default { } + location /safe { modsecurity_phase4_mode safe; } + location /strict { modsecurity_phase4_mode strict; } } } EOC -$t->write_file('/m', 'Hello minimal'); -$t->write_file('/s', 'Hello safe'); -$t->write_file('/x', 'Hello strict'); +for my $mode (qw/off default safe strict/) { + $t->write_file('/' . $mode, 'Hello ' . $mode); +} $t->run(); -$t->plan(11); +$t->plan(19); -like(http_get('/m'), qr/Hello minimal/, 'minimal no fake deny'); -like(http_get('/s'), qr/Hello safe/, 'safe no fake deny'); -is(http_get('/x'), '', 'strict abort after headers sent'); +is(http_get('/off'), '', 'off preserves native late-deny failure'); +is(http_get('/default'), '', 'off is the default'); +like(http_get('/safe'), qr/Hello safe/, 'safe continues after a late rule intervention'); +is(http_get('/strict'), '', 'strict aborts after headers are committed'); my $log = $t->read_file('phase4.log'); -like($log, qr/"actual_action":"log_only"/, 'log_only present'); -like($log, qr/"reason":"mode_safe"/, 'safe reason present'); -like($log, qr/"actual_action":"connection_abort"/, 'strict action logged'); -like($log, qr/"event":"phase4_intervention"/, 'event field present'); -like($log, qr/"header_sent":true/, 'json boolean header_sent'); -unlike($log, qr/Hello minimal|Hello safe|Hello strict/, 'no response body data in phase4 log'); +my @events; +my $valid_json = eval { + @events = map { decode_json($_) } grep { length $_ } split /\n/, $log; + 1; +}; +ok($valid_json, 'every Phase4 event is valid JSON'); +diag($@) unless $valid_json; +is(scalar @events, 2, 'off and default off emit no dedicated Phase4 event'); +my %event = map { $_->{uri} => $_ } @events; +is($event{'/safe'}->{actual_action}, 'log_only', 'safe action logged'); +is($event{'/safe'}->{reason}, 'response_committed_safe', 'safe reason logged'); +is($event{'/strict'}->{actual_action}, 'connection_abort', 'strict action logged'); +is($event{'/strict'}->{reason}, 'response_committed_strict', 'strict reason logged'); +is($event{'/safe'}->{mode}, 'safe', 'safe mode logged'); +is($event{'/strict'}->{mode}, 'strict', 'strict mode logged'); +ok(@events && !grep({ !$_->{header_sent} } @events), 'header_sent is true for late interventions'); +ok(@events && !grep({ $_->{event} ne 'phase4_intervention' } @events), 'event kind is stable'); +unlike($log, qr/Hello off|Hello default|Hello safe|Hello strict/, 'response body is absent from Phase4 log'); -my @lines = grep { length $_ } split /\n/, $log; -ok(@lines >= 1, 'phase4 log has one or more json lines'); -my @bad_lines = grep { $_ !~ /^\{.*\}\r?$/ } @lines; -note('phase4.log lines=' . scalar(@lines)); -if (@bad_lines) { - diag('non-json-lines: ' . join(' || ', map { my $x = $_; $x =~ s/\r/\\r/g; $x } @bad_lines)); -} -ok(scalar(@bad_lines) == 0, 'each log line is a single JSON object line'); +# Stop first so the explicit alert audit includes all worker output. +$t->stop(); +my @alerts = $t->read_file('error.log') =~ /^.*\[alert\].*$/gm; +my $expected_alert = qr/\[alert\].*header already sent\b.*request: "GET \/(?:off|default) HTTP\/1\.[01]"/; +my @late_alerts = grep { /$expected_alert/ } @alerts; +is(scalar @late_alerts, 2, 'native off/default each cause one expected late-finalization alert'); +is(scalar(grep { /request: "GET \/off HTTP\/1\.[01]"/ } @late_alerts), 1, 'explicit off has one expected alert'); +is(scalar(grep { /request: "GET \/default HTTP\/1\.[01]"/ } @late_alerts), 1, 'default off has one expected alert'); +my @unexpected_alerts = grep { !/$expected_alert/ } @alerts; +is(join("\n", @unexpected_alerts), '', 'no unexpected nginx alerts'); diff --git a/tests/modsecurity-phase4-regression.t b/tests/modsecurity-phase4-regression.t index e68c75a9..40875eb4 100644 --- a/tests/modsecurity-phase4-regression.t +++ b/tests/modsecurity-phase4-regression.t @@ -6,8 +6,7 @@ use lib 'lib'; use Test::Nginx; my $t = Test::Nginx->new()->has(qw/http/); -my $big = ('A' x 70000) . 'TAIL'; - +my $big = ('A' x 1100000) . 'TAIL'; $t->write_file_expand('nginx.conf', <<'EOC'); %%TEST_GLOBALS%% daemon off; @@ -17,30 +16,35 @@ http { server { listen 127.0.0.1:8080; server_name localhost; - - location /big { - modsecurity on; - modsecurity_phase4_mode minimal; - modsecurity_phase4_log %%TESTDIR%%/phase4-regression.log; - modsecurity_rules ' - SecRuleEngine On - SecResponseBodyAccess On - SecRule RESPONSE_BODY "@rx TAIL" "id:930001,phase:4,deny,log,status:403,msg:\"reg\"" - '; - } + default_type text/plain; + modsecurity on; + modsecurity_phase4_body_limit 64k; + modsecurity_phase4_log %%TESTDIR%%/phase4-regression.log; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyMimeType text/plain + SecResponseBodyLimit 2097152 + SecRule RESPONSE_BODY "@endsWith TAIL" "id:930001,phase:4,pass,log,msg:phase4-tail-seen" + '; + location /off { modsecurity_phase4_mode off; } + location /safe { modsecurity_phase4_mode safe; } + location /strict { modsecurity_phase4_mode strict; } } } EOC -$t->write_file('/big', $big); +for my $mode (qw/off safe strict/) { + $t->write_file('/' . $mode, $big); +} $t->run(); -$t->plan(5); - -my $resp = http_get('/big'); -like($resp, qr/HTTP\/1\.1 200 OK/, 'big response status remains 200 in minimal mode'); -like($resp, qr/A{1024}/, 'big response body contains expected prefix chunk'); -like($resp, qr/TAIL/, 'big response body tail present (no truncation)'); +$t->plan(6); -my $log = $t->read_file('phase4-regression.log'); -like($log, qr/"actual_action":"log_only"/, 'minimal mode logs only'); -unlike($log, qr/A{100,}|TAIL/, 'no large response data leaked to phase4 log'); +my $resp = http_get('/off'); +like($resp, qr/HTTP\/1\.1 200 OK/, 'off ignores optional connector body budget'); +my ($headers, $body) = split /\x0d\x0a\x0d\x0a/, $resp, 2; +is($body, $big, 'off forwards the complete large response without truncation'); +like($t->read_file('error.log'), qr/phase4-tail-seen/, 'off still processes the response body through the engine'); +unlike(http_get('/safe'), qr/TAIL/, 'safe rejects a response exceeding the connector budget'); +unlike(http_get('/strict'), qr/TAIL/, 'strict rejects a response exceeding the connector budget'); +unlike($t->read_file('phase4-regression.log'), qr/A{100,}|TAIL/, 'response bytes are absent from event logs'); diff --git a/tests/modsecurity-phase4-subrequest.t b/tests/modsecurity-phase4-subrequest.t new file mode 100644 index 00000000..347fa1e2 --- /dev/null +++ b/tests/modsecurity-phase4-subrequest.t @@ -0,0 +1,83 @@ +#!/usr/bin/perl +use warnings; use strict; +use Test::More; +use JSON::PP qw(decode_json); +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +# auth_request shares the main request's pool but owns a separate response. +# Its header-only image response must not consume the main transaction's +# response headers or MIME selection. Direct subrequest EOS is covered by +# the compiled owner-isolation regression; auth_request itself is header-only. +my $t = Test::Nginx->new()->has(qw/http auth_request/); +$t->write_file_expand('nginx.conf', <<'EOC'); +%%TEST_GLOBALS%% +daemon off; +events {} +http { + %%TEST_GLOBALS_HTTP%% + log_format auth_probe '$uri $status'; + server { + listen 127.0.0.1:8080; + server_name localhost; + default_type text/plain; + log_subrequest on; + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log %%TESTDIR%%/phase4-subrequest.log; + modsecurity_rules 'SecResponseBodyMimeTypesClear'; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyMimeType text/plain + SecRule RESPONSE_HEADERS:Content-Type "@contains image/png" "id:940002,phase:3,deny,log,status:403,msg:auth-response-headers-leaked" + SecRule RESPONSE_HEADERS:Content-Type "@contains text/plain" "id:940003,phase:3,pass,log,msg:main-response-headers-seen" + SecRule RESPONSE_BODY "@contains P4-MAIN-MARKER" "id:940001,phase:4,deny,log,status:403" + '; + location = /_auth { + internal; + modsecurity off; + default_type image/png; + access_log %%TESTDIR%%/auth.log auth_probe; + } + location = /control { auth_request /_auth; } + location = /main { auth_request /_auth; } + } +} +EOC + +$t->write_file('/_auth', 'BENIGN AUTH BODY'); +$t->write_file('/control', 'BENIGN MAIN BODY'); +$t->write_file('/main', 'P4-MAIN-MARKER'); +$t->run(); +$t->plan(16); + +my $control = http_get('/control'); +like($control, qr/^HTTP\/1\.1 200 OK/, 'benign main response passes after authorization'); +like($control, qr/BENIGN MAIN BODY/, 'main body remains intact'); +unlike($control, qr/BENIGN AUTH BODY/, 'authorization body is absent from main response'); +is(http_get('/main'), '', 'main Phase4 marker still triggers strict abort after authorization'); +$t->stop(); +my @auth_results = $t->read_file('auth.log') =~ /^\/_auth 200\r?$/gm; +is(scalar @auth_results, 2, 'both authorization subrequests completed successfully'); +my $error_log = $t->read_file('error.log'); +like($error_log, qr/main-response-headers-seen/, 'main response headers are inspected after the subrequest'); +unlike($error_log, qr/auth-response-headers-leaked/, 'auth response headers never enter the main transaction'); + +my $log = $t->read_file('phase4-subrequest.log'); +my @events; +my $valid_json = eval { + @events = map { decode_json($_) } grep { length $_ } split /\n/, $log; + 1; +}; +ok($valid_json, 'subrequest regression event is valid JSON'); +diag($@) unless $valid_json; +is(scalar @events, 1, 'only the matching main response produces a Phase4 event'); +is($events[0]->{uri}, '/main', 'event belongs to the main request'); +is($events[0]->{mode}, 'strict', 'main strict mode is retained'); +is($events[0]->{actual_action}, 'connection_abort', 'main intervention aborts the connection'); +ok($events[0]->{header_sent}, 'main response headers were committed'); +is($events[0]->{waf_status}, 403, 'main rule decision is retained'); +is($events[0]->{rule_id}, '940001', 'main body rule caused the event'); +unlike($log, qr/P4-MAIN-MARKER|BENIGN AUTH BODY|BENIGN MAIN BODY/, 'event log excludes both response bodies'); diff --git a/tests/test_phase4_runtime.py b/tests/test_phase4_runtime.py new file mode 100644 index 00000000..55691f01 --- /dev/null +++ b/tests/test_phase4_runtime.py @@ -0,0 +1,560 @@ +"""Compile and exercise the production Phase4 helpers with small host doubles. + +These tests cover accounting, buffer representations, EOS, and intervention +dispatch. Native NGINX/libModSecurity HTTP tests remain in modsecurity*.t. +There is no dependency on the Common connector or its runtime framework. +""" + +from __future__ import annotations + +import os +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +BODY = ROOT / "src" / "ngx_http_modsecurity_body_filter.c" +MODULE = ROOT / "src" / "ngx_http_modsecurity_module.c" + + +def matching_delimiter(source: str, opening: int, left: str, right: str) -> int: + depth = 0 + for index in range(opening, len(source)): + if source[index] == left: + depth += 1 + elif source[index] == right: + depth -= 1 + if depth == 0: + return index + raise AssertionError(f"unterminated {left}{right} pair") + + +def function_definition(source: str, name: str) -> str: + for match in re.finditer(rf"\b{re.escape(name)}\s*\(", source): + opening = source.index("(", match.start()) + closing = matching_delimiter(source, opening, "(", ")") + cursor = closing + 1 + while cursor < len(source) and source[cursor].isspace(): + cursor += 1 + if cursor >= len(source) or source[cursor] != "{": + continue + end = matching_delimiter(source, cursor, "{", "}") + start = source.rfind("\n", 0, match.start()) + 1 + return source[start:end + 1] + raise AssertionError(f"{name} definition was not found") + + +PREAMBLE = r""" +#include +#include +#include +#include +#include +#include +#ifdef _MSC_VER +typedef intptr_t ssize_t; +#endif +typedef unsigned char u_char; +typedef int ngx_int_t; +typedef struct ngx_http_request_s ngx_http_request_t; +typedef struct ngx_pool_cleanup_s { + void (*handler)(void *); + void *data; + struct ngx_pool_cleanup_s *next; +} ngx_pool_cleanup_t; +typedef struct { ngx_pool_cleanup_t *cleanup; } ngx_pool_t; +typedef struct { int error; void *log; } ngx_connection_t; +typedef struct { u_char *data; size_t length; } ngx_file_t; +typedef struct { + u_char *pos, *last; + off_t file_pos, file_last; + ngx_file_t *file; + unsigned memory, in_file, last_buf, last_in_chain; +} ngx_buf_t; +typedef struct ngx_chain_s { + ngx_buf_t *buf; + struct ngx_chain_s *next; +} ngx_chain_t; +typedef struct { + size_t response_body_bytes_seen, response_body_bytes_inspected; + int response_body_seen, response_body_truncated, phase4_processed; + int phase4_strict_abort, phase4_headers_checked, intervention_triggered; + int intervention_late, intervention_failed, response_replaced, last_intervention_status; + ngx_http_request_t *r; + void *modsec_transaction; + u_char *phase4_file_scratch; +} ngx_http_modsecurity_ctx_t; +typedef struct { + unsigned phase4_mode; + size_t phase4_body_limit; +} ngx_http_modsecurity_conf_t; +struct ngx_http_request_s { + int header_sent; + struct ngx_http_request_s *main; + ngx_pool_t *pool; + ngx_connection_t *connection; + ngx_http_modsecurity_ctx_t *ctx; + ngx_http_modsecurity_conf_t *conf; +}; +#define NGX_OK 0 +#define NGX_ERROR (-1) +#define NGX_AGAIN (-2) +#define NGX_HTTP_INTERNAL_SERVER_ERROR 500 +#define NGX_HTTP_FORBIDDEN 403 +#define NGX_HTTP_MODSEC_PHASE4_MODE_OFF 0 +#define NGX_HTTP_MODSEC_PHASE4_MODE_SAFE 1 +#define NGX_HTTP_MODSEC_PHASE4_MODE_STRICT 2 +#define NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK 32768U +#define ngx_buf_in_memory(buffer) ((buffer)->memory) +#define ngx_http_get_module_ctx(r, module) ((r)->ctx) +#define ngx_http_get_module_loc_conf(r, module) ((r)->conf) +#define ngx_log_error(...) ((void)0) +static int ngx_http_modsecurity_module; +static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r); +static void ngx_http_modsecurity_cleanup(void *data) { (void)data; } +static int allocation_calls, read_calls, append_calls, process_calls; +static int fail_allocation, fail_read, short_read, fail_append, append_result = 1; +static int engine_result = 1, native_result, native_late, native_failed; +static int log_result, event_calls, pcre_enter, pcre_leave; +static int forward_calls, forwarded_links, forward_result; +static size_t read_max, appended_bytes; +static u_char scratch[32768], inspected[200000]; +static const char *last_action, *last_reason; + +static void *ngx_pnalloc(ngx_pool_t *pool, size_t bytes) +{ + (void)pool; + ++allocation_calls; + return !fail_allocation && bytes <= sizeof(scratch) ? scratch : NULL; +} +static ssize_t ngx_read_file(ngx_file_t *file, u_char *out, + size_t bytes, off_t offset) +{ + ++read_calls; + if (bytes > read_max) read_max = bytes; + if (fail_read || offset < 0) return -1; + if ((size_t)offset > file->length || bytes > file->length - (size_t)offset) + return 0; + if (short_read && bytes != 0U) --bytes; + memcpy(out, file->data + offset, bytes); + return (ssize_t)bytes; +} +static int msc_append_response_body(void *transaction, u_char *data, size_t bytes) +{ + (void)transaction; + ++append_calls; + if (fail_append || bytes > sizeof(inspected) - appended_bytes) return -1; + memcpy(inspected + appended_bytes, data, bytes); + appended_bytes += bytes; + return append_result; +} +static int msc_process_response_body(void *transaction) +{ + (void)transaction; + ++process_calls; + return engine_result; +} +static ngx_pool_t *ngx_http_modsecurity_pcre_malloc_init(ngx_pool_t *pool) +{ + ++pcre_enter; + return pool; +} +static void ngx_http_modsecurity_pcre_malloc_done(ngx_pool_t *pool) +{ + (void)pool; + ++pcre_leave; +} +static int ngx_http_modsecurity_process_intervention(void *transaction, + ngx_http_request_t *r, int early) +{ + ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); + (void)transaction; (void)early; + if (ctx == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; + ctx->intervention_late = native_late; + ctx->intervention_failed = native_failed; + return native_result; +} +static ngx_int_t ngx_http_filter_finalize_request(ngx_http_request_t *r, + int *module, int status) +{ + (void)module; + return r->header_sent ? NGX_ERROR : status; +} +static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, + ngx_http_modsecurity_conf_t *conf, const char *wanted, + const char *actual, const char *reason) +{ + (void)r; (void)conf; (void)wanted; + ++event_calls; + last_action = actual; last_reason = reason; + return log_result; +} +static ngx_int_t ngx_http_next_body_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + (void)r; + ++forward_calls; + for (; in != NULL; in = in->next) ++forwarded_links; + return forward_result; +} +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "line %d: %s\n", __LINE__, #condition); return 1; \ +} } while (0) +""" + +CASES = r""" +int main(int argc, char **argv) +{ + ngx_http_modsecurity_ctx_t ctx = {0}; + ngx_http_modsecurity_conf_t conf = {NGX_HTTP_MODSEC_PHASE4_MODE_SAFE, 1048576U}; + ngx_pool_t pool = {0}; + ngx_connection_t connection = {0}; + ngx_http_request_t request = {0}; + ngx_http_request_t main_request = {0}; + u_char payload[70000]; + ngx_file_t file = {payload, sizeof(payload)}; + ngx_buf_t buffer = {0}; + ngx_buf_t final = {0}; + ngx_chain_t tail = {&final, NULL}; + ngx_chain_t chain = {&buffer, &tail}; + size_t allowed = 999U; + size_t i; + CHECK(argc == 2); + memset(payload, 'A', sizeof(payload)); + request.pool = &pool; request.connection = &connection; + request.ctx = &ctx; request.conf = &conf; request.main = &request; ctx.r = &request; + buffer.file = &file; buffer.in_file = 1; buffer.file_last = sizeof(payload); + + if (strcmp(argv[1], "request-ownership") == 0) { + ngx_http_modsecurity_ctx_t other_ctx = {0}; + ngx_http_request_t subrequest = {0}; + ngx_pool_cleanup_t owner_cleanup = {ngx_http_modsecurity_cleanup, &ctx, NULL}; + ngx_pool_cleanup_t other_cleanup = {ngx_http_modsecurity_cleanup, &other_ctx, &owner_cleanup}; + other_ctx.r = &subrequest; + subrequest.pool = &pool; subrequest.main = &request; + pool.cleanup = &other_cleanup; + CHECK(ngx_http_modsecurity_get_module_ctx(&request) == &ctx); + request.ctx = NULL; /* Same-request internal redirect clears module context. */ + CHECK(ngx_http_modsecurity_get_module_ctx(&request) == &ctx); + CHECK(ngx_http_modsecurity_get_module_ctx(&subrequest) == &other_ctx); + subrequest.ctx = &ctx; /* Reject a foreign direct context too. */ + CHECK(ngx_http_modsecurity_get_module_ctx(&subrequest) == &other_ctx); + pool.cleanup = &owner_cleanup; /* Auth subrequest has no own transaction. */ + CHECK(ngx_http_modsecurity_get_module_ctx(&subrequest) == NULL); + subrequest.ctx = NULL; subrequest.connection = &connection; + final.last_in_chain = 1; + CHECK(ngx_http_modsecurity_body_filter(&subrequest, &tail) == NGX_OK); + CHECK(process_calls == 0 && !ctx.phase4_processed && forward_calls == 1); + final.last_in_chain = 0; final.last_buf = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &tail) == NGX_OK); + CHECK(process_calls == 1 && ctx.phase4_processed && forward_calls == 2); + } else if (strcmp(argv[1], "off-large") == 0) { + conf.phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_OFF; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1048577U, &allowed) == NGX_OK); + CHECK(allowed == 1048577U && ctx.response_body_bytes_seen == allowed); + CHECK(ctx.response_body_seen && !ctx.response_body_truncated); + CHECK(ctx.response_body_bytes_inspected == 0U); + } else if (strcmp(argv[1], "off-multiple") == 0) { + conf.phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_OFF; + for (i = 0; i < 3; ++i) { + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1048576U, &allowed) == NGX_OK); + CHECK(allowed == 1048576U); + ctx.response_body_bytes_inspected += allowed; + } + CHECK(ctx.response_body_bytes_seen == 3U * 1048576U); + CHECK(!ctx.response_body_truncated); + } else if (strcmp(argv[1], "budget-boundary") == 0) { + for (i = 1; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1048575U, &allowed) == NGX_OK); + ctx.response_body_bytes_inspected += allowed; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1U, &allowed) == NGX_OK); + ctx.response_body_bytes_inspected += allowed; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1U, &allowed) == NGX_ERROR); + CHECK(allowed == 0U && ctx.response_body_truncated); + CHECK(ctx.response_body_bytes_seen == 1048577U); + CHECK(ctx.response_body_bytes_inspected == 1048576U); + } + } else if (strcmp(argv[1], "oversized-first") == 0) { + for (i = 1; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1048577U, &allowed) == NGX_ERROR); + CHECK(allowed == 0U && ctx.response_body_truncated); + CHECK(ctx.response_body_bytes_inspected == 0U); + } + } else if (strcmp(argv[1], "overflow") == 0) { + conf.phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_OFF; + ctx.response_body_bytes_seen = SIZE_MAX - 1U; + ctx.response_body_bytes_inspected = SIZE_MAX - 1U; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 2U, &allowed) == NGX_ERROR); + CHECK(allowed == 0U && ctx.response_body_truncated); + CHECK(ctx.response_body_bytes_seen == SIZE_MAX); + CHECK(ctx.response_body_bytes_inspected == SIZE_MAX - 1U); + } else if (strcmp(argv[1], "invalid-accounting") == 0) { + ctx.response_body_bytes_inspected = 1U; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1U, &allowed) == NGX_ERROR); + CHECK(allowed == 0U && ctx.response_body_truncated); + CHECK(ctx.response_body_bytes_seen == 0U); + } else if (strcmp(argv[1], "null-empty-mode") == 0) { + CHECK(ngx_http_modsecurity_plan_limited_response_body(NULL, &conf, + 1U, &allowed) == NGX_ERROR && allowed == 0U); + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, NULL, + 1U, &allowed) == NGX_ERROR && allowed == 0U); + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1U, NULL) == NGX_ERROR); + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 0U, &allowed) == NGX_OK && allowed == 0U); + CHECK(!ctx.response_body_seen); + conf.phase4_mode = 99U; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1U, &allowed) == NGX_ERROR && allowed == 0U); + } else if (strcmp(argv[1], "file-chunks") == 0) { + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(read_calls == 3 && append_calls == 3 && read_max == 32768U); + CHECK(allocation_calls == 1 && appended_bytes == sizeof(payload)); + CHECK(memcmp(inspected, payload, sizeof(payload)) == 0); + CHECK(buffer.file_pos == 0 && buffer.file_last == sizeof(payload)); + buffer.file_last = 1; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(allocation_calls == 1 && appended_bytes == sizeof(payload) + 1U); + } else if (strcmp(argv[1], "mixed-once") == 0) { + buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 15; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(appended_bytes == 15U && append_calls == 1 && read_calls == 0); + CHECK(ctx.response_body_bytes_seen == 15U); + CHECK(buffer.pos == payload && buffer.last == payload + 15); + } else if (strcmp(argv[1], "file-limit") == 0) { + conf.phase4_body_limit = sizeof(payload) - 1U; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(read_calls == 0 && allocation_calls == 0 && append_calls == 0); + CHECK(ctx.response_body_truncated); + } else if (strcmp(argv[1], "file-invalid") == 0) { + buffer.file_pos = -1; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + buffer.file_pos = 10; buffer.file_last = 9; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + buffer.file_pos = 0; buffer.file_last = 10; buffer.file = NULL; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(read_calls == 0 && append_calls == 0); + } else if (strcmp(argv[1], "allocation-failure") == 0) { + fail_allocation = 1; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(read_calls == 0 && appended_bytes == 0U); + } else if (strcmp(argv[1], "short-read") == 0 || strcmp(argv[1], "read-error") == 0) { + short_read = strcmp(argv[1], "short-read") == 0; + fail_read = !short_read; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(read_calls == 1 && append_calls == 0 && appended_bytes == 0U); + } else if (strcmp(argv[1], "append-error") == 0) { + fail_append = 1; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(ctx.response_body_bytes_inspected == 0U); + } else if (strcmp(argv[1], "append-zero") == 0) { + append_result = 0; + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(ctx.response_body_bytes_inspected == 10U && append_calls == 1); + } else if (strcmp(argv[1], "final-once") == 0) { + CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == NGX_OK); + CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == NGX_OK); + CHECK(process_calls == 1 && pcre_enter == 1 && pcre_leave == 1); + CHECK(ctx.phase4_processed); + } else if (strcmp(argv[1], "engine-error") == 0) { + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + request.header_sent = 1; connection.error = 0; engine_result = -1; + CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == NGX_ERROR); + CHECK(ctx.phase4_processed && ctx.intervention_triggered && connection.error); + } + CHECK(process_calls == 3 && pcre_enter == pcre_leave); + } else if (strcmp(argv[1], "process-zero") == 0) { + engine_result = 0; + CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == 500); + CHECK(ctx.intervention_triggered && ctx.phase4_processed); + CHECK(process_calls == 1 && pcre_enter == pcre_leave); + } else if (strcmp(argv[1], "main-eos") == 0 || strcmp(argv[1], "subrequest-eos") == 0) { + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; buffer.last_in_chain = 1; + chain.next = NULL; + if (strcmp(argv[1], "subrequest-eos") == 0) request.main = &main_request; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(appended_bytes == 10U && process_calls == (request.main != &request)); + chain.buf = &final; final.last_buf = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(process_calls == 1 && appended_bytes == 10U); + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(process_calls == 1 && forward_calls == 3); + } else if (strcmp(argv[1], "chain-restored") == 0) { + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; final.last_buf = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(chain.next == &tail && tail.next == NULL); + CHECK(forward_calls == 1 && forwarded_links == 2 && process_calls == 1); + CHECK(appended_bytes == 10U && buffer.pos == payload); + } else if (strcmp(argv[1], "forward-again") == 0) { + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; final.last_buf = 1; + forward_result = NGX_AGAIN; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_AGAIN); + CHECK(chain.next == &tail && forwarded_links == 2 && process_calls == 1); + CHECK(buffer.pos == payload && appended_bytes == 10U); + forward_result = NGX_OK; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(forwarded_links == 4 && appended_bytes == 10U && process_calls == 1); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_OK); + CHECK(process_calls == 1 && chain.next == &tail); + } else if (strcmp(argv[1], "terminal-trailing") == 0) { + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; buffer.last_buf = 1; + final.memory = 1; final.pos = payload + 10; final.last = payload + 20; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(process_calls == 1 && appended_bytes == 10U); + CHECK(chain.next == &tail && forwarded_links == 2); + } else if (strcmp(argv[1], "replaced-body") == 0) { + ctx.response_replaced = 1; + buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 10; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(buffer.pos == buffer.last && !buffer.in_file); + CHECK(buffer.file_last == buffer.file_pos); + CHECK(append_calls == 0 && process_calls == 0 && forwarded_links == 2); + } else if (strcmp(argv[1], "late-safe") == 0 || strcmp(argv[1], "late-strict") == 0 || strcmp(argv[1], "late-off") == 0) { + request.header_sent = 1; native_result = -1; native_late = 1; + ctx.last_intervention_status = 403; + if (strcmp(argv[1], "late-off") == 0) conf.phase4_mode = 0; + if (strcmp(argv[1], "late-strict") == 0) conf.phase4_mode = 2; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == + (conf.phase4_mode == 1 ? NGX_OK : NGX_ERROR)); + CHECK(event_calls == (conf.phase4_mode != 0)); + CHECK(ctx.intervention_triggered == (conf.phase4_mode != 1)); + if (conf.phase4_mode == 1) CHECK(strcmp(last_action, "log_only") == 0); + if (conf.phase4_mode == 2) { + CHECK(strcmp(last_action, "connection_abort") == 0); + CHECK(ctx.phase4_strict_abort && connection.error); + } + } else if (strcmp(argv[1], "native-failure-safe") == 0) { + request.header_sent = 1; native_result = 500; native_failed = 1; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == NGX_ERROR); + CHECK(connection.error && ctx.intervention_triggered && event_calls == 0); + } else if (strcmp(argv[1], "native-failure-uncommitted") == 0) { + native_result = 500; native_failed = 1; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == 500); + CHECK(ctx.intervention_triggered && event_calls == 0); + } else if (strcmp(argv[1], "uncommitted-deny") == 0) { + const ngx_int_t statuses[] = {403, 302}; + for (size_t i = 0; i < sizeof(statuses) / sizeof(statuses[0]); i++) { + ctx.phase4_headers_checked = 0; ctx.intervention_triggered = 0; + event_calls = 0; native_result = statuses[i]; + ctx.last_intervention_status = statuses[i]; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == statuses[i]); + CHECK(ctx.intervention_triggered && event_calls == 1); + CHECK(strcmp(last_action, "deny_status") == 0); + CHECK(strcmp(last_reason, "response_not_committed") == 0); + } + } else if (strcmp(argv[1], "log-failure-safe") == 0) { + request.header_sent = 1; native_result = -1; native_late = 1; log_result = NGX_ERROR; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == NGX_ERROR); + CHECK(ctx.intervention_triggered && connection.error); + } else { + return 2; + } + return 0; +} +""" + + +def unit_program(body: str, module: str) -> str: + functions = ( + "response_body_failure", "plan_limited_response_body", + "append_response_body_chunk", "append_limited_response_body", + "append_file_response_body", "append_response_body_buffer", + "phase4_handle_intervention", "process_response_intervention", + "process_final_response_body", + ) + production = "".join( + "\nstatic ngx_int_t\n" + function_definition(body, "ngx_http_modsecurity_" + name) + + "\n" for name in functions + ) + production += "\nstatic void\n" + function_definition( + body, "ngx_http_modsecurity_discard_replaced_response_body" + ) + production += "\nngx_int_t\n" + function_definition( + body, "ngx_http_modsecurity_body_filter" + ) + context_lookup = "\nstatic ngx_http_modsecurity_ctx_t *\n" + function_definition( + module, "ngx_http_modsecurity_get_module_ctx" + ) + "\n" + return PREAMBLE + context_lookup + production + CASES + + +class Phase4RuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler or shutil.which(compiler[0]) is None: + raise RuntimeError("set CC to a GCC- or Clang-compatible C17 compiler") + directory = tempfile.TemporaryDirectory(prefix="nginx-phase4-") + cls.addClassCleanup(directory.cleanup) + source = Path(directory.name) / "phase4.c" + cls.binary = Path(directory.name) / ("phase4.exe" if os.name == "nt" else "phase4") + source.write_text(unit_program(BODY.read_text(encoding="utf-8"), MODULE.read_text(encoding="utf-8")), encoding="utf-8") + build = subprocess.run( + [*compiler, "-std=c17", "-Wall", "-Wextra", "-Werror", + str(source), "-o", str(cls.binary)], + capture_output=True, text=True, timeout=60, check=False, + ) + if build.returncode: + raise AssertionError("Phase4 helper compilation failed:\n" + build.stderr) + + def run_case(self, name: str) -> None: + result = subprocess.run( + [str(self.binary), name], capture_output=True, text=True, + timeout=10, check=False, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +def case_test(name: str): + def test(self: Phase4RuntimeTests) -> None: + self.run_case(name) + return test + + +for case in ( + "request-ownership", "off-large", "off-multiple", "budget-boundary", "oversized-first", + "overflow", "invalid-accounting", "null-empty-mode", "file-chunks", + "mixed-once", "file-limit", "file-invalid", "allocation-failure", + "short-read", "read-error", "append-error", "append-zero", "final-once", + "engine-error", "process-zero", + "main-eos", "subrequest-eos", "chain-restored", "forward-again", + "terminal-trailing", "replaced-body", + "late-safe", "late-strict", "late-off", "native-failure-safe", + "native-failure-uncommitted", "uncommitted-deny", "log-failure-safe", +): + setattr(Phase4RuntimeTests, "test_" + case.replace("-", "_"), case_test(case)) + + +if __name__ == "__main__": + unittest.main() From addf2cb130adcecedb71e4c2e5fadbe3527bbb9b Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:54:11 +0200 Subject: [PATCH 24/40] Delegate phase-4 response inspection limits to libModSecurity Sync the relevant NGINX changes from ModSecurity-conector at 820b6975495bdf0f90aca67eee86e27a3b7d329b. Keep modsecurity_phase4_body_limit as an ignored compatibility setting in off, safe, and strict while retaining checked accounting and bounded file reads. Update EN/DE guidance and add native Reject/ProcessPartial regressions. --- README.md | 29 ++-- docs/examples/phase4-engine-mime.conf | 5 +- docs/examples/phase4-off.conf | 9 +- docs/examples/phase4-safe.conf | 10 +- docs/examples/phase4-strict.conf | 8 +- docs/phase4-handling.de.md | 92 +++++++----- docs/phase4-handling.en.md | 91 +++++++----- src/ngx_http_modsecurity_body_filter.c | 24 +-- src/ngx_http_modsecurity_common.h | 1 + src/ngx_http_modsecurity_module.c | 1 + tests/README.md | 22 +-- tests/modsecurity-phase4-engine-limits.t | 123 +++++++++++++++ tests/modsecurity-phase4-regression.t | 22 +-- tests/test_phase4_runtime.py | 181 +++++++++++++++-------- 14 files changed, 436 insertions(+), 182 deletions(-) create mode 100644 tests/modsecurity-phase4-engine-limits.t diff --git a/README.md b/README.md index ea3fefce..d644a062 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ are sent. Before headers are sent, both modes can apply the intervention's HTTP status. Neither mode guarantees a clean 403 after headers have been sent. `off` uses the native intervention path and disables the connector's additional -phase 4 policy and body budget. It does not disable ModSecurity response-body +phase 4 policy. It does not disable ModSecurity response-body inspection; late native interventions can still fail the response. Configure inspection with `SecResponseBodyAccess`, `SecResponseBodyMimeType`, and `SecResponseBodyMimeTypesClear` in ModSecurity rules. @@ -203,12 +203,14 @@ modsecurity_phase4_body_limit **default:** *1m (1 MiB)* -Sets a positive cumulative response-body budget for `safe` and `strict`, in -bytes or nginx size notation such as `256k` or `2m`. The connector counts bytes -seen, including file-backed buffers, and rejects a chunk that would exceed the -budget before forwarding that chunk. This is independent of the engine's -response-body limit and does not introduce global response buffering. -`off` ignores this budget but retains overflow checks. +Deprecated compatibility setting. Positive values in bytes or nginx size +notation such as `256k` or `2m` remain accepted and inherited; zero remains +invalid. The historical default remains `1m`, but the value is ignored in +`off`, `safe`, and `strict` and does not impose a response-inspection limit. + +Configure the engine's inspection policy with `SecResponseBodyLimit` and +`SecResponseBodyLimitAction`. The connector retains checked byte accounting, +overflow guards, and bounded file reads, without global response buffering. modsecurity_phase4_log ---------------------- @@ -220,15 +222,20 @@ Sets the file used for phase 4 JSON event logging. ### Phase 4 migration -This branch adopts the phase 4 behavior from +The initial phase 4 migration adopted behavior from [ModSecurity-conector at b0f3bdab429717b5b0311c30c5b4d1153c672ac0](https://github.com/Easton97-Jens/ModSecurity-conector/tree/b0f3bdab429717b5b0311c30c5b4d1153c672ac0/connectors/nginx/src), adapted for this standalone nginx module. It does not require that repository's -multi-connector runtime. +multi-connector runtime. Response-inspection limit ownership now follows +[ModSecurity-conector at 820b6975495bdf0f90aca67eee86e27a3b7d329b](https://github.com/Easton97-Jens/ModSecurity-conector/blob/820b6975495bdf0f90aca67eee86e27a3b7d329b/docs/phase4-mode-budget.md): +libModSecurity owns the inspection limits in every valid phase 4 mode. -This changes configuration compatibility: `minimal` is rejected, the default -mode changes from `safe` to `off`, and +The earlier migration changed configuration compatibility: `minimal` is rejected, +the default changed from `safe` to `off`, and `modsecurity_phase4_content_types_file` is removed. Set `safe` or `strict` explicitly when migrating, and move MIME selection into ModSecurity rules. +The latest limit-ownership update retains those modes and their late policies. +Existing `modsecurity_phase4_body_limit` values no longer limit inspection; +migrate that policy to `SecResponseBodyLimit` and `SecResponseBodyLimitAction`. See [English](docs/phase4-handling.en.md) or [Deutsch](docs/phase4-handling.de.md) for the behavior, limits, logging schema, and examples. The documentation describes the source implementation; test diff --git a/docs/examples/phase4-engine-mime.conf b/docs/examples/phase4-engine-mime.conf index 1597e41e..2b8f7461 100644 --- a/docs/examples/phase4-engine-mime.conf +++ b/docs/examples/phase4-engine-mime.conf @@ -1,4 +1,4 @@ -# ModSecurity rules file: engine response-body MIME additions +# ModSecurity rules file: engine response-body MIME additions and limit policy # Load with modsecurity_rules_file alongside your other rules. # This is not an nginx include and not a connector MIME allowlist. # To replace the engine MIME list, load a separate reset BEFORE this file: @@ -8,4 +8,7 @@ # Adapt this list to the response types your application needs to inspect. SecResponseBodyAccess On +# Example engine inspection limit; independent of modsecurity_phase4_mode. +SecResponseBodyLimit 1048576 +SecResponseBodyLimitAction ProcessPartial SecResponseBodyMimeType text/html text/plain application/json application/problem+json application/xml application/xhtml+xml diff --git a/docs/examples/phase4-off.conf b/docs/examples/phase4-off.conf index 81d2f699..2a803a38 100644 --- a/docs/examples/phase4-off.conf +++ b/docs/examples/phase4-off.conf @@ -1,7 +1,7 @@ # nginx example: phase-4 mode "off" (the default) # Uses native intervention handling. This does not disable ModSecurity or # response-body inspection; late native interventions can fail the response. -# The connector's additional policy and body budget are disabled. +# The connector's additional intervention policy is disabled. http { upstream app_backend { @@ -16,12 +16,15 @@ http { modsecurity_phase4_mode off; # Illustrative rules; adapt to your application before deployment. - # MIME selection is configured in the engine, not in the connector. + # MIME selection and inspection limits belong to the engine. + # The legacy modsecurity_phase4_body_limit setting is ignored. # Reset the MIME list in a separate engine rule load. modsecurity_rules 'SecResponseBodyMimeTypesClear'; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On + SecResponseBodyLimit 1048576 + SecResponseBodyLimitAction ProcessPartial SecResponseBodyMimeType text/html text/plain application/json SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940101,phase:4,deny,log,status:403" '; @@ -30,4 +33,4 @@ http { proxy_pass http://app_backend; } } -} \ No newline at end of file +} diff --git a/docs/examples/phase4-safe.conf b/docs/examples/phase4-safe.conf index c07bf09b..a6eb94ac 100644 --- a/docs/examples/phase4-safe.conf +++ b/docs/examples/phase4-safe.conf @@ -1,5 +1,5 @@ # nginx example: phase-4 mode "safe" (must be selected explicitly) -# Late interventions are logged as log_only. Engine, I/O, and body-budget +# Late interventions are logged as log_only. Genuine engine, I/O, and overflow # failures still stop the affected response. http { @@ -13,16 +13,18 @@ http { modsecurity on; modsecurity_phase4_mode safe; - modsecurity_phase4_body_limit 1m; modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; # Illustrative rules; adapt to your application before deployment. - # MIME selection is configured in the engine, not in the connector. + # MIME selection and inspection limits belong to the engine. + # The legacy modsecurity_phase4_body_limit setting is ignored. # Reset the MIME list in a separate engine rule load. modsecurity_rules 'SecResponseBodyMimeTypesClear'; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On + SecResponseBodyLimit 1048576 + SecResponseBodyLimitAction ProcessPartial SecResponseBodyMimeType text/html text/plain application/json SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940102,phase:4,deny,log,status:403" '; @@ -31,4 +33,4 @@ http { proxy_pass http://app_backend; } } -} \ No newline at end of file +} diff --git a/docs/examples/phase4-strict.conf b/docs/examples/phase4-strict.conf index fba1d726..d78598ac 100644 --- a/docs/examples/phase4-strict.conf +++ b/docs/examples/phase4-strict.conf @@ -13,16 +13,18 @@ http { modsecurity on; modsecurity_phase4_mode strict; - modsecurity_phase4_body_limit 1m; modsecurity_phase4_log /var/log/nginx/modsecurity-phase4.log; # Illustrative rules; adapt to your application before deployment. - # MIME selection is configured in the engine, not in the connector. + # MIME selection and inspection limits belong to the engine. + # The legacy modsecurity_phase4_body_limit setting is ignored. # Reset the MIME list in a separate engine rule load. modsecurity_rules 'SecResponseBodyMimeTypesClear'; modsecurity_rules ' SecRuleEngine On SecResponseBodyAccess On + SecResponseBodyLimit 1048576 + SecResponseBodyLimitAction ProcessPartial SecResponseBodyMimeType text/html text/plain application/json SecRule RESPONSE_BODY "@rx sensitive-marker" "id:940103,phase:4,deny,log,status:403" '; @@ -31,4 +33,4 @@ http { proxy_pass http://app_backend; } } -} \ No newline at end of file +} diff --git a/docs/phase4-handling.de.md b/docs/phase4-handling.de.md index 4ee7fac4..ca296196 100644 --- a/docs/phase4-handling.de.md +++ b/docs/phase4-handling.de.md @@ -9,8 +9,10 @@ HTTP-Status nicht zuverlässig ersetzen. Dieser Branch übernimmt die relevante nginx-Implementierung aus [Easton97-Jens/ModSecurity-conector, Commit b0f3bdab429717b5b0311c30c5b4d1153c672ac0](https://github.com/Easton97-Jens/ModSecurity-conector/tree/b0f3bdab429717b5b0311c30c5b4d1153c672ac0/connectors/nginx/src). -Sie wird an dieses eigenständige Modul angepasst; das Phase-4-JSON-Lines-Schema -dieses Branches bleibt erhalten. Die Multi-Connector-Laufzeit und +Die Aktualisierung der Zuständigkeit für Response-Limits folgt +[Commit 820b6975495bdf0f90aca67eee86e27a3b7d329b](https://github.com/Easton97-Jens/ModSecurity-conector/blob/820b6975495bdf0f90aca67eee86e27a3b7d329b/docs/phase4-mode-budget.de.md). +Diese Änderungen werden an dieses eigenständige Modul angepasst; dessen +Phase-4-JSON-Lines-Schema bleibt erhalten. Die Multi-Connector-Laufzeit und Laufzeitnachweise des Quellrepositories werden nicht übernommen. Dieses Dokument beschreibt das Verhalten im Quellcode. Es behauptet nicht, @@ -25,27 +27,30 @@ Alle drei Connector-Direktiven gelten in `http`-, `server`- und | Direktive | Werte | Standard | | --- | --- | --- | | `modsecurity_phase4_mode` | `off`, `safe`, `strict` | `off` | -| `modsecurity_phase4_body_limit` | Positive Byte-Anzahl oder nginx-Größe, z. B. `256k`, `2m` | `1m` (1 MiB) | +| `modsecurity_phase4_body_limit` | Veralteter, ignorierter Kompatibilitätswert; positive Bytes oder nginx-Größe, z. B. `256k`, `2m` | `1m` (1 MiB), ignoriert | | `modsecurity_phase4_log` | Dateipfad für Phase-4-JSON-Lines-Ereignisse | Kein dediziertes Log | -Bei der Migration der bisherigen `master-phase4`-Konfiguration sind diese -Änderungen erforderlich: +Bei der Migration älterer Phase-4-Konfigurationen gilt: - `minimal` ist nicht mehr gültig und kein Alias für `off`. -- Der Standardmodus wechselt von `safe` auf `off`. Für die zusätzliche - Connector-Behandlung muss `safe` oder `strict` explizit gesetzt werden. +- Der Standard ist `off`; ältere Konfigurationen konnten sich auf `safe` + verlassen. Für die zusätzliche Connector-Behandlung muss `safe` oder + `strict` explizit gesetzt werden. - `modsecurity_phase4_content_types_file` entfällt und verursacht einen nginx-Konfigurationsfehler. Die MIME-Auswahl gehört jetzt in ModSecurity-Regeln. -- Das positive Connector-Body-Budget ist neu. Bei `safe` oder `strict` muss - ein passender Wert für die Antworten gewählt werden; null ist ungültig. +- `modsecurity_phase4_body_limit` ist jetzt veraltet und wird in allen gültigen + Modi ignoriert. Parser für positive Werte, Vererbung und historischer + Standardwert von 1 MiB bleiben kompatibel; null ist weiterhin ungültig. + Inspection-Limits auf `SecResponseBodyLimit` und `SecResponseBodyLimitAction` + umstellen. Die migrierte Konfiguration vor dem nginx-Reload mit `nginx -t` in der Zielumgebung prüfen. ## 3) Modusverhalten und Header-Zeitpunkt -`off` deaktiviert die zusätzliche Phase-4-Interventionsbehandlung und das -Body-Budget des Connectors. Es **deaktiviert weder ModSecurity** noch die +`off` deaktiviert die zusätzliche Phase-4-Interventionsbehandlung des Connectors. +Es **deaktiviert weder ModSecurity** noch die Response-Body-Prüfung oder Phase-4-Regeln. Die native Interventionsbehandlung bleibt aktiv. Tritt eine native Intervention nach dem Header-Versand auf, kann die Finalisierung einen Transportfehler verursachen; `off` garantiert @@ -60,7 +65,8 @@ Für die zusätzliche Behandlung in `safe` und `strict` gilt: `safe` degradiert nur späte **Interventionen**. Fehler der Engine-API, ungültige Buffer, fehlgeschlagene Dateizugriffe, Speicherallokationsfehler, -Zählerüberläufe und überschrittene Body-Budgets bleiben Fehler. Sie werden +Zählerüberläufe und Fehler bei der finalen Verarbeitung bleiben Fehler. +Sie werden nicht in erfolgreiche Weiterleitung umgewandelt. `strict` kann eine abgeschnittene Antwort oder einen Transportfehler beim @@ -68,19 +74,35 @@ Client/Proxy auslösen. Nach dem Header-Versand garantiert es keinen sauberen 403-, 401-, 301- oder 302-Status. Bereits weitergeleitete Body-Bytes lassen sich nicht zurückholen. -## 4) Response-Body-Budget und Streaming +## 4) Engine-Inspection-Limits und Streaming -`modsecurity_phase4_body_limit` ist in `safe` und `strict` ein -Connector-Budget für die kumulativ gesehenen Response-Bytes, einschließlich -dateibasierter Buffer. Es ist unabhängig von `SecResponseBodyLimit` der -Engine und zählt nicht nur die Bytes, die die Engine zur Prüfung speichert. +libModSecurity bestimmt das Byte-Limit der WAF-Response-Inspection über +`SecResponseBodyLimit` und `SecResponseBodyLimitAction` in jedem Phase-4-Modus. +Der Connector ergänzt keine kumulierte Inspection-Grenze für `safe`, `strict` +oder `off` und weist eine Antwort nicht allein deshalb ab, weil sie +`modsecurity_phase4_body_limit` überschreitet. -Eine Antwort darf das Budget exakt erreichen. Ein Chunk, der es überschreiten -würde, wird **vor seiner Weiterleitung** abgewiesen. Frühere Chunks können -bereits beim Client angekommen sein; ein ersetzender HTTP-Fehlerstatus ist -daher nicht garantiert. In `off` wird das konfigurierte Connector-Budget -ignoriert, die kumulative Zählung weist Überläufe über `SIZE_MAX` aber weiterhin -ab. +Beispielsweise diese Engine-Policy in den ModSecurity-Regeln konfigurieren: + +```apache +SecResponseBodyLimit 1048576 +SecResponseBodyLimitAction ProcessPartial +``` + +1 MiB ist hier ein expliziter Beispielwert, kein neuer Engine-Standard. +`ProcessPartial` wählt die Prüfung des Anteils innerhalb des Engine-Limits; +`Reject` ist die alternative Engine-Aktion, wenn Abweisung erforderlich ist. +Der Modus überschreibt diese Engine-Policy nicht. Daraus entstehende späte +Interventionen folgen weiterhin dem ausgewählten Connector-Modus. + +Die veraltete Connector-Einstellung bleibt parsebar und wird vererbt, ihr Wert +wird aber in keinem gültigen Modus durchgesetzt. Erfolgreiches Laden der +Konfiguration belegt daher nicht die alte Connector-Inspection-Grenze. +Diese Policy auf die oben genannten Engine-Einstellungen umstellen. + +Die geprüfte kumulierte Bytezählung weist Überläufe über `SIZE_MAX` weiterhin +in allen Modi ab. Echte Verarbeitungs- und Datei-Read-Fehler bleiben Fehler; +nichtfatale Engine-Aufnahme mit `ProcessPartial` ist kein Connector-Fehler. Der Connector puffert Antworten nicht global und ordnet nginx-Body-Ketten nicht um. Speicherbuffer werden direkt geprüft. Rein dateibasierte Buffer werden @@ -116,7 +138,8 @@ Das Zurücksetzen erfolgt getrennt, weil die MIME-Werte entfernen kann, die im selben Regel-Ladevorgang hinzugefügt werden. Das eigenständige [Engine-MIME-Beispiel](examples/phase4-engine-mime.conf) -enthält MIME-Ergänzungen und aktiviert die Response-Body-Prüfung. Es ist eine +enthält MIME-Ergänzungen, Response-Body-Prüfung und eine explizite +Engine-Limit-Policy. Es ist eine ModSecurity-Regeldatei, kein nginx-Include. Um die MIME-Liste der Engine durch diese Datei zu ersetzen, zuerst das Zurücksetzen und dann die Datei zusätzlich zu den anderen Regeln laden: @@ -129,8 +152,8 @@ modsecurity_rules_file /etc/modsecurity/phase4-engine-mime.conf; Die vollständigen nginx-Beispiele unten konfigurieren die MIME-Auswahl inline mit derselben Reihenfolge getrennter Ladevorgänge. -Die Engine-Auswahl deaktiviert das unabhängige Connector-Body-Budget in -`safe` oder `strict` nicht. +MIME-Auswahl und Inspection-Limits der Engine gelten in jedem Phase-4-Modus; +es gibt kein zusätzliches Connector-Inspection-Budget. ## 6) Logging-Format und Sicherheitsgrenze @@ -154,15 +177,17 @@ vollständige Erfassung aller Engine-, Buffer- oder I/O-Fehler. ## 7) Konfigurationsbeispiele und Verifikation -- [off](examples/phase4-off.conf): native Interventionsbehandlung bei - eingeschalteter Engine-Prüfung. -- [safe](examples/phase4-safe.conf): explizite späte `log_only`-Behandlung und - 1 MiB Connector-Budget. -- [strict](examples/phase4-strict.conf): später Verbindungsabbruch und - 1 MiB Connector-Budget. +- [off](examples/phase4-off.conf): native Interventionsbehandlung mit expliziter + Engine-Inspection-Policy. +- [safe](examples/phase4-safe.conf): explizite späte `log_only`-Behandlung. +- [strict](examples/phase4-strict.conf): später Verbindungsabbruch. + - [Engine-MIME-Auswahl](examples/phase4-engine-mime.conf): ModSecurity-Konfiguration für den Response-Body. +Alle drei Beispiele konfigurieren dasselbe Engine-Limit von 1 MiB mit +`ProcessPartial`; diese Limit-Policy ist vom Connector-Modus unabhängig. + Die nginx-Beispiele verwenden `location /` und einen Beispiel-Upstream unter `127.0.0.1:8081`. Listen-Adresse, Upstream, Logpfad und Regeln an die Zielumgebung anpassen. Die Regel mit `sensitive-marker` dient zur Illustration. @@ -170,6 +195,7 @@ Zielumgebung anpassen. Die Regel mit `sensitive-marker` dient zur Illustration. Repository-Testendpunkte wie `/phase4` sind Testfixtures, keine erforderlichen Produktionspfade. Laufzeitprüfungen sollten späte Deny-/Redirect-Interventionen, HTTP/1.1 und HTTP/2, dateibasierte Antworten, Subrequests, wiederholte -Finalisierung, Budgetgrenzen und Fehlerpfade abdecken. Ergebnisse aus einem +Finalisierung, Engine-Limit-Grenzen, Antworten oberhalb der ignorierten alten +Einstellung und Fehlerpfade abdecken. Ergebnisse aus einem anderen Repository oder Build belegen diese Eigenschaften des eingesetzten Moduls nicht. diff --git a/docs/phase4-handling.en.md b/docs/phase4-handling.en.md index 41f44318..ac6253f9 100644 --- a/docs/phase4-handling.en.md +++ b/docs/phase4-handling.en.md @@ -9,7 +9,9 @@ client. This branch adopts the relevant nginx implementation from [Easton97-Jens/ModSecurity-conector, commit b0f3bdab429717b5b0311c30c5b4d1153c672ac0](https://github.com/Easton97-Jens/ModSecurity-conector/tree/b0f3bdab429717b5b0311c30c5b4d1153c672ac0/connectors/nginx/src). -It is adapted to this standalone module and preserves this branch's phase 4 +The response-limit ownership update follows +[commit 820b6975495bdf0f90aca67eee86e27a3b7d329b](https://github.com/Easton97-Jens/ModSecurity-conector/blob/820b6975495bdf0f90aca67eee86e27a3b7d329b/docs/phase4-mode-budget.md). +These changes are adapted to this standalone module and preserve its phase 4 JSON-lines schema. The source repository's multi-connector runtime and run evidence are not imported. @@ -24,27 +26,28 @@ All three connector directives below are valid in `http`, `server`, and | Directive | Values | Default | | --- | --- | --- | | `modsecurity_phase4_mode` | `off`, `safe`, `strict` | `off` | -| `modsecurity_phase4_body_limit` | Positive byte count or nginx size, e.g. `256k`, `2m` | `1m` (1 MiB) | +| `modsecurity_phase4_body_limit` | Deprecated, ignored compatibility value; positive bytes or nginx size, e.g. `256k`, `2m` | `1m` (1 MiB), ignored | | `modsecurity_phase4_log` | File path for phase 4 JSON-lines events | No dedicated log | -Migration from the previous `master-phase4` configuration requires these -changes: +When migrating older phase 4 configurations, note: - `minimal` is no longer valid and is not an alias for `off`. -- The default mode changes from `safe` to `off`. Set `safe` or `strict` - explicitly to enable the additional connector policy. +- The default is `off`; older configurations may have relied on `safe`. + Set `safe` or `strict` explicitly to enable the additional connector policy. - `modsecurity_phase4_content_types_file` is removed and causes an nginx configuration error. Move MIME selection into ModSecurity rules. -- The positive connector body budget is new. Choose a value suitable for your - responses when enabling `safe` or `strict`; zero is invalid. +- `modsecurity_phase4_body_limit` is now deprecated and ignored in all valid + modes. Its positive-value parser, inheritance, and historical 1 MiB default + remain for compatibility; zero is still invalid. Move inspection limits into + `SecResponseBodyLimit` and `SecResponseBodyLimitAction`. Validate the migrated configuration with `nginx -t` in the deployment environment before reloading nginx. ## 3) Mode behavior and header timing -`off` disables the connector's additional phase 4 intervention policy and body -budget. It **does not disable ModSecurity**, response-body inspection, or phase +`off` disables the connector's additional phase 4 intervention policy. +It **does not disable ModSecurity**, response-body inspection, or phase 4 rules. Native intervention handling remains active. If a native intervention occurs after headers have been committed, response finalization can fail the transport; `off` is not a promise to deliver every response. @@ -57,25 +60,43 @@ For the additional policy in `safe` and `strict`: | Headers already sent | Log the intervention as `log_only` and continue | Log `connection_abort` and terminate the response | `safe` only downgrades late **interventions**. An engine API failure, invalid -buffer, failed file read, allocation failure, counter overflow, or exceeded -body budget remains a failure. It is not converted into successful forwarding. +buffer, failed file read, allocation failure, counter overflow, or final +processing failure remains a failure. It is not converted into successful +forwarding. `strict` can produce a truncated response or a client/proxy transport error. It cannot guarantee a clean 403, 401, 301, or 302 after headers have been sent. Previously forwarded body bytes cannot be recalled. -## 4) Response-body budget and streaming +## 4) Engine inspection limits and streaming -`modsecurity_phase4_body_limit` is a connector budget for the cumulative -response bytes it sees in `safe` and `strict`, including file-backed buffers. -It is separate from the engine's `SecResponseBodyLimit` and is not restricted -to the bytes the engine retains for inspection. +libModSecurity owns the WAF response-inspection byte limit through +`SecResponseBodyLimit` and `SecResponseBodyLimitAction` in every phase 4 mode. +The connector does not add a cumulative inspection ceiling for `safe`, +`strict`, or `off`, and does not reject a response merely because it exceeds +`modsecurity_phase4_body_limit`. -A response may reach the budget exactly. A chunk that would cross it is -rejected **before that chunk is forwarded**. Earlier chunks may already have -reached the client, so this does not guarantee a replacement HTTP error -status. In `off`, the configured connector budget is ignored, but cumulative -accounting still rejects overflow beyond `SIZE_MAX`. +For example, configure this engine policy in your ModSecurity rules: + +```apache +SecResponseBodyLimit 1048576 +SecResponseBodyLimitAction ProcessPartial +``` + +The 1 MiB value is an explicit example setting, not a new engine default. +`ProcessPartial` selects inspection of the portion within the engine limit; +`Reject` is the alternative engine action when rejection is required. Mode +selection does not override that engine policy. Any resulting late intervention +still follows the selected connector mode. + +The deprecated connector setting remains parseable and inherited, but its +value has no enforcement effect in any valid mode. A successful configuration +load therefore does not establish the old connector inspection limit. Migrate +that policy to the engine settings above. + +Checked cumulative byte accounting still rejects overflow beyond `SIZE_MAX` +in all modes. Genuine processing and file-read failures remain failures; +nonfatal engine `ProcessPartial` ingestion is not a connector failure. The connector does not globally buffer responses or reorder nginx body chains. Memory buffers are inspected directly. File-only buffers are read @@ -109,7 +130,8 @@ The reset is separate because libModSecurity's can clear MIME values added in the same rule load. The standalone [engine MIME example](examples/phase4-engine-mime.conf) contains -MIME additions and enables response-body access. It is a ModSecurity rules +MIME additions, response-body access, and an explicit engine limit policy. +It is a ModSecurity rules file, not an nginx include. To replace the engine's MIME list with this file, load the reset first, then the file, alongside your other rules: @@ -121,8 +143,8 @@ modsecurity_rules_file /etc/modsecurity/phase4-engine-mime.conf; The complete nginx examples below configure MIME selection inline using the same separate-load sequence. -Engine selection does not disable the independent connector body budget in -`safe` or `strict`. +Engine MIME selection and inspection limits apply in every phase 4 mode; +there is no additional connector inspection budget. ## 6) Logging format and security boundary @@ -145,15 +167,17 @@ all engine, buffer, or I/O errors. ## 7) Configuration examples and verification -- [off](examples/phase4-off.conf): native intervention handling with engine - inspection enabled. -- [safe](examples/phase4-safe.conf): explicit late `log_only` policy and a - 1 MiB connector budget. -- [strict](examples/phase4-strict.conf): late connection termination and a - 1 MiB connector budget. +- [off](examples/phase4-off.conf): native intervention handling with explicit + engine inspection policy. +- [safe](examples/phase4-safe.conf): explicit late `log_only` policy. +- [strict](examples/phase4-strict.conf): late connection termination. + - [engine MIME selection](examples/phase4-engine-mime.conf): ModSecurity response-body configuration. +All three examples configure the same 1 MiB engine limit with `ProcessPartial`; +that limit policy is independent of the connector mode. + The nginx examples use `location /` and an example upstream at `127.0.0.1:8081`. Adapt the listen address, upstream, log path, and rules to your environment. The `sensitive-marker` rule is illustrative. @@ -161,5 +185,6 @@ your environment. The `sensitive-marker` rule is illustrative. Repository test endpoints such as `/phase4` are test fixtures, not required production paths. Runtime checks should cover late deny/redirect behavior, HTTP/1.1 and HTTP/2, file-backed responses, subrequests, repeated finalization, -budget boundaries, and failure paths. Results from another repository or build -do not establish those behaviors for the deployed module. \ No newline at end of file +engine-limit boundaries, responses above the ignored legacy setting, and +failure paths. Results from another repository or build +do not establish those behaviors for the deployed module. diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index b23d0f14..5abe6003 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -58,15 +58,14 @@ ngx_http_modsecurity_response_body_failure(ngx_http_request_t *r, NGX_HTTP_INTERNAL_SERVER_ERROR); } -/* This is an inspection budget. SIZE_MAX in off is an accounting ceiling, - * never an allocation size; native engine and host limits remain active. */ +/* libModSecurity owns response inspection limits in every Phase-4 mode. + * Keep checked accounting without imposing the legacy connector budget; + * independent host, file-buffer and allocation limits remain active. */ static ngx_int_t ngx_http_modsecurity_plan_limited_response_body( ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf, size_t len, size_t *allowed) { - size_t limit; - if (allowed == NULL) { return NGX_ERROR; } @@ -78,17 +77,12 @@ ngx_http_modsecurity_plan_limited_response_body( return NGX_OK; } ctx->response_body_seen = 1; - if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_OFF) { - limit = SIZE_MAX; - } else if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_SAFE || - mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { - limit = mcf->phase4_body_limit; - } else { + if (mcf->phase4_mode != NGX_HTTP_MODSEC_PHASE4_MODE_OFF && + mcf->phase4_mode != NGX_HTTP_MODSEC_PHASE4_MODE_SAFE && + mcf->phase4_mode != NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { return NGX_ERROR; } - if (limit == 0U || - ctx->response_body_bytes_inspected > ctx->response_body_bytes_seen || - ctx->response_body_bytes_inspected > limit) { + if (ctx->response_body_bytes_inspected > ctx->response_body_bytes_seen) { ctx->response_body_truncated = 1; return NGX_ERROR; } @@ -98,10 +92,6 @@ ngx_http_modsecurity_plan_limited_response_body( return NGX_ERROR; } ctx->response_body_bytes_seen += len; - if (ctx->response_body_bytes_seen > limit) { - ctx->response_body_truncated = 1; - return NGX_ERROR; - } *allowed = len; return NGX_OK; } diff --git a/src/ngx_http_modsecurity_common.h b/src/ngx_http_modsecurity_common.h index 30f5fca8..ad64d327 100644 --- a/src/ngx_http_modsecurity_common.h +++ b/src/ngx_http_modsecurity_common.h @@ -144,6 +144,7 @@ typedef struct { ngx_http_complex_value_t *transaction_id; ngx_uint_t phase4_mode; + /* Deprecated compatibility value; libModSecurity owns inspection limits. */ size_t phase4_body_limit; ngx_open_file_t *phase4_log_file; ngx_str_t phase4_log_path; diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index 823023f7..03eb5f95 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -528,6 +528,7 @@ ngx_conf_set_phase4_mode(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_OK; } +/* Keep legacy configurations parseable; use SecResponseBodyLimit in rules. */ static char * ngx_conf_set_phase4_body_limit(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/tests/README.md b/tests/README.md index bb863e97..268f0408 100644 --- a/tests/README.md +++ b/tests/README.md @@ -14,16 +14,19 @@ http://hg.nginx.org/nginx-tests/ Run from the repository root with Python 3 and GCC or Clang: ```sh -CC=gcc python3 tests/test_phase4_runtime.py -v -CC=clang python3 tests/test_phase4_runtime.py -v +CC=gcc python3 -m unittest discover -s tests -p 'test_*runtime.py' -v +CC=clang python3 -m unittest discover -s tests -p 'test_*runtime.py' -v ``` -The suite compiles the production body-filter functions and request-context -lookup with small NGINX and libModSecurity doubles. It checks mode-aware byte -limits, overflow, memory/file buffers, bounded file reads and allocation/read +The suites compile the production body-filter functions, request-context +lookup, and intervention helpers with small NGINX and libModSecurity doubles. +They check forwarding above the deprecated connector limit in every mode, +checked accounting, overflow, memory/file buffers, bounded file reads and allocation/read errors, finalization at EOS, native failures, late interventions, downstream -`NGX_AGAIN`, request ownership, and internal redirect recovery. It needs no -Common connector library. These are helper behavior tests; they do not replace +`NGX_AGAIN`, request ownership, and internal redirect recovery. They need no +Common connector library. The intervention cases run with sanity checks enabled +and disabled and cover redirect ownership, status values, and cleanup. These +are helper behavior tests; they do not replace the native HTTP integration tests. ## Native Phase4 integration tests @@ -38,8 +41,9 @@ TEST_NGINX_BINARY=/absolute/path/to/nginx prove modsecurity*.t The Phase4 suites cover `off` (the default), `safe`, and `strict`, JSON event logging, engine-owned `SecResponseBodyMimeType` selection, complete response -delivery above the optional connector budget in `off`, and budget rejection -in `safe`/`strict`. They also reject the removed `minimal` mode and connector +delivery above the deprecated connector limit in every mode, and engine-owned +`SecResponseBodyLimit` behavior with `Reject` and `ProcessPartial`. They also +reject the removed `minimal` mode and connector MIME directive. Late deny/redirect assertions check transport interruption, because headers have already been committed; they do not promise a clean 403. diff --git a/tests/modsecurity-phase4-engine-limits.t b/tests/modsecurity-phase4-engine-limits.t new file mode 100644 index 00000000..1c92905c --- /dev/null +++ b/tests/modsecurity-phase4-engine-limits.t @@ -0,0 +1,123 @@ +#!/usr/bin/perl +use warnings; use strict; +use Test::More; +use JSON::PP qw(decode_json); +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +# Exercise libModSecurity's real append result and queued intervention. The +# static file exceeds the connector's file-reading chunk size; its legacy +# limit is deliberately smaller than the engine's inspection limit. +my $t = Test::Nginx->new()->has(qw/http/)->todo_alerts(); +my $prefix = 'ENGINE_PREFIX_' . ('A' x 50); # exactly 64 bytes +my $body = $prefix . ('B' x 70000) . 'ENGINE_TAIL_MARKER'; +my $locations = ''; +my $rule_id = 950000; + +for my $action (qw/partial reject/) { + my $engine_action = $action eq 'partial' ? 'ProcessPartial' : 'Reject'; + for my $mode (qw/off safe strict/) { + my $prefix_id = ++$rule_id; + my $tail_id = ++$rule_id; + $locations .= <write_file('/' . $action . '-' . $mode, $body); + } +} + +$t->write_file_expand('nginx.conf', <run(); +$t->plan(37); + +for my $mode (qw/off safe strict/) { + my $response = http_get('/partial-' . $mode); + like($response, qr/^HTTP\/1\.1 200 OK/, "$mode accepts engine ProcessPartial"); + my ($headers, $payload) = split /\x0d\x0a\x0d\x0a/, $response, 2; + is($payload, $body, "$mode ProcessPartial forwards the entire static file"); +} + +# Static response headers have already passed the header filter when the +# engine queues its body-limit 403. Preserve each existing late-deny policy. +is(http_get('/reject-off'), '', 'off preserves native late engine Reject abort'); +my $safe_response = http_get('/reject-safe'); +like($safe_response, qr/^HTTP\/1\.1 200 OK/, 'safe continues after engine Reject'); +my ($safe_headers, $safe_payload) = split /\x0d\x0a\x0d\x0a/, $safe_response, 2; +is($safe_payload, $body, 'safe engine Reject forwards the complete static file'); +is(http_get('/reject-strict'), '', 'strict aborts after engine Reject'); + +$t->stop(); +my $log = $t->read_file('phase4-engine-limits.log'); +my @events; +my $valid_json = eval { + @events = map { decode_json($_) } grep { length $_ } split /\n/, $log; + 1; +}; +ok($valid_json, 'every engine-limit Phase4 event is valid JSON'); +diag($@) unless $valid_json; +is(scalar @events, 2, 'only safe and strict engine Reject emit dedicated events'); +my %event = map { $_->{uri} => $_ } @events; +ok(!grep({ $_->{uri} =~ m{^/partial-} } @events), 'ProcessPartial emits no intervention event in any mode'); +ok(!exists $event{'/reject-off'}, 'off engine Reject emits no dedicated event'); + +for my $mode (qw/safe strict/) { + my $item = $event{'/reject-' . $mode} || {}; + is($item->{event}, 'phase4_intervention', "$mode engine Reject event kind"); + is($item->{mode}, $mode, "$mode engine Reject retains the selected mode"); + is($item->{waf_status}, 403, "$mode retains the engine's body-limit 403"); + is($item->{wanted_action}, 'deny', "$mode retains the engine's deny decision"); + is($item->{actual_action}, $mode eq 'safe' ? 'log_only' : 'connection_abort', + "$mode applies the existing late-intervention action"); + is($item->{reason}, 'response_committed_' . $mode, "$mode records the existing late-intervention reason"); + ok($item->{header_sent}, "$mode engine Reject occurs after response headers are committed"); +} +unlike($log, qr/ENGINE_PREFIX_|B{100,}|ENGINE_TAIL_MARKER/, + 'raw Phase4 event log contains no response bytes'); + +my $error_log = $t->read_file('error.log'); +for my $mode (qw/off safe strict/) { + is(scalar(() = $error_log =~ /engine-prefix-exact64-partial-\Q$mode\E/g), 1, + "$mode engine RESPONSE_BODY is exactly the first 64 bytes"); +} +unlike($error_log, qr/engine-tail-seen-/, 'engine inspection excludes the tail in every mode'); +unlike($error_log, qr/engine-prefix-exact64-reject-/, + 'engine Reject does not expose a partial inspection prefix'); +unlike($error_log, qr/response body inspection append failed/, + 'the engine Reject append result is not mistaken for an API failure'); + +# Native off finalization has the same expected alert as other late denials. +my @alerts = $error_log =~ /^.*\[alert\].*$/gm; +my $expected_alert = qr/\[alert\].*header already sent\b.*request: "GET \/reject-off HTTP\/1\.[01]"/; +is(scalar(grep { /$expected_alert/ } @alerts), 1, 'off engine Reject has one expected late-finalization alert'); +is(join("\n", grep { !/$expected_alert/ } @alerts), '', 'no unexpected nginx alerts'); diff --git a/tests/modsecurity-phase4-regression.t b/tests/modsecurity-phase4-regression.t index 40875eb4..8e8b62e3 100644 --- a/tests/modsecurity-phase4-regression.t +++ b/tests/modsecurity-phase4-regression.t @@ -38,13 +38,17 @@ for my $mode (qw/off safe strict/) { $t->write_file('/' . $mode, $big); } $t->run(); -$t->plan(6); +$t->plan(8); -my $resp = http_get('/off'); -like($resp, qr/HTTP\/1\.1 200 OK/, 'off ignores optional connector body budget'); -my ($headers, $body) = split /\x0d\x0a\x0d\x0a/, $resp, 2; -is($body, $big, 'off forwards the complete large response without truncation'); -like($t->read_file('error.log'), qr/phase4-tail-seen/, 'off still processes the response body through the engine'); -unlike(http_get('/safe'), qr/TAIL/, 'safe rejects a response exceeding the connector budget'); -unlike(http_get('/strict'), qr/TAIL/, 'strict rejects a response exceeding the connector budget'); -unlike($t->read_file('phase4-regression.log'), qr/A{100,}|TAIL/, 'response bytes are absent from event logs'); +for my $mode (qw/off safe strict/) { + my $resp = http_get('/' . $mode); + like($resp, qr/HTTP\/1\.1 200 OK/, "$mode ignores the legacy connector limit"); + my ($headers, $body) = split /\x0d\x0a\x0d\x0a/, $resp, 2; + is($body, $big, "$mode forwards the complete large response"); +} +$t->stop(); +my $error_log = $t->read_file('error.log'); +is(scalar(() = $error_log =~ /phase4-tail-seen/g), 3, + 'the engine inspects the tail beyond the legacy limit in every mode'); +unlike($t->read_file('phase4-regression.log'), qr/A{100,}|TAIL/, + 'response bytes are absent from event logs'); diff --git a/tests/test_phase4_runtime.py b/tests/test_phase4_runtime.py index 55691f01..d3be5f03 100644 --- a/tests/test_phase4_runtime.py +++ b/tests/test_phase4_runtime.py @@ -251,25 +251,31 @@ def function_definition(source: str, name: str) -> str: final.last_in_chain = 0; final.last_buf = 1; CHECK(ngx_http_modsecurity_body_filter(&request, &tail) == NGX_OK); CHECK(process_calls == 1 && ctx.phase4_processed && forward_calls == 2); - } else if (strcmp(argv[1], "off-large") == 0) { - conf.phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_OFF; - CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, - 1048577U, &allowed) == NGX_OK); - CHECK(allowed == 1048577U && ctx.response_body_bytes_seen == allowed); - CHECK(ctx.response_body_seen && !ctx.response_body_truncated); - CHECK(ctx.response_body_bytes_inspected == 0U); - } else if (strcmp(argv[1], "off-multiple") == 0) { - conf.phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_OFF; - for (i = 0; i < 3; ++i) { + } else if (strcmp(argv[1], "all-modes-large") == 0) { + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, - 1048576U, &allowed) == NGX_OK); - CHECK(allowed == 1048576U); - ctx.response_body_bytes_inspected += allowed; + 1048577U, &allowed) == NGX_OK); + CHECK(allowed == 1048577U && ctx.response_body_bytes_seen == allowed); + CHECK(ctx.response_body_seen && !ctx.response_body_truncated); + CHECK(ctx.response_body_bytes_inspected == 0U); + } + } else if (strcmp(argv[1], "all-modes-multiple") == 0) { + size_t chunk; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + for (chunk = 0; chunk < 3; ++chunk) { + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1048576U, &allowed) == NGX_OK); + CHECK(allowed == 1048576U); + ctx.response_body_bytes_inspected += allowed; + } + CHECK(ctx.response_body_bytes_seen == 3U * 1048576U); + CHECK(ctx.response_body_bytes_inspected == ctx.response_body_bytes_seen); + CHECK(!ctx.response_body_truncated); } - CHECK(ctx.response_body_bytes_seen == 3U * 1048576U); - CHECK(!ctx.response_body_truncated); - } else if (strcmp(argv[1], "budget-boundary") == 0) { - for (i = 1; i <= 2; ++i) { + } else if (strcmp(argv[1], "legacy-boundary") == 0) { + for (i = 0; i <= 2; ++i) { memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, 1048575U, &allowed) == NGX_OK); @@ -278,34 +284,52 @@ def function_definition(source: str, name: str) -> str: 1U, &allowed) == NGX_OK); ctx.response_body_bytes_inspected += allowed; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, - 1U, &allowed) == NGX_ERROR); - CHECK(allowed == 0U && ctx.response_body_truncated); + 1U, &allowed) == NGX_OK); + ctx.response_body_bytes_inspected += allowed; + CHECK(allowed == 1U && !ctx.response_body_truncated); CHECK(ctx.response_body_bytes_seen == 1048577U); - CHECK(ctx.response_body_bytes_inspected == 1048576U); + CHECK(ctx.response_body_bytes_inspected == 1048577U); + } + } else if (strcmp(argv[1], "memory-over-legacy-limit") == 0) { + conf.phase4_body_limit = 65536U; + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + sizeof(payload); + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + append_calls = 0; appended_bytes = 0; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(append_calls == 1 && appended_bytes == sizeof(payload)); + CHECK(read_calls == 0 && allocation_calls == 0); + CHECK(ctx.response_body_bytes_seen == sizeof(payload)); + CHECK(ctx.response_body_bytes_inspected == sizeof(payload)); + CHECK(!ctx.response_body_truncated); + CHECK(memcmp(inspected, payload, sizeof(payload)) == 0); + CHECK(buffer.pos == payload && buffer.last == payload + sizeof(payload)); } - } else if (strcmp(argv[1], "oversized-first") == 0) { - for (i = 1; i <= 2; ++i) { + } else if (strcmp(argv[1], "overflow") == 0) { + for (i = 0; i <= 2; ++i) { memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + ctx.response_body_bytes_seen = SIZE_MAX - 1U; + ctx.response_body_bytes_inspected = SIZE_MAX - 1U; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, - 1048577U, &allowed) == NGX_ERROR); + 2U, &allowed) == NGX_ERROR); CHECK(allowed == 0U && ctx.response_body_truncated); - CHECK(ctx.response_body_bytes_inspected == 0U); + CHECK(ctx.response_body_bytes_seen == SIZE_MAX); + CHECK(ctx.response_body_bytes_inspected == SIZE_MAX - 1U); + CHECK(ngx_http_modsecurity_append_response_body_chunk(&ctx, + payload, 2U) == NGX_ERROR); + CHECK(append_calls == 0); } - } else if (strcmp(argv[1], "overflow") == 0) { - conf.phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_OFF; - ctx.response_body_bytes_seen = SIZE_MAX - 1U; - ctx.response_body_bytes_inspected = SIZE_MAX - 1U; - CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, - 2U, &allowed) == NGX_ERROR); - CHECK(allowed == 0U && ctx.response_body_truncated); - CHECK(ctx.response_body_bytes_seen == SIZE_MAX); - CHECK(ctx.response_body_bytes_inspected == SIZE_MAX - 1U); } else if (strcmp(argv[1], "invalid-accounting") == 0) { - ctx.response_body_bytes_inspected = 1U; - CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, - 1U, &allowed) == NGX_ERROR); - CHECK(allowed == 0U && ctx.response_body_truncated); - CHECK(ctx.response_body_bytes_seen == 0U); + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + ctx.response_body_bytes_inspected = 1U; + CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, + 1U, &allowed) == NGX_ERROR); + CHECK(allowed == 0U && ctx.response_body_truncated); + CHECK(ctx.response_body_bytes_seen == 0U); + } } else if (strcmp(argv[1], "null-empty-mode") == 0) { CHECK(ngx_http_modsecurity_plan_limited_response_body(NULL, &conf, 1U, &allowed) == NGX_ERROR && allowed == 0U); @@ -330,19 +354,39 @@ def function_definition(source: str, name: str) -> str: CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, &conf, &buffer) == NGX_OK); CHECK(allocation_calls == 1 && appended_bytes == sizeof(payload) + 1U); - } else if (strcmp(argv[1], "mixed-once") == 0) { - buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 15; - CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, - &conf, &buffer) == NGX_OK); - CHECK(appended_bytes == 15U && append_calls == 1 && read_calls == 0); - CHECK(ctx.response_body_bytes_seen == 15U); - CHECK(buffer.pos == payload && buffer.last == payload + 15); - } else if (strcmp(argv[1], "file-limit") == 0) { - conf.phase4_body_limit = sizeof(payload) - 1U; - CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, - &conf, &buffer) == NGX_ERROR); - CHECK(read_calls == 0 && allocation_calls == 0 && append_calls == 0); - CHECK(ctx.response_body_truncated); + } else if (strcmp(argv[1], "mixed-over-legacy-limit") == 0) { + conf.phase4_body_limit = 65536U; + buffer.memory = 1; buffer.pos = payload; buffer.last = payload + sizeof(payload); + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + append_calls = 0; appended_bytes = 0; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(appended_bytes == sizeof(payload) && append_calls == 1); + CHECK(read_calls == 0 && allocation_calls == 0); + CHECK(ctx.response_body_bytes_seen == sizeof(payload)); + CHECK(ctx.response_body_bytes_inspected == sizeof(payload)); + CHECK(!ctx.response_body_truncated); + CHECK(memcmp(inspected, payload, sizeof(payload)) == 0); + CHECK(buffer.pos == payload && buffer.last == payload + sizeof(payload)); + CHECK(buffer.file_pos == 0 && buffer.file_last == sizeof(payload)); + } + } else if (strcmp(argv[1], "file-over-legacy-limit") == 0) { + conf.phase4_body_limit = 65536U; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + read_calls = 0; append_calls = 0; allocation_calls = 0; + appended_bytes = 0; read_max = 0; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(read_calls == 3 && append_calls == 3 && read_max == 32768U); + CHECK(allocation_calls == 1 && appended_bytes == sizeof(payload)); + CHECK(ctx.response_body_bytes_seen == sizeof(payload)); + CHECK(ctx.response_body_bytes_inspected == sizeof(payload)); + CHECK(!ctx.response_body_truncated); + CHECK(memcmp(inspected, payload, sizeof(payload)) == 0); + CHECK(buffer.file_pos == 0 && buffer.file_last == sizeof(payload)); + } } else if (strcmp(argv[1], "file-invalid") == 0) { buffer.file_pos = -1; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, @@ -353,6 +397,16 @@ def function_definition(source: str, name: str) -> str: buffer.file_pos = 0; buffer.file_last = 10; buffer.file = NULL; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, &conf, &buffer) == NGX_ERROR); + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, NULL) == NGX_ERROR); + buffer.memory = 1; buffer.pos = NULL; buffer.last = payload + 10; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + buffer.pos = payload + 10; buffer.last = payload; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(ngx_http_modsecurity_append_response_body_chunk(&ctx, + NULL, 1U) == NGX_ERROR); CHECK(read_calls == 0 && append_calls == 0); } else if (strcmp(argv[1], "allocation-failure") == 0) { fail_allocation = 1; @@ -367,16 +421,25 @@ def function_definition(source: str, name: str) -> str: CHECK(read_calls == 1 && append_calls == 0 && appended_bytes == 0U); } else if (strcmp(argv[1], "append-error") == 0) { fail_append = 1; - CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, - &conf, &buffer) == NGX_ERROR); - CHECK(ctx.response_body_bytes_inspected == 0U); + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_ERROR); + CHECK(ctx.response_body_bytes_inspected == 0U); + } } else if (strcmp(argv[1], "append-zero") == 0) { append_result = 0; + conf.phase4_body_limit = 1U; buffer.in_file = 0; buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 10; - CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, - &conf, &buffer) == NGX_OK); - CHECK(ctx.response_body_bytes_inspected == 10U && append_calls == 1); + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + append_calls = 0; appended_bytes = 0; + CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, + &conf, &buffer) == NGX_OK); + CHECK(ctx.response_body_bytes_inspected == 10U && append_calls == 1); + CHECK(!ctx.response_body_truncated); + } } else if (strcmp(argv[1], "final-once") == 0) { CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == NGX_OK); CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == NGX_OK); @@ -543,9 +606,9 @@ def test(self: Phase4RuntimeTests) -> None: for case in ( - "request-ownership", "off-large", "off-multiple", "budget-boundary", "oversized-first", + "request-ownership", "all-modes-large", "all-modes-multiple", "legacy-boundary", "memory-over-legacy-limit", "overflow", "invalid-accounting", "null-empty-mode", "file-chunks", - "mixed-once", "file-limit", "file-invalid", "allocation-failure", + "mixed-over-legacy-limit", "file-over-legacy-limit", "file-invalid", "allocation-failure", "short-read", "read-error", "append-error", "append-zero", "final-once", "engine-error", "process-zero", "main-eos", "subrequest-eos", "chain-restored", "forward-again", From acb72df63e426bd17b209a99425a35e74c157b9c Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:59:45 +0200 Subject: [PATCH 25/40] Update Windows runner version in workflow --- .github/workflows/test_new.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_new.yml b/.github/workflows/test_new.yml index 7a981176..afb842cc 100644 --- a/.github/workflows/test_new.yml +++ b/.github/workflows/test_new.yml @@ -269,7 +269,7 @@ jobs: build-windows: name: windows-nginx-${{ matrix.nginx_version }} - runs-on: windows-2025 + runs-on: windows-2022 strategy: fail-fast: false matrix: From d53c9eaf45ccd613ac791042ac723a40e4553bb9 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:13:00 +0200 Subject: [PATCH 26/40] Update test.yml --- .github/workflows/test.yml | 109 ------------------------------------- 1 file changed, 109 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2f8809d1..4a19e956 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -248,112 +248,3 @@ jobs: echo "FAIL" exit 1 fi - - - build-windows: - runs-on: windows-2022 - defaults: - run: - shell: msys2 {0} - steps: - - name: Set up MSVC - uses: ilammy/msvc-dev-cmd@v1 - - name: Set up msys - uses: msys2/setup-msys2@v2 - with: - msystem: UCRT64 - path-type: inherit - - name: Get Nginx source - uses: actions/checkout@v4 - with: - repository: nginx/nginx - path: nginx - fetch-depth: 1 - - name: Get Nginx tests - uses: actions/checkout@v4 - with: - repository: nginx/nginx-tests - path: nginx/test - fetch-depth: 1 - - name: Set up third-party libraries - working-directory: nginx - run: | - mkdir objs - mkdir objs/lib - cd objs/lib - wget -q -O - https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.39/pcre2-10.39.tar.gz | tar -xzf - - wget -q -O - https://www.zlib.net/fossils/zlib-1.3.tar.gz | tar -xzf - - wget -q -O - https://www.openssl.org/source/openssl-3.0.13.tar.gz | tar -xzf - - - name: Get libModSecurity source - uses: actions/checkout@v4 - with: - repository: owasp-modsecurity/ModSecurity - submodules: recursive - path: nginx/objs/lib/ModSecurity - fetch-depth: 1 - - name: Setup Conan - shell: cmd - run: | - pip3 install conan --upgrade - conan profile detect - - name: Build libModSecurity - working-directory: nginx/objs/lib/ModSecurity - shell: cmd - run: | - vcbuild.bat - - name: Get ModSecurity-nginx source code - uses: actions/checkout@v4 - with: - path: nginx/objs/lib/ModSecurity-nginx - - name: Copy ModSecurity-nginx tests to nginx/test - working-directory: nginx/test - run: | - cp ../objs/lib/ModSecurity-nginx/tests/*.t ../objs/lib/ModSecurity-nginx/tests/*.pl . - - name: Remove /usr/bin/link conflicting with MSVC link.exe - run: | - set -ex - which link - rm /usr/bin/link - - name: Build nginx w/ModSecurity-nginx module - working-directory: nginx - run: | - : # Windows native version of Perl is required by nginx build - export PATH=/c/Strawberry/perl/bin:$PATH - : # Set env variables to point to libModSecurity v3 include & lib directories - export MODSECURITY_INC=objs/lib/ModSecurity/headers - export MODSECURITY_LIB=objs/lib/ModSecurity/build/win32/build/Release - : # Copy libModSecurity.dll to objs dir (to be able to run nginx later) - cp $MODSECURITY_LIB/libModSecurity.dll objs - : # Configure nginx build w/ModSecurity-nginx module - auto/configure \ - --with-cc=cl \ - --with-debug \ - --prefix= \ - --conf-path=conf/nginx.conf \ - --pid-path=logs/nginx.pid \ - --http-log-path=logs/access.log \ - --error-log-path=logs/error.log \ - --sbin-path=nginx.exe \ - --http-client-body-temp-path=temp/client_body_temp \ - --http-proxy-temp-path=temp/proxy_temp \ - --http-fastcgi-temp-path=temp/fastcgi_temp \ - --http-scgi-temp-path=temp/scgi_temp \ - --http-uwsgi-temp-path=temp/uwsgi_temp \ - --with-cc-opt=-DFD_SETSIZE=1024 \ - --with-pcre=objs/lib/pcre2-10.39 \ - --with-zlib=objs/lib/zlib-1.3 \ - --with-openssl=objs/lib/openssl-3.0.13 \ - --with-openssl-opt=no-asm \ - --with-http_ssl_module \ - --with-http_v2_module \ - --with-http_auth_request_module \ - --add-module=objs/lib/ModSecurity-nginx - nmake - - name: Run ModSecurity-nginx tests - working-directory: nginx/test - shell: cmd # tests need to run on a "windows" shell - run: | - md temp - set TEMP=temp - set TEST_NGINX_BINARY=..\objs\nginx.exe - prove modsecurity*.t From a4a420b640d35898b833b1d1e649c994f2eaeeb5 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:16:11 +0200 Subject: [PATCH 27/40] Enhance CI workflow with new OS and Nginx versions Updated the workflow to support additional OS and Nginx versions. Changed checkout action version from v6 to v7. --- .github/workflows/test_new.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_new.yml b/.github/workflows/test_new.yml index afb842cc..cd7e7ac2 100644 --- a/.github/workflows/test_new.yml +++ b/.github/workflows/test_new.yml @@ -18,9 +18,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04] + os: [ubuntu-24.04,ubuntu-26.04] compiler: [gcc, clang] - nginx_version: ["1.30.x", "1.29.x", "1.28.x"] + nginx_version: ["1.31.x", "1.30.x", "1.29.x"] env: NGINX_VERSION: ${{ matrix.nginx_version }} CC: "/usr/bin/${{ matrix.compiler }}" @@ -45,7 +45,7 @@ jobs: ./configure --without-lmdb --prefix=/usr make -j $(nproc) sudo make install - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: path: ModSecurity-nginx fetch-depth: 1 @@ -55,7 +55,7 @@ jobs: run: bash ModSecurity-nginx/.github/scripts/resolve-nginx-ref.sh - name: Get Nginx source - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: nginx/nginx ref: ${{ steps.nginx_ref.outputs.ref }} @@ -73,7 +73,7 @@ jobs: grep '#define NGINX_VERSION' src/core/nginx.h || true - name: Get Nginx tests - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: nginx/nginx-tests path: nginx/test @@ -273,7 +273,7 @@ jobs: strategy: fail-fast: false matrix: - nginx_version: ["1.30.x", "1.29.x", "1.28.x"] + nginx_version: ["1.31.x", "1.30.x", "1.29.x"] env: NGINX_VERSION: ${{ matrix.nginx_version }} defaults: @@ -288,7 +288,7 @@ jobs: msystem: UCRT64 path-type: inherit - name: Get ModSecurity-nginx workflow scripts - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: path: ModSecurity-nginx fetch-depth: 1 @@ -298,7 +298,7 @@ jobs: run: bash ModSecurity-nginx/.github/scripts/resolve-nginx-ref.sh - name: Get Nginx source - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: nginx/nginx ref: ${{ steps.nginx_ref.outputs.ref }} @@ -316,7 +316,7 @@ jobs: grep '#define NGINX_VERSION' src/core/nginx.h || true - name: Get Nginx tests - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: nginx/nginx-tests path: nginx/test @@ -339,7 +339,7 @@ jobs: wget -q -O - https://www.zlib.net/fossils/zlib-1.3.2.tar.gz | tar -xzf - wget -q -O - https://www.openssl.org/source/openssl-3.6.2.tar.gz | tar -xzf - - name: Get libModSecurity source - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: owasp-modsecurity/ModSecurity submodules: recursive @@ -356,7 +356,7 @@ jobs: run: | vcbuild.bat - name: Get ModSecurity-nginx source code - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: path: nginx/objs/lib/ModSecurity-nginx - name: Copy ModSecurity-nginx tests to nginx/test From 73cdb3bc08a5fbc505d5adcf47097fb11c3b22a7 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:16:51 +0200 Subject: [PATCH 28/40] fix: correct phase4 event status type and README directive count --- README.md | 3 +-- src/ngx_http_modsecurity_body_filter.c | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d644a062..9a8e0c64 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Further information about nginx third-party add-ons support are available [here] # Usage ModSecurity for nginx extends your nginx configuration directives. -It adds eight directives and they are: +It adds nine directives and they are: modsecurity ----------- @@ -336,4 +336,3 @@ feel free to open GitHub issues requesting for new features. Before opening a ne Having our packages in distros on time is something we highly desire. Let us know if there is anything we can do to facilitate your work as a packager. - diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index 5abe6003..d446ee03 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -469,7 +469,9 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit } p = ngx_snprintf(dbuf, need, "{\"event\":\"phase4_intervention\",\"uri\":\"%V\",\"method\":\"%V\",\"response_status\":%ui,\"waf_status\":%i,\"content_type\":\"%V\",\"header_sent\":%s,\"mode\":\"%s\",\"wanted_action\":\"%s\",\"actual_action\":\"%s\",\"reason\":\"%s\",\"intervention\":\"%V\",\"rule_id\":\"%V\"}\n", - &euri,&emethod,(ngx_uint_t)r->headers_out.status,ctx ? (int) ctx->last_intervention_status : 0,&ect,header_sent,mode,wanted,actual,reason,&elog,&erule); + &euri, &emethod, (ngx_uint_t) r->headers_out.status, + ctx ? ctx->last_intervention_status : (ngx_int_t) 0, + &ect, header_sent, mode, wanted, actual, reason, &elog, &erule); ssize_t n = ngx_write_fd(mcf->phase4_log_file->fd, dbuf, p - dbuf); if (n < 0 || (size_t) n != (size_t) (p - dbuf)) { ngx_log_error(NGX_LOG_WARN, r->connection->log, ngx_errno, From 828742dc5d212a15a3a36654299329418b8f5c02 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:18:35 +0200 Subject: [PATCH 29/40] docs: preserve README final blank line --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9a8e0c64..351a3647 100644 --- a/README.md +++ b/README.md @@ -336,3 +336,4 @@ feel free to open GitHub issues requesting for new features. Before opening a ne Having our packages in distros on time is something we highly desire. Let us know if there is anything we can do to facilitate your work as a packager. + From 58dd4ee1072b75ca05f49396eea311c53b9cc562 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:41:58 +0200 Subject: [PATCH 30/40] docs: document phase4 changes and keep English documentation --- CHANGES | 8 ++ README.md | 8 +- docs/phase4-handling.de.md | 201 ------------------------------------- 3 files changed, 12 insertions(+), 205 deletions(-) delete mode 100644 docs/phase4-handling.de.md diff --git a/CHANGES b/CHANGES index f6f2c7f7..1fff4347 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,11 @@ +Unreleased +---------- + + - feat: add opt-in Phase 4 safe/strict handling and optional JSON event + logging, with off as the default mode. Response body MIME selection and + inspection limits remain controlled by libModSecurity. + [PR #377 - @Easton97-Jens] + v1.0.4 - 2025-May-21 -------------------- diff --git a/README.md b/README.md index 351a3647..dc8c7b4e 100644 --- a/README.md +++ b/README.md @@ -236,10 +236,10 @@ explicitly when migrating, and move MIME selection into ModSecurity rules. The latest limit-ownership update retains those modes and their late policies. Existing `modsecurity_phase4_body_limit` values no longer limit inspection; migrate that policy to `SecResponseBodyLimit` and `SecResponseBodyLimitAction`. -See [English](docs/phase4-handling.en.md) or -[Deutsch](docs/phase4-handling.de.md) for the behavior, limits, logging schema, -and examples. The documentation describes the source implementation; test -results from the source repository are not validation of this migration. +See the [Phase 4 documentation](docs/phase4-handling.en.md) for the behavior, +limits, logging schema, and examples. The documentation describes the source +implementation; test results from the source repository are not validation of +this migration. modsecurity_use_error_log ----------- diff --git a/docs/phase4-handling.de.md b/docs/phase4-handling.de.md deleted file mode 100644 index ca296196..00000000 --- a/docs/phase4-handling.de.md +++ /dev/null @@ -1,201 +0,0 @@ -# ModSecurity-nginx: Phase-4-Handling (Deutsch) - -## 1) Geltungsbereich und Herkunft - -Phase 4 prüft den Response-Body. nginx kann bereits Header oder Body-Bytes -gesendet haben, wenn eine Phase-4-Regel einen Deny-Status oder Redirect -anfordert. Eine späte Intervention kann den bereits beim Client angekommenen -HTTP-Status nicht zuverlässig ersetzen. - -Dieser Branch übernimmt die relevante nginx-Implementierung aus -[Easton97-Jens/ModSecurity-conector, Commit b0f3bdab429717b5b0311c30c5b4d1153c672ac0](https://github.com/Easton97-Jens/ModSecurity-conector/tree/b0f3bdab429717b5b0311c30c5b4d1153c672ac0/connectors/nginx/src). -Die Aktualisierung der Zuständigkeit für Response-Limits folgt -[Commit 820b6975495bdf0f90aca67eee86e27a3b7d329b](https://github.com/Easton97-Jens/ModSecurity-conector/blob/820b6975495bdf0f90aca67eee86e27a3b7d329b/docs/phase4-mode-budget.de.md). -Diese Änderungen werden an dieses eigenständige Modul angepasst; dessen -Phase-4-JSON-Lines-Schema bleibt erhalten. Die Multi-Connector-Laufzeit und -Laufzeitnachweise des Quellrepositories werden nicht übernommen. - -Dieses Dokument beschreibt das Verhalten im Quellcode. Es behauptet nicht, -dass Laufzeit-, Protokoll- oder Integrationstests für diese Migration bestanden -wurden. - -## 2) Konfiguration und inkompatible Änderungen - -Alle drei Connector-Direktiven gelten in `http`-, `server`- und -`location`-Kontexten und werden vom umgebenden Kontext geerbt. - -| Direktive | Werte | Standard | -| --- | --- | --- | -| `modsecurity_phase4_mode` | `off`, `safe`, `strict` | `off` | -| `modsecurity_phase4_body_limit` | Veralteter, ignorierter Kompatibilitätswert; positive Bytes oder nginx-Größe, z. B. `256k`, `2m` | `1m` (1 MiB), ignoriert | -| `modsecurity_phase4_log` | Dateipfad für Phase-4-JSON-Lines-Ereignisse | Kein dediziertes Log | - -Bei der Migration älterer Phase-4-Konfigurationen gilt: - -- `minimal` ist nicht mehr gültig und kein Alias für `off`. -- Der Standard ist `off`; ältere Konfigurationen konnten sich auf `safe` - verlassen. Für die zusätzliche Connector-Behandlung muss `safe` oder - `strict` explizit gesetzt werden. -- `modsecurity_phase4_content_types_file` entfällt und verursacht einen - nginx-Konfigurationsfehler. Die MIME-Auswahl gehört jetzt in ModSecurity-Regeln. -- `modsecurity_phase4_body_limit` ist jetzt veraltet und wird in allen gültigen - Modi ignoriert. Parser für positive Werte, Vererbung und historischer - Standardwert von 1 MiB bleiben kompatibel; null ist weiterhin ungültig. - Inspection-Limits auf `SecResponseBodyLimit` und `SecResponseBodyLimitAction` - umstellen. - -Die migrierte Konfiguration vor dem nginx-Reload mit `nginx -t` in der -Zielumgebung prüfen. - -## 3) Modusverhalten und Header-Zeitpunkt - -`off` deaktiviert die zusätzliche Phase-4-Interventionsbehandlung des Connectors. -Es **deaktiviert weder ModSecurity** noch die -Response-Body-Prüfung oder Phase-4-Regeln. Die native Interventionsbehandlung -bleibt aktiv. Tritt eine native Intervention nach dem Header-Versand auf, -kann die Finalisierung einen Transportfehler verursachen; `off` garantiert -keine vollständige Auslieferung jeder Antwort. - -Für die zusätzliche Behandlung in `safe` und `strict` gilt: - -| Header-Zustand | `safe` | `strict` | -| --- | --- | --- | -| Header noch nicht gesendet | Status/Redirect über den normalen Interventionspfad anwenden; als `deny_status` geloggt | Gleiches Verhalten | -| Header bereits gesendet | Intervention als `log_only` protokollieren und fortfahren | `connection_abort` protokollieren und die Antwort abbrechen | - -`safe` degradiert nur späte **Interventionen**. Fehler der Engine-API, -ungültige Buffer, fehlgeschlagene Dateizugriffe, Speicherallokationsfehler, -Zählerüberläufe und Fehler bei der finalen Verarbeitung bleiben Fehler. -Sie werden -nicht in erfolgreiche Weiterleitung umgewandelt. - -`strict` kann eine abgeschnittene Antwort oder einen Transportfehler beim -Client/Proxy auslösen. Nach dem Header-Versand garantiert es keinen sauberen -403-, 401-, 301- oder 302-Status. Bereits weitergeleitete Body-Bytes lassen -sich nicht zurückholen. - -## 4) Engine-Inspection-Limits und Streaming - -libModSecurity bestimmt das Byte-Limit der WAF-Response-Inspection über -`SecResponseBodyLimit` und `SecResponseBodyLimitAction` in jedem Phase-4-Modus. -Der Connector ergänzt keine kumulierte Inspection-Grenze für `safe`, `strict` -oder `off` und weist eine Antwort nicht allein deshalb ab, weil sie -`modsecurity_phase4_body_limit` überschreitet. - -Beispielsweise diese Engine-Policy in den ModSecurity-Regeln konfigurieren: - -```apache -SecResponseBodyLimit 1048576 -SecResponseBodyLimitAction ProcessPartial -``` - -1 MiB ist hier ein expliziter Beispielwert, kein neuer Engine-Standard. -`ProcessPartial` wählt die Prüfung des Anteils innerhalb des Engine-Limits; -`Reject` ist die alternative Engine-Aktion, wenn Abweisung erforderlich ist. -Der Modus überschreibt diese Engine-Policy nicht. Daraus entstehende späte -Interventionen folgen weiterhin dem ausgewählten Connector-Modus. - -Die veraltete Connector-Einstellung bleibt parsebar und wird vererbt, ihr Wert -wird aber in keinem gültigen Modus durchgesetzt. Erfolgreiches Laden der -Konfiguration belegt daher nicht die alte Connector-Inspection-Grenze. -Diese Policy auf die oben genannten Engine-Einstellungen umstellen. - -Die geprüfte kumulierte Bytezählung weist Überläufe über `SIZE_MAX` weiterhin -in allen Modi ab. Echte Verarbeitungs- und Datei-Read-Fehler bleiben Fehler; -nichtfatale Engine-Aufnahme mit `ProcessPartial` ist kein Connector-Fehler. - -Der Connector puffert Antworten nicht global und ordnet nginx-Body-Ketten nicht -um. Speicherbuffer werden direkt geprüft. Rein dateibasierte Buffer werden -über einen auf 32 KiB begrenzten Arbeitsbuffer gelesen, ohne die ganze Datei -in den Speicher zu laden oder die ausgehende Originalkette zu ersetzen. Echte -Verarbeitungs- oder I/O-Fehler stoppen die betroffene Antwort. - -Der Response-Body wird pro Transaktion genau einmal finalisiert: Beim -Hauptrequest wird `last_buf` ausgewertet, beim Subrequest `last_in_chain`. -Wiederholte Filteraufrufe nach der Finalisierung führen Phase 4 nicht erneut -aus. - -## 5) MIME-Auswahl durch ModSecurity - -Die Engine entscheidet über die Response-Body-Prüfung anhand von -`SecResponseBodyAccess`, `SecResponseBodyMimeType` und -`SecResponseBodyMimeTypesClear`. Es gibt keine Connector-Content-Type-Liste -und keine MIME-basierte Degradierung durch den Connector. - -Beispielsweise die MIME-Liste in einem Regel-Ladevorgang zurücksetzen und -die ausgewählten Typen in einem getrennten Ladevorgang hinzufügen: - -```nginx -modsecurity_rules 'SecResponseBodyMimeTypesClear'; -modsecurity_rules ' - SecResponseBodyAccess On - SecResponseBodyMimeType text/html text/plain application/json -'; -``` - -Das Zurücksetzen erfolgt getrennt, weil die -[Merge-Implementierung von libModSecurity](https://raw.githubusercontent.com/owasp-modsecurity/ModSecurity/v3/master/headers/modsecurity/rules_set_properties.h) -MIME-Werte entfernen kann, die im selben Regel-Ladevorgang hinzugefügt werden. - -Das eigenständige [Engine-MIME-Beispiel](examples/phase4-engine-mime.conf) -enthält MIME-Ergänzungen, Response-Body-Prüfung und eine explizite -Engine-Limit-Policy. Es ist eine -ModSecurity-Regeldatei, kein nginx-Include. Um die MIME-Liste der Engine durch -diese Datei zu ersetzen, zuerst das Zurücksetzen und dann die Datei zusätzlich -zu den anderen Regeln laden: - -```nginx -modsecurity_rules 'SecResponseBodyMimeTypesClear'; -modsecurity_rules_file /etc/modsecurity/phase4-engine-mime.conf; -``` - -Die vollständigen nginx-Beispiele unten konfigurieren die MIME-Auswahl inline -mit derselben Reihenfolge getrennter Ladevorgänge. - -MIME-Auswahl und Inspection-Limits der Engine gelten in jedem Phase-4-Modus; -es gibt kein zusätzliches Connector-Inspection-Budget. - -## 6) Logging-Format und Sicherheitsgrenze - -`modsecurity_phase4_log` erhält das bisherige JSON-Lines-Schema der -Interventionsereignisse: - -- `event` (`phase4_intervention`), `uri`, `method`; -- `response_status`, `waf_status`, `content_type`, `header_sent`, `mode`; -- `wanted_action`, `actual_action`, `reason`; -- `intervention`, `rule_id`. - -Die Werte von `actual_action` bleiben `deny_status`, `log_only` und -`connection_abort`; `mode` nutzt die aktuellen Modusnamen. Die -Interventionsnachricht wird maskiert, statt aus der Engine kopiert zu werden, -und das Ereignis enthält keine hinzugefügten Response-Body-Inhalte. Das -nginx-Error-Log kann weitere Diagnosen enthalten. - -`off` nutzt die native Interventionsbehandlung und erzeugt diese dedizierten -Policy-Ereignisse nicht. Ein dediziertes Interventionsereignis ist keine -vollständige Erfassung aller Engine-, Buffer- oder I/O-Fehler. - -## 7) Konfigurationsbeispiele und Verifikation - -- [off](examples/phase4-off.conf): native Interventionsbehandlung mit expliziter - Engine-Inspection-Policy. -- [safe](examples/phase4-safe.conf): explizite späte `log_only`-Behandlung. -- [strict](examples/phase4-strict.conf): später Verbindungsabbruch. - -- [Engine-MIME-Auswahl](examples/phase4-engine-mime.conf): - ModSecurity-Konfiguration für den Response-Body. - -Alle drei Beispiele konfigurieren dasselbe Engine-Limit von 1 MiB mit -`ProcessPartial`; diese Limit-Policy ist vom Connector-Modus unabhängig. - -Die nginx-Beispiele verwenden `location /` und einen Beispiel-Upstream unter -`127.0.0.1:8081`. Listen-Adresse, Upstream, Logpfad und Regeln an die -Zielumgebung anpassen. Die Regel mit `sensitive-marker` dient zur Illustration. - -Repository-Testendpunkte wie `/phase4` sind Testfixtures, keine erforderlichen -Produktionspfade. Laufzeitprüfungen sollten späte Deny-/Redirect-Interventionen, -HTTP/1.1 und HTTP/2, dateibasierte Antworten, Subrequests, wiederholte -Finalisierung, Engine-Limit-Grenzen, Antworten oberhalb der ignorierten alten -Einstellung und Fehlerpfade abdecken. Ergebnisse aus einem -anderen Repository oder Build belegen diese Eigenschaften des eingesetzten -Moduls nicht. From 85604d5853a0c5b4ad74b317f7aeb8add38f166e Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:59:05 +0200 Subject: [PATCH 31/40] fix: move phase4 helper CI and use locale-independent log normalization --- .github/workflows/test.yml | 14 -------------- .github/workflows/test_new.yml | 25 +++++++++++++++++++++++++ src/ngx_http_modsecurity_body_filter.c | 10 ++++++++-- src/ngx_http_modsecurity_module.c | 1 - 4 files changed, 33 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4a19e956..d7e3ac8e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,20 +8,6 @@ on: permissions: {} jobs: - phase4-unit-tests: - permissions: - contents: read - runs-on: ubuntu-22.04 - strategy: - matrix: - compiler: [gcc, clang] - env: - CC: "/usr/bin/${{ matrix.compiler }}" - steps: - - uses: actions/checkout@v4 - - name: Run standalone connector helper regressions - run: python3 -m unittest discover -s tests -p 'test_*runtime.py' -v - build-linux: permissions: contents: read diff --git a/.github/workflows/test_new.yml b/.github/workflows/test_new.yml index cd7e7ac2..d442ab1b 100644 --- a/.github/workflows/test_new.yml +++ b/.github/workflows/test_new.yml @@ -10,6 +10,22 @@ on: permissions: {} jobs: + phase4-unit-tests: + permissions: + contents: read + runs-on: ubuntu-22.04 + strategy: + matrix: + compiler: [gcc, clang] + env: + CC: "/usr/bin/${{ matrix.compiler }}" + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Run standalone connector helper regressions + run: python3 -m unittest discover -s tests -p 'test_*runtime.py' -v + build-linux: name: linux-${{ matrix.compiler }}-nginx-${{ matrix.nginx_version }} permissions: @@ -34,6 +50,7 @@ jobs: - name: Get libModSecurity source uses: actions/checkout@v6 with: + persist-credentials: false repository: owasp-modsecurity/ModSecurity path: ModSecurity submodules: recursive @@ -47,6 +64,7 @@ jobs: sudo make install - uses: actions/checkout@v7 with: + persist-credentials: false path: ModSecurity-nginx fetch-depth: 1 - name: Resolve latest Nginx version @@ -57,6 +75,7 @@ jobs: - name: Get Nginx source uses: actions/checkout@v7 with: + persist-credentials: false repository: nginx/nginx ref: ${{ steps.nginx_ref.outputs.ref }} path: nginx @@ -75,6 +94,7 @@ jobs: - name: Get Nginx tests uses: actions/checkout@v7 with: + persist-credentials: false repository: nginx/nginx-tests path: nginx/test fetch-depth: 0 @@ -290,6 +310,7 @@ jobs: - name: Get ModSecurity-nginx workflow scripts uses: actions/checkout@v7 with: + persist-credentials: false path: ModSecurity-nginx fetch-depth: 1 - name: Resolve latest Nginx version @@ -300,6 +321,7 @@ jobs: - name: Get Nginx source uses: actions/checkout@v7 with: + persist-credentials: false repository: nginx/nginx ref: ${{ steps.nginx_ref.outputs.ref }} path: nginx @@ -318,6 +340,7 @@ jobs: - name: Get Nginx tests uses: actions/checkout@v7 with: + persist-credentials: false repository: nginx/nginx-tests path: nginx/test fetch-depth: 0 @@ -341,6 +364,7 @@ jobs: - name: Get libModSecurity source uses: actions/checkout@v7 with: + persist-credentials: false repository: owasp-modsecurity/ModSecurity submodules: recursive path: nginx/objs/lib/ModSecurity @@ -358,6 +382,7 @@ jobs: - name: Get ModSecurity-nginx source code uses: actions/checkout@v7 with: + persist-credentials: false path: nginx/objs/lib/ModSecurity-nginx - name: Copy ModSecurity-nginx tests to nginx/test working-directory: nginx/test diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index d446ee03..a8bff08e 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -14,7 +14,6 @@ */ #include -#include #include #ifndef MODSECURITY_DDEBUG @@ -491,7 +490,14 @@ ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in) if (out.data == NULL || out.len == 0) return out; semi = (u_char *)ngx_strlchr(out.data, out.data + out.len, ';'); if (semi) out.len = semi - out.data; - while (out.len > 0 && isspace((unsigned char) out.data[out.len - 1])) out.len--; + while (out.len > 0) { + u_char c = out.data[out.len - 1]; + if (c != ' ' && c != '\t' && c != '\n' && + c != '\r' && c != '\f' && c != '\v') { + break; + } + out.len--; + } out.data = ngx_pnalloc(pool, out.len); if (out.data == NULL) { out.len = 0; return out; } for (i = 0; i < out.len; i++) out.data[i] = ngx_tolower(in.data[i]); diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index 03eb5f95..b3db37b5 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -22,7 +22,6 @@ #include "ngx_http_modsecurity_common.h" #include "stdio.h" -#include #include #include From ba8361b8a29d9a11a51c9d9bdd648d1efc69253a Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:23:48 +0200 Subject: [PATCH 32/40] docs: expand phase4 changelog --- CHANGES | 52 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 1fff4347..30688f68 100644 --- a/CHANGES +++ b/CHANGES @@ -1,9 +1,55 @@ Unreleased ---------- - - feat: add opt-in Phase 4 safe/strict handling and optional JSON event - logging, with off as the default mode. Response body MIME selection and - inspection limits remain controlled by libModSecurity. + - feat: add off (default), safe and strict Phase 4 intervention modes. + Safe and strict explicitly enable the additional connector policy; off + leaves ModSecurity response inspection and native interventions active. + [PR #377 - @Easton97-Jens] + - feat: handle late deny and redirect interventions explicitly. Safe logs + and continues after headers are sent; strict terminates the response. + A late intervention cannot replace an HTTP status or body already sent. + [PR #377 - @Easton97-Jens] + - feat: add optional JSON-lines events with requested and actual actions, + response and WAF status, mode, reason, content type and rule ID. Redact + engine intervention messages and omit response-body bytes from the event. + [PR #377 - @Easton97-Jens] + - fix: inherit Phase 4 configuration and log files from http/server blocks + into nested locations, with validation of mode and size arguments. + [PR #377 - @Easton97-Jens] + - fix: leave response MIME selection and inspection limits to libModSecurity + through SecResponseBodyAccess, SecResponseBodyMimeType, + SecResponseBodyMimeTypesClear, SecResponseBodyLimit and + SecResponseBodyLimitAction. Earlier revisions of this PR used a connector + MIME-file directive and minimal mode; these are no longer supported. + Retain modsecurity_phase4_body_limit as a deprecated, + ignored compatibility setting with positive-value validation. + [PR #377 - @Easton97-Jens] + - fix: inspect memory and file-backed response buffers, reading file data + in bounded 32 KiB chunks while preserving the outgoing chain. Check byte + accounting, engine results, allocations and file reads; genuine errors + remain failures in every mode, including safe. + [PR #377 - @Easton97-Jens] + - fix: finalize response-body processing once per transaction and respect + main-request/subrequest ownership and end-of-stream markers. Correct + response-header error handling, redirect ownership, cleanup and internal + redirect recovery. + [PR #377 - @Easton97-Jens] + - fix: distinguish redirect and deny actions in events, escape JSON fields, + detect log-write failures, pass the correct NGINX integer type for WAF + status and normalize logged content types with fixed ASCII whitespace. + [PR #377 - @Easton97-Jens] + - test: extend regressions for modes, log inheritance, engine MIME selection, + Reject/ProcessPartial limits, large responses, file buffers, overflow, + subrequests, redirects, processing failures and proxy/HTTP/2 fixtures. + [PR #377 - @Easton97-Jens] + - ci: run the GCC/Clang Phase 4 helper tests in test_new.yml and disable + credential persistence for all ten checkouts in that workflow. Keep the + native Linux/Windows matrix there and remove the duplicate legacy + Windows job from test.yml. + [PR #377 - @Easton97-Jens] + - docs: document Phase 4 timing limits, mode behavior, logging and engine + policy in English, with off/safe/strict and MIME-selection examples. + Remove the German translation and correct the README directive count. [PR #377 - @Easton97-Jens] v1.0.4 - 2025-May-21 From 468ebd22ae59a5e58aba9da351c8ee315a187c69 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:28:50 +0200 Subject: [PATCH 33/40] docs: clarify safe mode logging failures --- CHANGES | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 30688f68..e20916ea 100644 --- a/CHANGES +++ b/CHANGES @@ -5,9 +5,10 @@ Unreleased Safe and strict explicitly enable the additional connector policy; off leaves ModSecurity response inspection and native interventions active. [PR #377 - @Easton97-Jens] - - feat: handle late deny and redirect interventions explicitly. Safe logs - and continues after headers are sent; strict terminates the response. - A late intervention cannot replace an HTTP status or body already sent. + - feat: handle late deny and redirect interventions explicitly. After + headers are sent, safe logs and continues when event logging succeeds; + strict terminates the response. A late intervention cannot replace an HTTP + status or body already sent. [PR #377 - @Easton97-Jens] - feat: add optional JSON-lines events with requested and actual actions, response and WAF status, mode, reason, content type and rule ID. Redact From 96bd3f76218b6aafc24f780dade2a93167c35e6b Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:53:23 +0100 Subject: [PATCH 34/40] Port native intervention and lifecycle fixes from connector --- CHANGES | 27 +- docs/phase4-handling.en.md | 42 +- src/ngx_http_modsecurity_access.c | 93 ++- src/ngx_http_modsecurity_body_filter.c | 65 +- src/ngx_http_modsecurity_common.h | 5 + src/ngx_http_modsecurity_log.c | 39 +- src/ngx_http_modsecurity_module.c | 164 ++++- tests/README.md | 13 +- tests/modsecurity-config-custom-error-page.t | 45 +- tests/modsecurity-phase4-engine-limits.t | 51 +- tests/modsecurity-phase4-modes.t | 22 +- tests/test_intervention_runtime.py | 140 +++- tests/test_lifecycle_runtime.py | 639 +++++++++++++++++++ tests/test_phase4_runtime.py | 153 ++++- 14 files changed, 1342 insertions(+), 156 deletions(-) create mode 100644 tests/test_lifecycle_runtime.py diff --git a/CHANGES b/CHANGES index e20916ea..addeff15 100644 --- a/CHANGES +++ b/CHANGES @@ -39,6 +39,30 @@ Unreleased detect log-write failures, pass the correct NGINX integer type for WAF status and normalize logged content types with fixed ASCII whitespace. [PR #377 - @Easton97-Jens] + - fix: validate native intervention and body-append return values as closed + 0/1 contracts, respect nondisruptive results, release owned outputs on + every path and propagate early audit-log failures. + [PR #377 - @Easton97-Jens] + - fix: retain native response-limit Reject in every mode, including safe. + Emit 403 before headers or stop the committed response; never downgrade + the engine limit into a rule-policy event. Collect file interventions + after each chunk while preserving ProcessPartial and the outgoing chain. + [PR #377 - @Easton97-Jens] + - fix: keep response failures terminal across data and NULL-input filter + re-entry, permitting only one synchronous core error response. Accept + valid empty memory/EOS buffers without repeating phase completion. + [PR #377 - @Easton97-Jens] + - fix: preserve request-phase deny/redirect decisions across error_page + routing and start a fresh transaction for an internal target location. + Check request-phase, file-ingestion and append results independently. + [PR #377 - @Easton97-Jens] + - fix: attempt native audit logging once and retain failed native results + across re-entry without attempting another client response. + [PR #377 - @Easton97-Jens] + - fix: return NULL for transaction/context creation failures, release + partially created native owners, make cleanup idempotent and terminate + length-delimited custom transaction IDs before passing them to the SDK. + [PR #377 - @Easton97-Jens] - test: extend regressions for modes, log inheritance, engine MIME selection, Reject/ProcessPartial limits, large responses, file buffers, overflow, subrequests, redirects, processing failures and proxy/HTTP/2 fixtures. @@ -46,7 +70,8 @@ Unreleased - ci: run the GCC/Clang Phase 4 helper tests in test_new.yml and disable credential persistence for all ten checkouts in that workflow. Keep the native Linux/Windows matrix there and remove the duplicate legacy - Windows job from test.yml. + Windows job from test.yml. Integrate upstream dependency-download retry + fixes without restoring the removed job or credential persistence. [PR #377 - @Easton97-Jens] - docs: document Phase 4 timing limits, mode behavior, logging and engine policy in English, with off/safe/strict and MIME-selection examples. diff --git a/docs/phase4-handling.en.md b/docs/phase4-handling.en.md index ac6253f9..f1719de5 100644 --- a/docs/phase4-handling.en.md +++ b/docs/phase4-handling.en.md @@ -11,6 +11,8 @@ This branch adopts the relevant nginx implementation from [Easton97-Jens/ModSecurity-conector, commit b0f3bdab429717b5b0311c30c5b4d1153c672ac0](https://github.com/Easton97-Jens/ModSecurity-conector/tree/b0f3bdab429717b5b0311c30c5b4d1153c672ac0/connectors/nginx/src). The response-limit ownership update follows [commit 820b6975495bdf0f90aca67eee86e27a3b7d329b](https://github.com/Easton97-Jens/ModSecurity-conector/blob/820b6975495bdf0f90aca67eee86e27a3b7d329b/docs/phase4-mode-budget.md). +Native-result, response completion, and lifecycle corrections follow +[commit f5523da040bd6325b369269ada074be6ba069c4f](https://github.com/Easton97-Jens/ModSecurity-conector/tree/f5523da040bd6325b369269ada074be6ba069c4f/connectors/nginx/src). These changes are adapted to this standalone module and preserve its phase 4 JSON-lines schema. The source repository's multi-connector runtime and run evidence are not imported. @@ -49,20 +51,26 @@ environment before reloading nginx. `off` disables the connector's additional phase 4 intervention policy. It **does not disable ModSecurity**, response-body inspection, or phase 4 rules. Native intervention handling remains active. If a native intervention -occurs after headers have been committed, response finalization can fail the -transport; `off` is not a promise to deliver every response. +occurs after headers have been committed, the affected transport is failed +without trying to send another HTTP status; `off` is not a promise to deliver +every response or to reproduce every pre-PR intervention control-flow decision. For the additional policy in `safe` and `strict`: | Header state | `safe` | `strict` | | --- | --- | --- | | Headers not sent | Apply the intervention's status/redirect through the normal path; logged as `deny_status` | Same | -| Headers already sent | Log the intervention as `log_only` and continue | Log `connection_abort` and terminate the response | +| Headers already sent | Log the rule intervention as `log_only` and continue if logging succeeds | Log `connection_abort` and terminate the response | -`safe` only downgrades late **interventions**. An engine API failure, invalid +`safe` only downgrades late **rule interventions**. An engine API failure, invalid buffer, failed file read, allocation failure, counter overflow, or final processing failure remains a failure. It is not converted into successful -forwarding. +forwarding. Native response-limit Reject is not a rule-policy downgrade. + +Once a response has failed, later data callbacks and NULL-input flushes cannot +resume forwarding. Before headers are sent, nginx may render its synchronous +core error response once; subsequent callbacks retain the failure. Ordinary +successful downstream `NGX_AGAIN` handling remains unchanged. `strict` can produce a truncated response or a client/proxy transport error. It cannot guarantee a clean 403, 401, 301, or 302 after headers have been sent. @@ -86,8 +94,11 @@ SecResponseBodyLimitAction ProcessPartial The 1 MiB value is an explicit example setting, not a new engine default. `ProcessPartial` selects inspection of the portion within the engine limit; `Reject` is the alternative engine action when rejection is required. Mode -selection does not override that engine policy. Any resulting late intervention -still follows the selected connector mode. +selection does not override that engine policy. Native Reject is recognized +by its exact rule-free engine signature while processing a response body. It +produces 403 before headers are sent, or fails the transport after commitment, +in all modes including `safe`. It does not produce a `phase4_intervention` +rule-policy event; a fixed diagnostic is written to the nginx error log. The deprecated connector setting remains parseable and inherited, but its value has no enforcement effect in any valid mode. A successful configuration @@ -101,12 +112,17 @@ nonfatal engine `ProcessPartial` ingestion is not a connector failure. The connector does not globally buffer responses or reorder nginx body chains. Memory buffers are inspected directly. File-only buffers are read through a bounded 32 KiB scratch buffer, without loading the entire file into -memory or replacing the original outgoing chain. Genuine processing or I/O -failures stop the affected response. +memory or replacing the original outgoing chain. Pending interventions are +checked after every file chunk so Reject stops further reads immediately. +Genuine processing or I/O failures stop the affected response. The response body is finalized once per transaction: a main request uses `last_buf`; a subrequest uses `last_in_chain`. Repeated filter calls after -finalization do not run phase 4 again. +finalization do not run phase 4 again. Empty memory/EOS buffers with both data +pointers NULL are valid; a missing single pointer or inverted range remains an +error. Native byte appends accept only 0/1 (`ProcessPartial` may return 0), +phase evaluation succeeds only with 1, and native intervention collection +accepts only 0/1 and honors the disruptive flag. ## 5) MIME selection belongs to ModSecurity @@ -165,6 +181,12 @@ diagnostics. policy events. A dedicated intervention event is not a complete inventory of all engine, buffer, or I/O errors. +Native audit logging is attempted once, including re-entry, and its return +value is checked. A failed audit attempt is retained as a failure without +trying to generate a new HTTP response. Request inspection also preserves +deny/redirect decisions across `error_page` routing; a new internal target +location starts its own transaction rather than reusing processed state. + ## 7) Configuration examples and verification - [off](examples/phase4-off.conf): native intervention handling with explicit diff --git a/src/ngx_http_modsecurity_access.c b/src/ngx_http_modsecurity_access.c index 47175e45..6c2c1f4b 100644 --- a/src/ngx_http_modsecurity_access.c +++ b/src/ngx_http_modsecurity_access.c @@ -22,6 +22,20 @@ #include "ngx_http_modsecurity_common.h" + +static ngx_int_t +ngx_http_modsecurity_request_failure(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) +{ + ctx->intervention_failed = 1; + ctx->intervention_triggered = 1; + if (r->header_sent) { + r->connection->error = 1; + return NGX_ERROR; + } + return NGX_HTTP_INTERNAL_SERVER_ERROR; +} + void ngx_http_modsecurity_request_read(ngx_http_request_t *r) { @@ -67,9 +81,17 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) dd("catching a new _access_ phase handler"); - ctx = ngx_http_modsecurity_get_module_ctx(r); + /* Internal redirects clear the current location's module context. + * Cleanup recovery belongs to finalization, not access initialization. */ + ctx = ngx_http_get_module_ctx(r, ngx_http_modsecurity_module); + if (ctx != NULL && ctx->r != r) { + ctx = NULL; + } + if (ctx != NULL && ctx->intervention_failed) { + return ngx_http_modsecurity_request_failure(r, ctx); + } - dd("recovering ctx: %p", ctx); + dd("current location ctx: %p", ctx); if (ctx == NULL) { @@ -164,8 +186,10 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) client_addr, client_port, server_addr, server_port); ngx_http_modsecurity_pcre_malloc_done(old_pool); - if (ret != 1){ - dd("Was not able to extract connection information."); + if (ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: connection phase processing failed"); + return ngx_http_modsecurity_request_failure(r, ctx); } /** * @@ -178,6 +202,9 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) */ dd("Processing intervention with the connection information filled in"); ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 1); + if (ret < 0) { + return ngx_http_modsecurity_request_failure(r, ctx); + } if (ret > 0) { ctx->intervention_triggered = 1; return ret; @@ -222,11 +249,19 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) return NGX_HTTP_INTERNAL_SERVER_ERROR; } old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_uri(ctx->modsec_transaction, n_uri, n_method, http_version); + ret = msc_process_uri(ctx->modsec_transaction, n_uri, n_method, http_version); ngx_http_modsecurity_pcre_malloc_done(old_pool); + if (ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: URI phase processing failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } dd("Processing intervention with the transaction information filled in (uri, method and version)"); ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 1); + if (ret < 0) { + return ngx_http_modsecurity_request_failure(r, ctx); + } if (ret > 0) { ctx->intervention_triggered = 1; return ret; @@ -272,13 +307,18 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) */ old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_request_headers(ctx->modsec_transaction); + ret = msc_process_request_headers(ctx->modsec_transaction); ngx_http_modsecurity_pcre_malloc_done(old_pool); + if (ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request headers phase processing failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } dd("Processing intervention with the request headers information filled in"); ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 1); - if (r->error_page) { - return NGX_DECLINED; - } + if (ret < 0) { + return ngx_http_modsecurity_request_failure(r, ctx); + } if (ret > 0) { ctx->intervention_triggered = 1; return ret; @@ -399,7 +439,12 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) */ dd("request body inspection: file -- %s", file_name); - msc_request_body_from_file(ctx->modsec_transaction, file_name); + ret = msc_request_body_from_file(ctx->modsec_transaction, file_name); + if (ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request body file processing failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } already_inspected = 1; } else { @@ -410,8 +455,14 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) { u_char *data = chain->buf->pos; - msc_append_request_body(ctx->modsec_transaction, data, + ret = msc_append_request_body(ctx->modsec_transaction, data, chain->buf->last - data); + /* Direct append zero is the engine's ProcessPartial result. */ + if (ret != 0 && ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request body append failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } if (chain->buf->last_buf) { break; @@ -426,7 +477,11 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) * */ ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); + if (ret < 0) { + return ngx_http_modsecurity_request_failure(r, ctx); + } if (ret > 0) { + ctx->intervention_triggered = 1; return ret; } } @@ -441,15 +496,21 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) /* XXX: once more -- is body can be modified ? content-length need to be adjusted ? */ old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_request_body(ctx->modsec_transaction); - ctx->request_body_processed = 1; + ret = msc_process_request_body(ctx->modsec_transaction); ngx_http_modsecurity_pcre_malloc_done(old_pool); + if (ret != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request body phase processing failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } + ctx->request_body_processed = 1; ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); - if (r->error_page) { - return NGX_DECLINED; - } + if (ret < 0) { + return ngx_http_modsecurity_request_failure(r, ctx); + } if (ret > 0) { + ctx->intervention_triggered = 1; return ret; } } diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index a8bff08e..bea20963 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -27,6 +27,7 @@ #define NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK 32768U static ngx_http_output_body_filter_pt ngx_http_next_body_filter; +static ngx_int_t ngx_http_modsecurity_process_response_intervention(ngx_http_request_t *r, ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf); static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason); static ngx_int_t ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf); static void ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst); @@ -44,17 +45,44 @@ ngx_http_modsecurity_body_filter_init(void) return NGX_OK; } +static ngx_int_t +ngx_http_modsecurity_phase4_error_gate(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) +{ + if (!ctx->phase4_failed) { + return NGX_OK; + } + /* Only the synchronous core-generated error body may pass the gate. + * A subsequent flush or data callback cannot resume the failed chain. */ + if (ctx->phase4_error_emitting && r->filter_finalize) { + return NGX_DECLINED; + } + r->connection->error = 1; + return NGX_ERROR; +} + static ngx_int_t ngx_http_modsecurity_response_body_failure(ngx_http_request_t *r, ngx_http_modsecurity_ctx_t *ctx) { + ngx_int_t ret; + + if (ctx->phase4_failed) { + r->connection->error = 1; + return NGX_ERROR; + } + ctx->phase4_failed = 1; ctx->intervention_triggered = 1; if (r->header_sent) { r->connection->error = 1; return NGX_ERROR; } - return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, - NGX_HTTP_INTERNAL_SERVER_ERROR); + ctx->phase4_error_emitting = 1; + ret = ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, + ctx->intervention_body_limit ? NGX_HTTP_FORBIDDEN : + NGX_HTTP_INTERNAL_SERVER_ERROR); + ctx->phase4_error_emitting = 0; + return ret; } /* libModSecurity owns response inspection limits in every Phase-4 mode. @@ -99,13 +127,16 @@ static ngx_int_t ngx_http_modsecurity_append_response_body_chunk( ngx_http_modsecurity_ctx_t *ctx, u_char *data, size_t bytes) { + int ret; + if (bytes == 0U) { return NGX_OK; } if (data == NULL || ctx->response_body_bytes_inspected > SIZE_MAX - bytes) { return NGX_ERROR; } - if (msc_append_response_body(ctx->modsec_transaction, data, bytes) < 0) { + ret = msc_append_response_body(ctx->modsec_transaction, data, bytes); + if (ret != 0 && ret != 1) { return NGX_ERROR; } ctx->response_body_bytes_inspected += bytes; @@ -137,6 +168,7 @@ ngx_http_modsecurity_append_file_response_body(ngx_http_request_t *r, size_t chunk; off_t file_offset; ssize_t read_count; + ngx_int_t ret; if (buffer == NULL || buffer->file_pos < 0 || buffer->file_last < buffer->file_pos) { @@ -178,6 +210,12 @@ ngx_http_modsecurity_append_file_response_body(ngx_http_request_t *r, ctx->phase4_file_scratch, chunk) != NGX_OK) { return NGX_ERROR; } + /* Native Reject may return a successful append with a pending + * intervention. Collect it before reading the next file chunk. */ + ret = ngx_http_modsecurity_process_response_intervention(r, ctx, mcf); + if (ret != NGX_OK || ctx->intervention_triggered) { + return ret; + } file_offset += (off_t)chunk; remaining -= chunk; } @@ -194,6 +232,9 @@ ngx_http_modsecurity_append_response_body_buffer(ngx_http_request_t *r, } /* Buffers backed by both memory and a file describe the same bytes. */ if (ngx_buf_in_memory(buffer)) { + if (buffer->pos == NULL && buffer->last == NULL) { + return NGX_OK; + } if (buffer->pos == NULL || buffer->last == NULL || buffer->last < buffer->pos) { return NGX_ERROR; @@ -223,6 +264,9 @@ ngx_http_modsecurity_process_response_intervention(ngx_http_request_t *r, return ngx_http_modsecurity_response_body_failure(r, ctx); } if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_OFF) { + if (r->header_sent) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } ctx->intervention_triggered = 1; /* Retain the native intervention failure path in off. */ return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, @@ -325,13 +369,17 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) ngx_http_modsecurity_conf_t *mcf; ngx_int_t ret; - if (in == NULL) { - return ngx_http_next_body_filter(r, in); - } ctx = ngx_http_modsecurity_get_module_ctx(r); if (ctx == NULL) { return ngx_http_next_body_filter(r, in); } + ret = ngx_http_modsecurity_phase4_error_gate(r, ctx); + if (ret != NGX_OK) { + return ret == NGX_DECLINED ? ngx_http_next_body_filter(r, in) : ret; + } + if (in == NULL) { + return ngx_http_next_body_filter(r, in); + } if (ctx->response_replaced) { ngx_http_modsecurity_discard_replaced_response_body(in); return ngx_http_next_body_filter(r, in); @@ -349,9 +397,13 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) return ret; } #endif + ctx->phase4_started = 1; for (chain = in; chain != NULL; chain = chain->next) { ret = ngx_http_modsecurity_append_response_body_buffer(r, ctx, mcf, chain->buf); + if (ctx->intervention_triggered) { + return ret; + } if (ret != NGX_OK) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ModSecurity: response body inspection append failed"); @@ -413,6 +465,7 @@ ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, } if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) { ctx->phase4_strict_abort = 1; + ctx->phase4_failed = 1; ctx->intervention_triggered = 1; r->connection->error = 1; ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, diff --git a/src/ngx_http_modsecurity_common.h b/src/ngx_http_modsecurity_common.h index ad64d327..1070c38e 100644 --- a/src/ngx_http_modsecurity_common.h +++ b/src/ngx_http_modsecurity_common.h @@ -103,15 +103,20 @@ typedef struct { unsigned body_requested:1; unsigned processed:1; unsigned logged:1; + unsigned native_logging_failed:1; unsigned intervention_triggered:1; unsigned request_body_processed:1; unsigned phase4_headers_checked:1; unsigned phase4_processed:1; unsigned phase4_strict_abort:1; + unsigned phase4_started:1; + unsigned phase4_failed:1; + unsigned phase4_error_emitting:1; unsigned response_body_seen:1; unsigned response_body_truncated:1; unsigned intervention_late:1; unsigned intervention_failed:1; + unsigned intervention_body_limit:1; unsigned response_replaced:1; unsigned intervention_redirect_location_installed:1; size_t response_body_bytes_seen; diff --git a/src/ngx_http_modsecurity_log.c b/src/ngx_http_modsecurity_log.c index 094685c5..d699d069 100644 --- a/src/ngx_http_modsecurity_log.c +++ b/src/ngx_http_modsecurity_log.c @@ -41,35 +41,36 @@ ngx_http_modsecurity_log_handler(ngx_http_request_t *r) { ngx_pool_t *old_pool; ngx_http_modsecurity_ctx_t *ctx; + int result; - dd("catching a new _log_ phase handler"); - - /* - if (r->method != NGX_HTTP_GET && - r->method != NGX_HTTP_POST && r->method != NGX_HTTP_HEAD) { - dd("ModSecurity is not ready to deal with anything different from " \ - "POST, GET or HEAD"); - return NGX_OK; + if (r == NULL || r->connection == NULL) { + return NGX_ERROR; } - */ ctx = ngx_http_modsecurity_get_module_ctx(r); - - dd("recovering ctx: %p", ctx); - if (ctx == NULL) { - dd("ModSecurity not enabled or error occurred"); return NGX_OK; } - if (ctx->logged) { - dd("already logged earlier"); - return NGX_OK; + return ctx->native_logging_failed ? NGX_ERROR : NGX_OK; } - dd("calling msc_process_logging for %p", ctx); + /* Claim the attempt before a native callback can re-enter this handler. */ + ctx->logged = 1; + ctx->native_logging_failed = 1; + if (ctx->modsec_transaction == NULL) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: native logging transaction is unavailable"); + return NGX_ERROR; + } old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - msc_process_logging(ctx->modsec_transaction); + result = msc_process_logging(ctx->modsec_transaction); ngx_http_modsecurity_pcre_malloc_done(old_pool); - + if (result != 1) { + /* The log phase cannot replace the response or retry the audit sink. */ + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: native logging phase processing failed"); + return NGX_ERROR; + } + ctx->native_logging_failed = 0; return NGX_OK; } diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index b3db37b5..efc24d4b 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -214,6 +214,85 @@ ngx_http_modsecurity_process_intervention_redirect(ngx_http_request_t *r, } +/* Native response-limit Reject is not a rule intervention and must never be + * downgraded by Safe. A rule log includes its formatted rule metadata, so + * matching the entire engine-owned message also excludes a same-msg rule. */ +static ngx_int_t +ngx_http_modsecurity_is_response_body_limit_rejection( + ngx_http_modsecurity_ctx_t *ctx, + const ModSecurityIntervention *intervention) +{ + static const char expected[] = + "Response body limit is marked to reject the request"; + + return ctx->phase4_started && intervention->disruptive == 1 && + intervention->status == NGX_HTTP_FORBIDDEN && + intervention->url == NULL && intervention->log != NULL && + strncmp(intervention->log, expected, sizeof(expected)) == 0; +} + +static ngx_int_t +ngx_http_modsecurity_intervention_failure_status(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) +{ + if (r->header_sent) { + return NGX_ERROR; + } + return ctx->intervention_body_limit ? NGX_HTTP_FORBIDDEN : + NGX_HTTP_INTERNAL_SERVER_ERROR; +} + +static ngx_int_t +ngx_http_modsecurity_collect_native_intervention(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, Transaction *transaction, + ModSecurityIntervention *intervention) +{ + int native_result; + + native_result = msc_intervention(transaction, intervention); + if (native_result != 0 && native_result != 1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: invalid native intervention result"); + return NGX_ERROR; + } + if (native_result == 0 || !intervention->disruptive) { + ctx->intervention_failed = 0; + return NGX_DECLINED; + } + if (ngx_http_modsecurity_is_response_body_limit_rejection(ctx, + intervention)) { + ctx->intervention_body_limit = 1; + ctx->last_intervention_status = NGX_HTTP_FORBIDDEN; + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: response body limit rejected the response"); + return NGX_ERROR; + } + /* Response rules run at EOS. Any other disruptive result while bytes + * are being appended is an engine failure, including an uncorrelated + * Reject whose native message allocation failed. */ + if (ctx->phase4_started && !ctx->phase4_processed) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: unexpected intervention during response append"); + return NGX_ERROR; + } + return NGX_OK; +} + +static ngx_int_t +ngx_http_modsecurity_defer_late_phase4_intervention(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf) +{ + if (!ctx->phase4_started || !ctx->phase4_processed || !r->header_sent || + mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_OFF) { + return 0; + } + /* Safe/Strict record a WAF decision, not an HTTP status replacement. + * Do not dispatch that late decision through native status/audit hooks. */ + ctx->intervention_late = 1; + ctx->intervention_failed = 0; + return 1; +} + int ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_request_t *r, ngx_int_t early_log) @@ -222,40 +301,48 @@ ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_modsecurity_ctx_t *ctx; ngx_http_modsecurity_conf_t *mcf; int result; + ngx_int_t native_result; ctx = ngx_http_modsecurity_get_module_ctx(r); mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); if (ctx != NULL) { ctx->intervention_late = 0; ctx->intervention_failed = 0; + ctx->intervention_body_limit = 0; ctx->intervention_redirect_location_installed = 0; + ctx->last_intervention_status = 0; + ngx_str_null(&ctx->last_intervention_log); } - if (ctx == NULL || mcf == NULL) { + if (ctx == NULL || mcf == NULL || transaction == NULL) { if (ctx != NULL) ctx->intervention_failed = 1; return r->header_sent ? NGX_ERROR : NGX_HTTP_INTERNAL_SERVER_ERROR; } ngx_memzero(&intervention, sizeof(intervention)); intervention.status = 200; - if (msc_intervention(transaction, &intervention) == 0) { - return 0; - } - - result = r->header_sent ? NGX_ERROR : NGX_HTTP_INTERNAL_SERVER_ERROR; ctx->intervention_failed = 1; + native_result = ngx_http_modsecurity_collect_native_intervention(r, ctx, + transaction, &intervention); + result = native_result == NGX_DECLINED ? 0 : + ngx_http_modsecurity_intervention_failure_status(r, ctx); + if (native_result != NGX_OK) { + goto done; + } ctx->last_intervention_status = intervention.status; if (ngx_http_modsecurity_log_intervention(r, ctx, mcf, &intervention) != NGX_OK) { goto done; } + if (ngx_http_modsecurity_defer_late_phase4_intervention(r, ctx, mcf)) { + goto done; + } if (intervention.url != NULL) { result = ngx_http_modsecurity_process_intervention_redirect(r, ctx, &intervention); } else if (intervention.status != 200) { msc_update_status_code(ctx->modsec_transaction, intervention.status); - if (early_log) { - ngx_http_modsecurity_log_handler(r); - ctx->logged = 1; + if (early_log && ngx_http_modsecurity_log_handler(r) != NGX_OK) { + goto done; } if (r->header_sent) { ctx->intervention_late = 1; @@ -284,15 +371,23 @@ ngx_http_modsecurity_cleanup(void *data) ngx_http_modsecurity_ctx_t *ctx; ctx = (ngx_http_modsecurity_ctx_t *) data; - - msc_transaction_cleanup(ctx->modsec_transaction); + if (ctx == NULL) { + return; + } + if (ctx->modsec_transaction != NULL) { + msc_transaction_cleanup(ctx->modsec_transaction); + ctx->modsec_transaction = NULL; + } #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) /* * Purge stored context headers. Memory allocated for individual stored header * name/value pair will be freed automatically when r->pool is destroyed. */ - ngx_array_destroy(ctx->sanity_headers_out); + if (ctx->sanity_headers_out != NULL) { + ngx_array_destroy(ctx->sanity_headers_out); + ctx->sanity_headers_out = NULL; + } #endif } @@ -305,6 +400,7 @@ ngx_http_modsecurity_create_ctx(ngx_http_request_t *r) ngx_http_modsecurity_ctx_t *ctx; ngx_http_modsecurity_conf_t *mcf; ngx_http_modsecurity_main_conf_t *mmcf; + char *transaction_id; ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_modsecurity_ctx_t)); if (ctx == NULL) @@ -317,37 +413,53 @@ ngx_http_modsecurity_create_ctx(ngx_http_request_t *r) mmcf = ngx_http_get_module_main_conf(r, ngx_http_modsecurity_module); mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + if (mmcf == NULL || mcf == NULL) { + return NULL; + } + dd("creating transaction with the following rules: '%p' -- ms: '%p'", mcf->rules_set, mmcf->modsec); if (mcf->transaction_id) { - if (ngx_http_complex_value(r, mcf->transaction_id, &s) != NGX_OK) { - return NGX_CONF_ERROR; + if (ngx_http_complex_value(r, mcf->transaction_id, &s) != NGX_OK || + s.len == (size_t) -1 || (s.len != 0 && s.data == NULL)) { + return NULL; + } + transaction_id = ngx_pnalloc(r->pool, s.len + 1); + if (transaction_id == NULL) { + return NULL; } - ctx->modsec_transaction = msc_new_transaction_with_id(mmcf->modsec, mcf->rules_set, (char *) s.data, r->connection->log); + if (s.len != 0) { + ngx_memcpy(transaction_id, s.data, s.len); + } + transaction_id[s.len] = '\0'; + ctx->modsec_transaction = msc_new_transaction_with_id(mmcf->modsec, mcf->rules_set, transaction_id, r->connection->log); } else { ctx->modsec_transaction = msc_new_transaction(mmcf->modsec, mcf->rules_set, r->connection->log); } - dd("transaction created"); - - ngx_http_set_ctx(r, ctx, ngx_http_modsecurity_module); + if (ctx->modsec_transaction == NULL) { + return NULL; + } +#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) + ctx->sanity_headers_out = ngx_array_create(r->pool, 12, sizeof(ngx_http_modsecurity_header_t)); + if (ctx->sanity_headers_out == NULL) { + ngx_http_modsecurity_cleanup(ctx); + return NULL; + } +#endif - cln = ngx_pool_cleanup_add(r->pool, sizeof(ngx_http_modsecurity_ctx_t)); + cln = ngx_pool_cleanup_add(r->pool, 0); if (cln == NULL) { dd("failed to create the ModSecurity context cleanup"); - return NGX_CONF_ERROR; + ngx_http_modsecurity_cleanup(ctx); + return NULL; } cln->handler = ngx_http_modsecurity_cleanup; cln->data = ctx; -#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) - ctx->sanity_headers_out = ngx_array_create(r->pool, 12, sizeof(ngx_http_modsecurity_header_t)); - if (ctx->sanity_headers_out == NULL) { - return NGX_CONF_ERROR; - } -#endif + ngx_http_set_ctx(r, ctx, ngx_http_modsecurity_module); return ctx; } diff --git a/tests/README.md b/tests/README.md index 268f0408..50052c65 100644 --- a/tests/README.md +++ b/tests/README.md @@ -19,7 +19,8 @@ CC=clang python3 -m unittest discover -s tests -p 'test_*runtime.py' -v ``` The suites compile the production body-filter functions, request-context -lookup, and intervention helpers with small NGINX and libModSecurity doubles. +creation/lookup/cleanup, intervention helpers, and request/audit handlers with +small NGINX and libModSecurity doubles. They check forwarding above the deprecated connector limit in every mode, checked accounting, overflow, memory/file buffers, bounded file reads and allocation/read errors, finalization at EOS, native failures, late interventions, downstream @@ -29,6 +30,13 @@ and disabled and cover redirect ownership, status values, and cleanup. These are helper behavior tests; they do not replace the native HTTP integration tests. +Further regressions exercise invalid native result values, nondisruptive +outputs, exact native response-limit rejection, valid empty EOS markers, +failed filter re-entry (including NULL-input flushes), a synchronous core +error response, and early rejection after a single file chunk. Request tests +exercise internal target-location isolation, error-page P1/P2 continuation, +once-only audit logging, failed context creation and custom-ID termination. + ## Native Phase4 integration tests Copy `tests/*.t` and `tests/*.pl` into an nginx-tests checkout after building @@ -46,6 +54,9 @@ delivery above the deprecated connector limit in every mode, and engine-owned reject the removed `minimal` mode and connector MIME directive. Late deny/redirect assertions check transport interruption, because headers have already been committed; they do not promise a clean 403. +Native engine Reject is an error in every mode, including safe; it does not +emit a rule-policy JSON event or attempt a second response header. The error +page suite also verifies target-location P1/P2 blocking after internal routing. A separate GitHub workflow job runs the standalone suite immediately with both Linux compilers, independently of the libModSecurity build. The native diff --git a/tests/modsecurity-config-custom-error-page.t b/tests/modsecurity-config-custom-error-page.t index a8f58622..f4bc076f 100644 --- a/tests/modsecurity-config-custom-error-page.t +++ b/tests/modsecurity-config-custom-error-page.t @@ -103,6 +103,33 @@ http { location / { } } + + server { + listen 127.0.0.1:8080; + server_name s3; + root %%TESTDIR%%/http; + + error_page 404 = /protected-error.txt; + + location = /trigger { + modsecurity on; + modsecurity_rules ' + SecRuleEngine On + SecAction "id:910501,phase:1,pass,nolog" + '; + try_files /missing-error-source =404; + } + + location = /protected-error.txt { + internal; + modsecurity on; + modsecurity_rules ' + SecRuleEngine On + SecRule REQUEST_HEADERS:X-Phase1-Deny "@streq yes" "id:910502,phase:1,deny,status:451,nolog" + SecRule REQUEST_HEADERS:X-Phase2-Deny "@streq yes" "id:910503,phase:2,deny,status:403,nolog" + '; + } + } } EOF @@ -112,9 +139,11 @@ my $custom_txt = "This is a custom error page."; $t->write_file("/index.html", $index_txt); mkdir($t->testdir() . '/http'); $t->write_file("/http/403.html", $custom_txt); +my $protected_txt = "This is a protected error target."; +$t->write_file("/http/protected-error.txt", $protected_txt); $t->run(); -$t->plan(10); +$t->plan(14); ############################################################################### @@ -160,13 +189,25 @@ unlike($global, qr/what=other/, 'ModSecurity at server / other not present in au like($local, qr/Access denied with code 403/, 'ModSecurity at location / 403 in auditlog'); like($global, qr/Access denied with code 403/, 'ModSecurity at server / 403 in auditlog'); +# The origin runs access processing before try_files redirects to a different +# protected location. The target must get its own transaction and run P1/P2. +my $target_allow = http_get_host('s3', '/trigger'); +my $target_p1 = http_get_host('s3', '/trigger', 'X-Phase1-Deny: yes'); +my $target_p2 = http_get_host('s3', '/trigger', 'X-Phase2-Deny: yes'); +like($target_allow, qr/^HTTP\/1\.[01] 200/, 'error-page target allows legitimate request'); +like($target_allow, qr/\Q$protected_txt\E/, 'error-page target delivers legitimate content'); +like($target_p1, qr/^HTTP\/1\.[01] 451/, 'error-page target preserves P1 deny'); +like($target_p2, qr/^HTTP\/1\.[01] 403/, 'error-page target continues from P1 allow to P2 deny'); + ############################################################################### sub http_get_host { - my ($host, $url) = @_; + my ($host, $url, $headers) = @_; + $headers = '' unless defined $headers; return http(<new()->has(qw/http/)->todo_alerts(); +my $t = Test::Nginx->new()->has(qw/http/); my $prefix = 'ENGINE_PREFIX_' . ('A' x 50); # exactly 64 bytes my $body = $prefix . ('B' x 70000) . 'ENGINE_TAIL_MARKER'; my $locations = ''; @@ -59,7 +58,7 @@ $locations EOC $t->run(); -$t->plan(37); +$t->plan(18); for my $mode (qw/off safe strict/) { my $response = http_get('/partial-' . $mode); @@ -69,41 +68,15 @@ for my $mode (qw/off safe strict/) { } # Static response headers have already passed the header filter when the -# engine queues its body-limit 403. Preserve each existing late-deny policy. -is(http_get('/reject-off'), '', 'off preserves native late engine Reject abort'); -my $safe_response = http_get('/reject-safe'); -like($safe_response, qr/^HTTP\/1\.1 200 OK/, 'safe continues after engine Reject'); -my ($safe_headers, $safe_payload) = split /\x0d\x0a\x0d\x0a/, $safe_response, 2; -is($safe_payload, $body, 'safe engine Reject forwards the complete static file'); -is(http_get('/reject-strict'), '', 'strict aborts after engine Reject'); +# engine queues its body-limit 403. Reject is an engine failure in every mode, +# not a rule intervention that Safe may downgrade. No body reaches downstream. +for my $mode (qw/off safe strict/) { + is(http_get('/reject-' . $mode), '', "$mode stops native engine Reject"); +} $t->stop(); my $log = $t->read_file('phase4-engine-limits.log'); -my @events; -my $valid_json = eval { - @events = map { decode_json($_) } grep { length $_ } split /\n/, $log; - 1; -}; -ok($valid_json, 'every engine-limit Phase4 event is valid JSON'); -diag($@) unless $valid_json; -is(scalar @events, 2, 'only safe and strict engine Reject emit dedicated events'); -my %event = map { $_->{uri} => $_ } @events; -ok(!grep({ $_->{uri} =~ m{^/partial-} } @events), 'ProcessPartial emits no intervention event in any mode'); -ok(!exists $event{'/reject-off'}, 'off engine Reject emits no dedicated event'); - -for my $mode (qw/safe strict/) { - my $item = $event{'/reject-' . $mode} || {}; - is($item->{event}, 'phase4_intervention', "$mode engine Reject event kind"); - is($item->{mode}, $mode, "$mode engine Reject retains the selected mode"); - is($item->{waf_status}, 403, "$mode retains the engine's body-limit 403"); - is($item->{wanted_action}, 'deny', "$mode retains the engine's deny decision"); - is($item->{actual_action}, $mode eq 'safe' ? 'log_only' : 'connection_abort', - "$mode applies the existing late-intervention action"); - is($item->{reason}, 'response_committed_' . $mode, "$mode records the existing late-intervention reason"); - ok($item->{header_sent}, "$mode engine Reject occurs after response headers are committed"); -} -unlike($log, qr/ENGINE_PREFIX_|B{100,}|ENGINE_TAIL_MARKER/, - 'raw Phase4 event log contains no response bytes'); +is($log, '', 'native Reject and ProcessPartial never emit rule-policy events'); my $error_log = $t->read_file('error.log'); for my $mode (qw/off safe strict/) { @@ -116,8 +89,6 @@ unlike($error_log, qr/engine-prefix-exact64-reject-/, unlike($error_log, qr/response body inspection append failed/, 'the engine Reject append result is not mistaken for an API failure'); -# Native off finalization has the same expected alert as other late denials. -my @alerts = $error_log =~ /^.*\[alert\].*$/gm; -my $expected_alert = qr/\[alert\].*header already sent\b.*request: "GET \/reject-off HTTP\/1\.[01]"/; -is(scalar(grep { /$expected_alert/ } @alerts), 1, 'off engine Reject has one expected late-finalization alert'); -is(join("\n", grep { !/$expected_alert/ } @alerts), '', 'no unexpected nginx alerts'); +is(scalar(() = $error_log =~ /response body limit rejected the response/g), 3, + 'every rejected response retains its native-limit diagnostic'); +unlike($error_log, qr/\[alert\]/, 'native Reject never attempts a second response header'); diff --git a/tests/modsecurity-phase4-modes.t b/tests/modsecurity-phase4-modes.t index dfce5a8d..7c2032f9 100644 --- a/tests/modsecurity-phase4-modes.t +++ b/tests/modsecurity-phase4-modes.t @@ -6,9 +6,8 @@ BEGIN { use FindBin; chdir($FindBin::Bin); } use lib 'lib'; use Test::Nginx; -# Native off/default late-deny finalization intentionally causes the nginx -# header-already-sent alert. Audit those alerts explicitly below. -my $t = Test::Nginx->new()->has(qw/http/)->todo_alerts(); +# A committed response must fail without attempting another HTTP status. +my $t = Test::Nginx->new()->has(qw/http/); $t->write_file_expand('nginx.conf', <<'EOC'); %%TEST_GLOBALS%% daemon off; @@ -68,11 +67,12 @@ unlike($log, qr/Hello off|Hello default|Hello safe|Hello strict/, 'response body # Stop first so the explicit alert audit includes all worker output. $t->stop(); -my @alerts = $t->read_file('error.log') =~ /^.*\[alert\].*$/gm; -my $expected_alert = qr/\[alert\].*header already sent\b.*request: "GET \/(?:off|default) HTTP\/1\.[01]"/; -my @late_alerts = grep { /$expected_alert/ } @alerts; -is(scalar @late_alerts, 2, 'native off/default each cause one expected late-finalization alert'); -is(scalar(grep { /request: "GET \/off HTTP\/1\.[01]"/ } @late_alerts), 1, 'explicit off has one expected alert'); -is(scalar(grep { /request: "GET \/default HTTP\/1\.[01]"/ } @late_alerts), 1, 'default off has one expected alert'); -my @unexpected_alerts = grep { !/$expected_alert/ } @alerts; -is(join("\n", @unexpected_alerts), '', 'no unexpected nginx alerts'); +my $error_log = $t->read_file('error.log'); +my @alerts = $error_log =~ /^.*\[alert\].*$/gm; +is(scalar @alerts, 0, 'no nginx alerts after late intervention'); +unlike($error_log, qr/header already sent\b.*request: "GET \/off HTTP\/1\.[01]"/, + 'explicit off never attempts a second response header'); +unlike($error_log, qr/header already sent\b.*request: "GET \/default HTTP\/1\.[01]"/, + 'default off never attempts a second response header'); +is(scalar(() = $error_log =~ /modsecurity phase4 intervention after headers sent, action=connection_abort, uri="\/strict"/g), + 1, 'strict abort is diagnosed once'); diff --git a/tests/test_intervention_runtime.py b/tests/test_intervention_runtime.py index c082a1f3..dcd4848c 100644 --- a/tests/test_intervention_runtime.py +++ b/tests/test_intervention_runtime.py @@ -8,6 +8,7 @@ from __future__ import annotations import os +import re from pathlib import Path import shlex import shutil @@ -61,6 +62,7 @@ ngx_str_t last_intervention_log; int last_intervention_status, intervention_late, intervention_failed; int intervention_redirect_location_installed, logged; + int phase4_failed, phase4_error_emitting, phase4_started, phase4_processed, intervention_body_limit; } ngx_http_modsecurity_ctx_t; typedef struct { void *phase4_log_file; @@ -77,7 +79,10 @@ }; #define NGX_OK 0 #define NGX_ERROR (-1) +#define NGX_DECLINED (-5) +#define NGX_HTTP_MODSEC_PHASE4_MODE_OFF 0 #define NGX_HTTP_BAD_REQUEST 400 +#define NGX_HTTP_FORBIDDEN 403 #define NGX_HTTP_INTERNAL_SERVER_ERROR 500 #define NGX_LOG_ERR 3 #define ngx_http_get_module_ctx(r, module) ((r)->ctx) @@ -91,12 +96,14 @@ } while (0) static int native_available = 1, native_status = 403, native_calls; +static int native_return = 1, native_disruptive = 1; static int native_has_log = 1, native_has_url, native_initialized; static char native_log[256] = "native intervention message"; static char native_url[256] = "https://example.test/new-location"; static int freed_log, freed_url, bad_free, allocation_calls, fail_allocation_at; static int list_calls, fail_list, clear_calls, error_log_calls; static int status_calls, updated_status, early_log_calls, sanity_calls, fail_sanity; +static int early_log_result; static char captured_error[256]; static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r); static void ngx_http_modsecurity_cleanup(void *data) { (void)data; } @@ -111,7 +118,8 @@ out->status = native_status; out->log = native_has_log ? native_log : NULL; out->url = native_has_url ? native_url : NULL; - return 1; + out->disruptive = native_disruptive; + return native_return; } /* Poison released native memory without releasing storage: a retained native * pointer remains readable but differs from the expected pool-owned value. */ @@ -161,10 +169,12 @@ ++status_calls; updated_status = status; } -static void ngx_http_modsecurity_log_handler(ngx_http_request_t *r) +static ngx_int_t ngx_http_modsecurity_log_handler(ngx_http_request_t *r) { - (void)r; + ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); + if (ctx != NULL) ctx->logged = 1; ++early_log_calls; + return early_log_result; } #if MODSECURITY_SANITY_CHECKS static ngx_int_t ngx_http_modsecurity_store_ctx_header(ngx_http_request_t *r, @@ -193,6 +203,7 @@ ngx_pool_cleanup_t cleanup = {ngx_http_modsecurity_cleanup, &ctx, NULL}; const char *expected_log = "native intervention message"; const char *expected_url = "https://example.test/new-location"; + const char *limit_message = "Response body limit is marked to reject the request"; int result; CHECK(argc == 2); request.pool = &pool; request.connection = &connection; @@ -212,6 +223,99 @@ CHECK(!ctx.intervention_redirect_location_installed); CHECK(allocation_calls == 0 && error_log_calls == 0); CHECK(freed_log == 0 && freed_url == 0 && bad_free == 0); + } else if (strcmp(argv[1], "invalid-native-results") == 0) { + const int invalid[] = {-2, -1, 2, 99}; + size_t index; + int mode, committed; + native_has_url = 1; native_status = 302; + for (mode = 0; mode <= 2; ++mode) { + conf.phase4_mode = mode; + for (committed = 0; committed <= 1; ++committed) { + request.header_sent = committed; + for (index = 0; index < sizeof(invalid) / sizeof(invalid[0]); ++index) { + strcpy(native_log, expected_log); strcpy(native_url, expected_url); + native_return = invalid[index]; freed_log = 0; freed_url = 0; + ctx.last_intervention_status = 451; + ctx.last_intervention_log.data = (u_char *)"stale"; + ctx.last_intervention_log.len = 5U; + ctx.intervention_body_limit = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); + CHECK(result == (committed ? NGX_ERROR : 500)); + CHECK(ctx.intervention_failed && !ctx.intervention_late); + CHECK(!ctx.intervention_body_limit && !ctx.intervention_redirect_location_installed); + CHECK(ctx.last_intervention_status == 0); + CHECK(ctx.last_intervention_log.data == NULL && ctx.last_intervention_log.len == 0U); + CHECK(freed_log == 1 && freed_url == 1 && bad_free == 0); + CHECK(allocation_calls == 0 && status_calls == 0 && list_calls == 0); + CHECK(early_log_calls == 0 && captured_error[0] == '\0'); + CHECK(request.headers_out.location == &previous_location && previous_location.hash == 1); + } + } + } + } else if (strcmp(argv[1], "nondisruptive-result") == 0 || + strcmp(argv[1], "no-result-owned-outputs") == 0) { + native_has_url = 1; native_status = 302; + if (strcmp(argv[1], "nondisruptive-result") == 0) native_disruptive = 0; + else native_return = 0; + ctx.last_intervention_status = 451; + ctx.last_intervention_log.data = (u_char *)"stale"; + ctx.last_intervention_log.len = 5U; ctx.intervention_body_limit = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); + CHECK(result == 0 && native_calls == 1); + CHECK(!ctx.intervention_failed && !ctx.intervention_late && !ctx.intervention_body_limit); + CHECK(!ctx.intervention_redirect_location_installed && ctx.last_intervention_status == 0); + CHECK(ctx.last_intervention_log.data == NULL && ctx.last_intervention_log.len == 0U); + CHECK(freed_log == 1 && freed_url == 1 && bad_free == 0); + CHECK(allocation_calls == 0 && status_calls == 0 && list_calls == 0); + CHECK(early_log_calls == 0 && error_log_calls == 0); + CHECK(request.headers_out.location == &previous_location && previous_location.hash == 1); + } else if (strcmp(argv[1], "body-limit-rejection") == 0) { + int mode, committed; + for (mode = 0; mode <= 2; ++mode) { + conf.phase4_mode = mode; + for (committed = 0; committed <= 1; ++committed) { + strcpy(native_log, limit_message); freed_log = 0; + request.header_sent = committed; ctx.phase4_started = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == (committed ? NGX_ERROR : 403)); + CHECK(ctx.intervention_failed && ctx.intervention_body_limit && !ctx.intervention_late); + CHECK(ctx.last_intervention_status == 403); + CHECK(!ctx.intervention_redirect_location_installed && freed_log == 1 && freed_url == 0); + CHECK(status_calls == 0 && list_calls == 0 && early_log_calls == 0); + } + } + } else if (strcmp(argv[1], "body-limit-other-phase") == 0 || + strcmp(argv[1], "body-limit-rule-lookalike") == 0) { + if (strcmp(argv[1], "body-limit-other-phase") == 0) strcpy(native_log, limit_message); + else { + ctx.phase4_started = 1; ctx.phase4_processed = 1; + strcpy(native_log, "[client 127.0.0.1] ModSecurity: Access denied with code 403 (phase 4). [id \"12345\"] [msg \"Response body limit is marked to reject the request\"]"); + } + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 403 && !ctx.intervention_failed && !ctx.intervention_body_limit); + CHECK(ctx.last_intervention_status == 403 && status_calls == 1); + CHECK(freed_log == 1 && freed_url == 0 && early_log_calls == 0); + } else if (strcmp(argv[1], "uncorrelated-append-intervention") == 0) { + int mode, committed; + ctx.phase4_started = 1; native_has_log = 0; + for (mode = 0; mode <= 2; ++mode) { + conf.phase4_mode = mode; + for (committed = 0; committed <= 1; ++committed) { + request.header_sent = committed; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == (committed ? NGX_ERROR : 500)); + CHECK(ctx.intervention_failed && !ctx.intervention_body_limit && !ctx.intervention_late); + CHECK(ctx.last_intervention_status == 0 && ctx.last_intervention_log.data == NULL); + CHECK(status_calls == 0 && early_log_calls == 0 && list_calls == 0); + CHECK(freed_log == 0 && freed_url == 0 && allocation_calls == 0); + } + } + } else if (strcmp(argv[1], "early-log-failure") == 0) { + early_log_result = NGX_ERROR; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); + CHECK(result == 500 && ctx.intervention_failed && !ctx.intervention_late); + CHECK(early_log_calls == 1 && freed_log == 1 && freed_url == 0); + CHECK(list_calls == 0 && !ctx.intervention_redirect_location_installed); } else if (strcmp(argv[1], "status-precommit") == 0) { result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); CHECK(result == 403 && native_initialized); @@ -234,6 +338,23 @@ CHECK(status_calls == 1 && early_log_calls == 0); CHECK(freed_log == 1 && freed_url == 0); CHECK(strcmp((const char *)ctx.last_intervention_log.data, expected_log) == 0); + } else if (strcmp(argv[1], "late-phase4-policy-deferral") == 0) { + int mode; + request.header_sent = 1; + ctx.phase4_started = 1; ctx.phase4_processed = 1; + for (mode = 0; mode <= 2; ++mode) { + conf.phase4_mode = mode; + strcpy(native_log, expected_log); freed_log = 0; + status_calls = 0; early_log_calls = 0; updated_status = 0; + ctx.logged = 0; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); + CHECK(result == NGX_ERROR && ctx.intervention_late && !ctx.intervention_failed); + CHECK(ctx.last_intervention_status == 403 && !ctx.intervention_body_limit); + CHECK(status_calls == (mode == 0) && early_log_calls == (mode == 0)); + CHECK(updated_status == (mode == 0 ? 403 : 0)); + CHECK(ctx.logged == (mode == 0) && list_calls == 0); + CHECK(freed_log == 1 && freed_url == 0 && !ctx.intervention_redirect_location_installed); + } } else if (strcmp(argv[1], "status-log-only") == 0) { native_status = 200; result = ngx_http_modsecurity_process_intervention(&transaction, &request, 1); @@ -362,6 +483,14 @@ def unit_program(module: str) -> str: helpers += "\nstatic int\n" + function_definition( module, "ngx_http_modsecurity_process_intervention_redirect" ) + for helper in ( + "ngx_http_modsecurity_is_response_body_limit_rejection", + "ngx_http_modsecurity_intervention_failure_status", + "ngx_http_modsecurity_collect_native_intervention", + "ngx_http_modsecurity_defer_late_phase4_intervention", + ): + if re.search(rf"\b{helper}\s*\(", module): + helpers += "\nstatic ngx_int_t\n" + function_definition(module, helper) helpers += "\nint\n" + function_definition( module, "ngx_http_modsecurity_process_intervention" ) @@ -410,6 +539,11 @@ def test(self: InterventionRuntimeTests) -> None: for case in ( "no-intervention", "status-precommit", "status-committed", "status-native-committed", "status-log-only", + "late-phase4-policy-deferral", + "invalid-native-results", "nondisruptive-result", "no-result-owned-outputs", + "body-limit-rejection", "body-limit-other-phase", "body-limit-rule-lookalike", + "early-log-failure", + "uncorrelated-append-intervention", "redirect-owned", "redirect-committed", "redirect-crlf", "redirect-list-failure", "redirect-allocation-failure", "redirect-sanity-failure", "log-allocation-precommit", "log-allocation-committed", "disabled-log-retention", diff --git a/tests/test_lifecycle_runtime.py b/tests/test_lifecycle_runtime.py new file mode 100644 index 00000000..a87d546d --- /dev/null +++ b/tests/test_lifecycle_runtime.py @@ -0,0 +1,639 @@ +"""Exercise access routing and native audit completion through production code. + +The complete access and log handlers run with small NGINX/SDK doubles. These +tests prove their control flow, not live error_page or client transport behavior. +""" + +from __future__ import annotations + +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import tempfile +import unittest + +from test_phase4_runtime import function_definition + +ROOT = Path(__file__).resolve().parents[1] + +PREAMBLE = r""" +#include +#include +#include +#include +#include +typedef unsigned char u_char; +typedef long ngx_int_t; +typedef unsigned long ngx_uint_t; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct { int unused; } ngx_pool_t; +typedef struct { int unused; } Transaction; +typedef struct ngx_http_request_s ngx_http_request_t; +typedef struct ngx_list_part_s { + void *elts; + ngx_uint_t nelts; + struct ngx_list_part_s *next; +} ngx_list_part_t; +typedef struct { ngx_list_part_t part; } ngx_list_t; +typedef struct { ngx_str_t key, value; } ngx_table_elt_t; +typedef struct { u_char *pos, *last; int last_buf; } ngx_buf_t; +typedef struct ngx_chain_s { ngx_buf_t *buf; struct ngx_chain_s *next; } ngx_chain_t; +typedef struct { struct { ngx_str_t name; } file; } ngx_temp_file_t; +typedef struct { ngx_temp_file_t *temp_file; ngx_chain_t *bufs; } ngx_http_request_body_t; +typedef struct { void *sockaddr, *local_sockaddr, *log; ngx_str_t addr_text; int error; } ngx_connection_t; +typedef struct { + ngx_http_request_t *r; + Transaction *modsec_transaction; + int waiting_more_body, body_requested, request_body_processed; + int intervention_triggered, intervention_failed; + int logged, native_logging_failed; +} ngx_http_modsecurity_ctx_t; +typedef struct { int enable; } ngx_http_modsecurity_conf_t; +struct ngx_http_request_s { + ngx_pool_t *pool; + ngx_connection_t *connection; + ngx_http_request_t *main; + int count, method, http_version, header_sent, error_page; + ngx_str_t http_protocol, unparsed_uri, method_name; + struct { ngx_str_t server; ngx_list_t headers; } headers_in; + ngx_http_request_body_t *request_body; + int request_body_in_single_buf, request_body_in_persistent_file; + int request_body_in_file_only, request_body_in_clean_file; + void (*write_event_handler)(ngx_http_request_t *); + ngx_http_modsecurity_ctx_t *ctx, *recovered; + ngx_http_modsecurity_conf_t *conf; +}; +#define NGX_OK 0 +#define NGX_ERROR (-1) +#define NGX_AGAIN (-2) +#define NGX_DONE (-4) +#define NGX_DECLINED (-5) +#define NGX_HTTP_INTERNAL_SERVER_ERROR 500 +#define NGX_HTTP_SPECIAL_RESPONSE 300 +#define NGX_HTTP_VERSION_9 9 +#define NGX_HTTP_VERSION_10 1000 +#define NGX_HTTP_VERSION_11 1001 +#define NGX_SOCKADDR_STRLEN 64 +#define NGX_LOG_ERR 3 +#define dd(...) ((void)0) +#define ngx_http_get_module_ctx(r, module) ((r)->ctx) +#define ngx_http_get_module_loc_conf(r, module) ((r)->conf) +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "line %d: %s\n", __LINE__, #condition); return 1; \ +} } while (0) + +static Transaction transaction; +static ngx_http_modsecurity_ctx_t created; +static int created_calls, recovered_calls, error_calls; +static int native_results[4] = {1, 1, 1, 1}; +static int native_calls[4], intervention_results[5], intervention_calls[5]; +static int active_phase, append_result = 1, append_calls, file_result = 1; +static int log_result = 1, log_calls, log_reenter, nested_log_result; +static ngx_http_request_t *active_request; +static ngx_int_t ngx_http_modsecurity_log_handler(ngx_http_request_t *r); +static void ngx_http_core_run_phases(ngx_http_request_t *r) { (void)r; } +static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r) +{ + ++recovered_calls; + if (r->ctx != NULL && r->ctx->r == r) return r->ctx; + return r->recovered != NULL && r->recovered->r == r ? r->recovered : NULL; +} +static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_create_ctx(ngx_http_request_t *r) +{ + ++created_calls; + memset(&created, 0, sizeof(created)); + created.r = r; + created.modsec_transaction = &transaction; + r->ctx = &created; + return &created; +} +static char *ngx_str_to_char(ngx_str_t value, ngx_pool_t *pool) +{ + (void)pool; + return value.len == 0 ? NULL : (char *)value.data; +} +static int ngx_inet_get_port(void *address) { (void)address; return 80; } +static ngx_int_t ngx_connection_local_sockaddr(ngx_connection_t *connection, ngx_str_t *out, int port) +{ + (void)connection; (void)port; + out->data = (u_char *)"127.0.0.1"; out->len = 9; + return NGX_OK; +} +static ngx_pool_t *ngx_http_modsecurity_pcre_malloc_init(ngx_pool_t *pool) { return pool; } +static void ngx_http_modsecurity_pcre_malloc_done(ngx_pool_t *pool) { (void)pool; } +static void ngx_log_error(int level, void *log, int error, const char *format, ...) +{ + (void)level; (void)log; (void)error; (void)format; + ++error_calls; +} +static int phase_result(int phase) +{ + active_phase = phase; + ++native_calls[phase]; + return native_results[phase]; +} +static int msc_process_connection(Transaction *t, const char *client, int cp, const char *server, int sp) +{ + (void)t; (void)client; (void)cp; (void)server; (void)sp; + return phase_result(0); +} +static int msc_process_uri(Transaction *t, const char *uri, const char *method, const char *version) +{ + (void)t; (void)uri; (void)method; (void)version; + return phase_result(1); +} +static int msc_add_n_request_header(Transaction *t, const u_char *name, size_t nl, const u_char *value, size_t vl) +{ + (void)t; (void)name; (void)nl; (void)value; (void)vl; + return 1; +} +static int msc_process_request_headers(Transaction *t) { (void)t; return phase_result(2); } +static int msc_process_request_body(Transaction *t) { (void)t; return phase_result(3); } +static int msc_append_request_body(Transaction *t, const u_char *data, size_t size) +{ + (void)t; (void)data; (void)size; + active_phase = 4; + ++append_calls; + return append_result; +} +static int msc_request_body_from_file(Transaction *t, const char *name) +{ + (void)t; (void)name; + return file_result; +} +static int ngx_http_modsecurity_process_intervention(Transaction *t, ngx_http_request_t *r, int early) +{ + (void)t; (void)r; (void)early; + ++intervention_calls[active_phase]; + return intervention_results[active_phase]; +} +static ngx_int_t ngx_http_read_client_request_body(ngx_http_request_t *r, void (*callback)(ngx_http_request_t *)) +{ + (void)r; (void)callback; + return NGX_OK; +} +static int msc_process_logging(Transaction *t) +{ + (void)t; + ++log_calls; + if (log_reenter && log_calls == 1) + nested_log_result = ngx_http_modsecurity_log_handler(active_request); + return log_result; +} +""" + +CASES = r""" +static int run_access(const char *scenario, int parameter) +{ + ngx_pool_t pool = {0}; + ngx_connection_t connection = {0}; + ngx_http_modsecurity_conf_t conf = {1}; + ngx_http_modsecurity_ctx_t previous = {0}; + ngx_http_request_t other_request = {0}; + ngx_http_request_body_t body = {0}; + ngx_http_request_t request = {0}; + u_char bytes[] = "abc"; + ngx_buf_t buffer = {bytes, bytes + 3, 0}; + ngx_chain_t chain = {&buffer, NULL}; + ngx_temp_file_t file = {{{0, NULL}}}; + int phase, before_calls, after_calls; + ngx_int_t result; + request.pool = &pool; request.connection = &connection; + request.main = &request; request.conf = &conf; request.request_body = &body; + request.http_version = NGX_HTTP_VERSION_11; + request.unparsed_uri.data = (u_char *)"/protected"; request.unparsed_uri.len = 10; + request.method_name.data = (u_char *)"POST"; request.method_name.len = 4; + connection.addr_text.data = (u_char *)"127.0.0.1"; connection.addr_text.len = 9; + request.error_page = 1; + + if (strcmp(scenario, "fresh-error-page") == 0 || + strcmp(scenario, "foreign-direct") == 0) { + previous.r = strcmp(scenario, "foreign-direct") == 0 ? &other_request : &request; + previous.modsec_transaction = &transaction; + previous.request_body_processed = 1; previous.intervention_triggered = 1; + if (strcmp(scenario, "foreign-direct") == 0) request.ctx = &previous; + else request.recovered = &previous; + intervention_results[3] = parameter; + } else if (strcmp(scenario, "p1-status") == 0) { + intervention_results[2] = parameter; + } else if (strcmp(scenario, "p2-status") == 0) { + intervention_results[3] = parameter; + } else if (strcmp(scenario, "intervention-error") == 0) { + intervention_results[parameter] = -1; + if (parameter == 4) body.bufs = &chain; + } else if (strcmp(scenario, "native-error") == 0) { + phase = parameter / 10; + native_results[phase] = parameter % 10 - 1; + } else if (strcmp(scenario, "append") == 0) { + body.bufs = &chain; append_result = parameter; + } else if (strcmp(scenario, "file") == 0) { + file.file.name.data = (u_char *)"body.tmp"; file.file.name.len = 8; + body.temp_file = &file; file_result = parameter; + } else { + return 2; + } + + result = ngx_http_modsecurity_access_handler(&request); + if (strcmp(scenario, "fresh-error-page") == 0 || + strcmp(scenario, "foreign-direct") == 0) { + CHECK(created_calls == 1 && recovered_calls == 0); + CHECK(native_calls[2] == 1 && native_calls[3] == 1); + CHECK(previous.request_body_processed && previous.intervention_triggered); + CHECK(result == (parameter > 0 ? parameter : NGX_DECLINED)); + } else if (strcmp(scenario, "p1-status") == 0) { + CHECK(result == parameter && created.intervention_triggered); + CHECK(native_calls[3] == 0); + } else if (strcmp(scenario, "p2-status") == 0) { + CHECK(result == parameter && created.intervention_triggered); + CHECK(native_calls[3] == 1 && created.request_body_processed); + } else if ((strcmp(scenario, "append") == 0 && (parameter == 0 || parameter == 1)) || + (strcmp(scenario, "file") == 0 && parameter == 1)) { + CHECK(result == NGX_DECLINED && native_calls[3] == 1); + CHECK(!created.intervention_failed && created.request_body_processed); + } else { + CHECK(result == 500 && created.intervention_failed); + CHECK(created.intervention_triggered); + if (strcmp(scenario, "native-error") == 0) { + phase = parameter / 10; + CHECK(intervention_calls[phase] == 0); + } + if (strcmp(scenario, "append") == 0 || strcmp(scenario, "file") == 0) + CHECK(native_calls[3] == 0); + before_calls = append_calls + log_calls; + for (phase = 0; phase < 4; ++phase) before_calls += native_calls[phase]; + result = ngx_http_modsecurity_access_handler(&request); + after_calls = append_calls + log_calls; + for (phase = 0; phase < 4; ++phase) after_calls += native_calls[phase]; + CHECK(result == 500 && before_calls == after_calls); + request.header_sent = 1; + CHECK(ngx_http_modsecurity_access_handler(&request) == NGX_ERROR); + CHECK(connection.error); + } + return 0; +} + +static int run_logging(const char *scenario, int parameter) +{ + ngx_pool_t pool = {0}; + ngx_connection_t connection = {0}; + ngx_http_modsecurity_ctx_t ctx = {0}; + ngx_http_request_t request = {0}; + ngx_int_t result; + request.pool = &pool; request.connection = &connection; request.ctx = &ctx; + ctx.r = &request; ctx.modsec_transaction = &transaction; + active_request = &request; log_result = parameter; + if (strcmp(scenario, "log-missing") == 0) ctx.modsec_transaction = NULL; + if (strcmp(scenario, "log-no-context") == 0) request.ctx = NULL; + if (strcmp(scenario, "log-reenter") == 0) log_reenter = 1; + result = ngx_http_modsecurity_log_handler(&request); + if (strcmp(scenario, "log-no-context") == 0) { + CHECK(result == NGX_OK && log_calls == 0); + return 0; + } + CHECK(ctx.logged); + CHECK(result == (parameter == 1 && ctx.modsec_transaction != NULL ? NGX_OK : NGX_ERROR)); + CHECK(ctx.native_logging_failed == (result != NGX_OK)); + CHECK(log_calls == (ctx.modsec_transaction != NULL ? 1 : 0)); + if (log_reenter) CHECK(nested_log_result == NGX_ERROR); + CHECK(ngx_http_modsecurity_log_handler(&request) == result); + CHECK(log_calls == (ctx.modsec_transaction != NULL ? 1 : 0)); + CHECK(!connection.error && !request.header_sent); + return 0; +} + +int main(int argc, char **argv) +{ + CHECK(argc == 3); + return strncmp(argv[1], "log-", 4) == 0 + ? run_logging(argv[1], atoi(argv[2])) + : run_access(argv[1], atoi(argv[2])); +} +""" + + +def unit_program(access: str, logging: str) -> str: + helpers = "" + if "ngx_http_modsecurity_request_failure(" in access: + helpers += "\nstatic ngx_int_t\n" + function_definition( + access, "ngx_http_modsecurity_request_failure" + ) + helpers += "\nvoid\n" + function_definition( + access, "ngx_http_modsecurity_request_read" + ) + helpers += "\nngx_int_t\n" + function_definition( + access, "ngx_http_modsecurity_access_handler" + ) + helpers += "\nstatic ngx_int_t\n" + function_definition( + logging, "ngx_http_modsecurity_log_handler" + ) + return PREAMBLE + helpers + CASES + + +OWNERSHIP_PREAMBLE = r""" +#include +#include +#include +#include +typedef unsigned char u_char; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct { int unused; } Transaction; +typedef struct { int unused; } ngx_array_t; +typedef struct { int unused; } ngx_http_modsecurity_header_t; +typedef struct ngx_http_request_s ngx_http_request_t; +typedef struct { + ngx_http_request_t *r; + Transaction *modsec_transaction; + ngx_array_t *sanity_headers_out; +} ngx_http_modsecurity_ctx_t; +typedef struct { void (*handler)(void *); void *data; } ngx_pool_cleanup_t; +typedef struct { + unsigned char storage[1024]; size_t used; ngx_pool_cleanup_t cleanup; +} ngx_pool_t; +typedef struct { void *rules_set, *transaction_id; } ngx_http_modsecurity_conf_t; +typedef struct { void *modsec; } ngx_http_modsecurity_main_conf_t; +typedef struct { void *log; } ngx_connection_t; +struct ngx_http_request_s { + ngx_pool_t *pool; ngx_connection_t *connection; + ngx_http_modsecurity_conf_t *conf; ngx_http_modsecurity_main_conf_t *main_conf; + ngx_http_modsecurity_ctx_t *ctx; +}; +#define NGX_OK 0 +#define NGX_CONF_ERROR ((void *)-1) +#define dd(...) ((void)0) +#define ngx_http_get_module_main_conf(r, module) ((r)->main_conf) +#define ngx_http_get_module_loc_conf(r, module) ((r)->conf) +#define ngx_http_set_ctx(r, value, module) ((r)->ctx = (value)) +#define ngx_memcpy(dst, src, size) memcpy((dst), (src), (size)) +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "line %d: %s\n", __LINE__, #condition); return 1; \ +} } while (0) +static Transaction transaction; +static int allocation_calls, fail_allocation_at, complex_failure, native_failure; +static int registration_failure, registration_calls, native_calls, cleanup_calls; +static int sanity_failure, sanity_allocations, sanity_destroys; +static int custom_id_copied, custom_id_terminated; +static char custom_id[4] = {'i', 'd', '7', '!'}; +static void ngx_http_modsecurity_cleanup(void *data); +static void *ngx_pnalloc(ngx_pool_t *pool, size_t bytes) +{ + void *result; + ++allocation_calls; + if (allocation_calls == fail_allocation_at || bytes > sizeof(pool->storage) - pool->used) + return NULL; + result = pool->storage + pool->used; + pool->used += bytes; + return result; +} +static void *ngx_pcalloc(ngx_pool_t *pool, size_t bytes) +{ + void *result = ngx_pnalloc(pool, bytes); + if (result != NULL) memset(result, 0, bytes); + return result; +} +static int ngx_http_complex_value(ngx_http_request_t *r, void *expression, ngx_str_t *out) +{ + (void)r; (void)expression; + out->data = (u_char *)custom_id; out->len = 3; + return complex_failure ? -1 : NGX_OK; +} +static Transaction *msc_new_transaction(void *modsec, void *rules, void *log) +{ + (void)modsec; (void)rules; (void)log; + ++native_calls; + return native_failure ? NULL : &transaction; +} +static Transaction *msc_new_transaction_with_id(void *modsec, void *rules, char *id, void *log) +{ + custom_id_copied = id != custom_id && memcmp(id, custom_id, 3) == 0; + custom_id_terminated = id[3] == '\0'; + return msc_new_transaction(modsec, rules, log); +} +static void msc_transaction_cleanup(Transaction *value) +{ + if (value != &transaction) abort(); + ++cleanup_calls; +} +static ngx_pool_cleanup_t *ngx_pool_cleanup_add(ngx_pool_t *pool, size_t size) +{ + (void)size; + ++registration_calls; + return registration_failure ? NULL : &pool->cleanup; +} +#if MODSECURITY_SANITY_CHECKS +static ngx_array_t sanity_array; +static ngx_array_t *ngx_array_create(ngx_pool_t *pool, unsigned count, size_t bytes) +{ + (void)pool; (void)count; (void)bytes; + ++sanity_allocations; + return sanity_failure ? NULL : &sanity_array; +} +static void ngx_array_destroy(ngx_array_t *array) +{ + if (array != &sanity_array) abort(); + ++sanity_destroys; +} +#endif +""" + +OWNERSHIP_CASES = r""" +int main(int argc, char **argv) +{ + ngx_pool_t pool = {0}; + ngx_connection_t connection = {0}; + ngx_http_modsecurity_conf_t conf = {0}; + ngx_http_modsecurity_main_conf_t main_conf = {0}; + ngx_http_request_t request = {0}; + ngx_http_modsecurity_ctx_t *ctx; + int expected_failure = 1; + CHECK(argc == 2); + request.pool = &pool; request.connection = &connection; + request.conf = &conf; request.main_conf = &main_conf; + conf.rules_set = &transaction; main_conf.modsec = &transaction; + if (strcmp(argv[1], "context-allocation") == 0) fail_allocation_at = 1; + else if (strcmp(argv[1], "complex-value") == 0) { + conf.transaction_id = &conf; complex_failure = 1; + } else if (strcmp(argv[1], "custom-id-allocation") == 0) { + conf.transaction_id = &conf; fail_allocation_at = 2; + } else if (strcmp(argv[1], "native-allocation") == 0) native_failure = 1; + else if (strcmp(argv[1], "cleanup-registration") == 0) registration_failure = 1; + else if (strcmp(argv[1], "sanity-allocation") == 0) { + sanity_failure = 1; expected_failure = MODSECURITY_SANITY_CHECKS; + } else if (strcmp(argv[1], "custom-id-owned") == 0) { + conf.transaction_id = &conf; expected_failure = 0; + } else if (strcmp(argv[1], "cleanup-once") == 0) expected_failure = 0; + else return 2; + + ctx = ngx_http_modsecurity_create_ctx(&request); + if (expected_failure) { + CHECK(ctx == NULL && request.ctx == NULL); + CHECK(pool.cleanup.handler == NULL && pool.cleanup.data == NULL); + if (registration_failure || sanity_failure) CHECK(cleanup_calls == 1); + else CHECK(cleanup_calls == 0); + if (fail_allocation_at || complex_failure) CHECK(native_calls == 0); + return 0; + } + CHECK(ctx != NULL && ctx != NGX_CONF_ERROR && ctx == request.ctx); + CHECK(native_calls == 1 && registration_calls == 1 && cleanup_calls == 0); + CHECK(pool.cleanup.handler == ngx_http_modsecurity_cleanup && pool.cleanup.data == ctx); + if (conf.transaction_id != NULL) CHECK(custom_id_copied && custom_id_terminated); + ngx_http_modsecurity_cleanup(NULL); + pool.cleanup.handler(pool.cleanup.data); + pool.cleanup.handler(pool.cleanup.data); + CHECK(ctx->modsec_transaction == NULL && cleanup_calls == 1); + CHECK(sanity_allocations == MODSECURITY_SANITY_CHECKS); + CHECK(sanity_destroys == MODSECURITY_SANITY_CHECKS); + return 0; +} +""" + + +def ownership_program(module: str) -> str: + helpers = "\nstatic void\n" + function_definition( + module, "ngx_http_modsecurity_cleanup" + ) + helpers += "\nngx_http_modsecurity_ctx_t *\n" + function_definition( + module, "ngx_http_modsecurity_create_ctx" + ) + return OWNERSHIP_PREAMBLE + helpers + OWNERSHIP_CASES + + +class LifecycleRuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler or shutil.which(compiler[0]) is None: + raise RuntimeError("set CC to a GCC- or Clang-compatible C17 compiler") + workspace = tempfile.TemporaryDirectory(prefix="nginx-lifecycle-") + cls.addClassCleanup(workspace.cleanup) + directory = Path(workspace.name) + source = directory / "lifecycle.c" + source.write_text(unit_program( + (ROOT / "src/ngx_http_modsecurity_access.c").read_text(encoding="utf-8"), + (ROOT / "src/ngx_http_modsecurity_log.c").read_text(encoding="utf-8"), + ), encoding="utf-8") + cls.binary = directory / ("lifecycle.exe" if os.name == "nt" else "lifecycle") + build = subprocess.run( + [*compiler, "-std=c17", "-Wall", "-Wextra", "-Werror", + str(source), "-o", str(cls.binary)], + capture_output=True, text=True, check=False, timeout=60, + ) + if build.returncode: + raise AssertionError(f"Lifecycle helper build failed:\n{build.stderr}") + + def case(self, name: str, parameter: int) -> None: + result = subprocess.run( + [str(self.binary), name, str(parameter)], + capture_output=True, text=True, check=False, timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_internal_error_page_uses_fresh_location_context(self): + for status in (0, 403): + with self.subTest(status=status): + self.case("fresh-error-page", status) + + def test_foreign_direct_context_does_not_skip_access_processing(self): + self.case("foreign-direct", 403) + + def test_error_page_preserves_p1_and_p2_status_decisions(self): + for scenario in ("p1-status", "p2-status"): + for status in (302, 403, 451): + with self.subTest(scenario=scenario, status=status): + self.case(scenario, status) + + def test_negative_interventions_fail_and_remain_terminal(self): + for phase in range(5): + with self.subTest(phase=phase): + self.case("intervention-error", phase) + + def test_native_phase_requires_exact_success(self): + for phase in range(4): + for result in (-1, 0, 2): + with self.subTest(phase=phase, result=result): + self.case("native-error", phase * 10 + result + 1) + + def test_native_append_partial_is_not_phase_failure(self): + for result in (-1, 0, 1, 2): + with self.subTest(result=result): + self.case("append", result) + + def test_native_file_result_is_not_append_partial(self): + for result in (-1, 0, 1, 2): + with self.subTest(result=result): + self.case("file", result) + + def test_audit_result_is_checked_and_attempted_once(self): + for result in (-1, 0, 1, 2): + with self.subTest(result=result): + self.case("log-result", result) + + def test_audit_reentry_cannot_repeat_native_call(self): + self.case("log-reenter", 1) + + def test_missing_native_transaction_does_not_call_audit(self): + self.case("log-missing", 1) + + def test_disabled_context_does_not_call_audit(self): + self.case("log-no-context", 1) + + +class LifecycleOwnershipTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler or shutil.which(compiler[0]) is None: + raise RuntimeError("set CC to a GCC- or Clang-compatible C17 compiler") + workspace = tempfile.TemporaryDirectory(prefix="nginx-ownership-") + cls.addClassCleanup(workspace.cleanup) + directory = Path(workspace.name) + source = directory / "ownership.c" + source.write_text(ownership_program( + (ROOT / "src/ngx_http_modsecurity_module.c").read_text(encoding="utf-8") + ), encoding="utf-8") + cls.binaries = {} + for sanity in (0, 1): + binary = directory / (f"ownership-{sanity}.exe" if os.name == "nt" else f"ownership-{sanity}") + build = subprocess.run( + [*compiler, "-std=c17", "-Wall", "-Wextra", "-Werror", + f"-DMODSECURITY_SANITY_CHECKS={sanity}", str(source), "-o", str(binary)], + capture_output=True, text=True, check=False, timeout=60, + ) + if build.returncode: + raise AssertionError(f"Ownership helper build failed:\n{build.stderr}") + cls.binaries[sanity] = binary + + def case(self, name: str) -> None: + for sanity, binary in self.binaries.items(): + with self.subTest(sanity=sanity): + result = subprocess.run( + [str(binary), name], capture_output=True, text=True, + check=False, timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_context_and_native_allocation_failures_are_null(self): + for name in ("context-allocation", "native-allocation"): + with self.subTest(name=name): + self.case(name) + + def test_custom_id_failures_do_not_create_native_transaction(self): + for name in ("complex-value", "custom-id-allocation"): + with self.subTest(name=name): + self.case(name) + + def test_partial_creation_failure_cleans_native_owner(self): + for name in ("cleanup-registration", "sanity-allocation"): + with self.subTest(name=name): + self.case(name) + + def test_custom_id_copy_is_terminated_before_native_call(self): + self.case("custom-id-owned") + + def test_cleanup_is_null_safe_and_idempotent(self): + self.case("cleanup-once") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_phase4_runtime.py b/tests/test_phase4_runtime.py index d3be5f03..9777a356 100644 --- a/tests/test_phase4_runtime.py +++ b/tests/test_phase4_runtime.py @@ -84,6 +84,7 @@ def function_definition(source: str, name: str) -> str: int response_body_seen, response_body_truncated, phase4_processed; int phase4_strict_abort, phase4_headers_checked, intervention_triggered; int intervention_late, intervention_failed, response_replaced, last_intervention_status; + int phase4_failed, phase4_error_emitting, phase4_started, intervention_body_limit; ngx_http_request_t *r; void *modsec_transaction; u_char *phase4_file_scratch; @@ -93,7 +94,7 @@ def function_definition(source: str, name: str) -> str: size_t phase4_body_limit; } ngx_http_modsecurity_conf_t; struct ngx_http_request_s { - int header_sent; + int header_sent, filter_finalize; struct ngx_http_request_s *main; ngx_pool_t *pool; ngx_connection_t *connection; @@ -103,6 +104,7 @@ def function_definition(source: str, name: str) -> str: #define NGX_OK 0 #define NGX_ERROR (-1) #define NGX_AGAIN (-2) +#define NGX_DECLINED (-5) #define NGX_HTTP_INTERNAL_SERVER_ERROR 500 #define NGX_HTTP_FORBIDDEN 403 #define NGX_HTTP_MODSEC_PHASE4_MODE_OFF 0 @@ -118,9 +120,12 @@ def function_definition(source: str, name: str) -> str: static void ngx_http_modsecurity_cleanup(void *data) { (void)data; } static int allocation_calls, read_calls, append_calls, process_calls; static int fail_allocation, fail_read, short_read, fail_append, append_result = 1; -static int engine_result = 1, native_result, native_late, native_failed; +static int engine_result = 1, native_result, native_late, native_failed, native_body_limit; static int log_result, event_calls, pcre_enter, pcre_leave; static int forward_calls, forwarded_links, forward_result; +static int finalize_calls, emulate_core_response, core_response_result; +static ngx_chain_t *core_response; +ngx_int_t ngx_http_modsecurity_body_filter(ngx_http_request_t *, ngx_chain_t *); static size_t read_max, appended_bytes; static u_char scratch[32768], inspected[200000]; static const char *last_action, *last_reason; @@ -176,12 +181,20 @@ def function_definition(source: str, name: str) -> str: if (ctx == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; ctx->intervention_late = native_late; ctx->intervention_failed = native_failed; + ctx->intervention_body_limit = native_body_limit; return native_result; } static ngx_int_t ngx_http_filter_finalize_request(ngx_http_request_t *r, int *module, int status) { (void)module; + ++finalize_calls; + if (!r->header_sent && emulate_core_response) { + int previous = r->filter_finalize; + r->filter_finalize = 1; + core_response_result = ngx_http_modsecurity_body_filter(r, core_response); + r->filter_finalize = previous; + } return r->header_sent ? NGX_ERROR : status; } static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, @@ -225,7 +238,7 @@ def function_definition(source: str, name: str) -> str: CHECK(argc == 2); memset(payload, 'A', sizeof(payload)); request.pool = &pool; request.connection = &connection; - request.ctx = &ctx; request.conf = &conf; request.main = &request; ctx.r = &request; + request.ctx = &ctx; request.conf = &conf; request.main = &request; ctx.r = &request; ctx.modsec_transaction = &ctx; buffer.file = &file; buffer.in_file = 1; buffer.file_last = sizeof(payload); if (strcmp(argv[1], "request-ownership") == 0) { @@ -253,7 +266,7 @@ def function_definition(source: str, name: str) -> str: CHECK(process_calls == 1 && ctx.phase4_processed && forward_calls == 2); } else if (strcmp(argv[1], "all-modes-large") == 0) { for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, 1048577U, &allowed) == NGX_OK); CHECK(allowed == 1048577U && ctx.response_body_bytes_seen == allowed); @@ -263,7 +276,7 @@ def function_definition(source: str, name: str) -> str: } else if (strcmp(argv[1], "all-modes-multiple") == 0) { size_t chunk; for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; for (chunk = 0; chunk < 3; ++chunk) { CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, 1048576U, &allowed) == NGX_OK); @@ -276,7 +289,7 @@ def function_definition(source: str, name: str) -> str: } } else if (strcmp(argv[1], "legacy-boundary") == 0) { for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, 1048575U, &allowed) == NGX_OK); ctx.response_body_bytes_inspected += allowed; @@ -295,7 +308,7 @@ def function_definition(source: str, name: str) -> str: buffer.in_file = 0; buffer.memory = 1; buffer.pos = payload; buffer.last = payload + sizeof(payload); for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; append_calls = 0; appended_bytes = 0; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, &conf, &buffer) == NGX_OK); @@ -309,7 +322,7 @@ def function_definition(source: str, name: str) -> str: } } else if (strcmp(argv[1], "overflow") == 0) { for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; ctx.response_body_bytes_seen = SIZE_MAX - 1U; ctx.response_body_bytes_inspected = SIZE_MAX - 1U; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, @@ -323,7 +336,7 @@ def function_definition(source: str, name: str) -> str: } } else if (strcmp(argv[1], "invalid-accounting") == 0) { for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; ctx.response_body_bytes_inspected = 1U; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, 1U, &allowed) == NGX_ERROR); @@ -358,7 +371,7 @@ def function_definition(source: str, name: str) -> str: conf.phase4_body_limit = 65536U; buffer.memory = 1; buffer.pos = payload; buffer.last = payload + sizeof(payload); for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; append_calls = 0; appended_bytes = 0; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, &conf, &buffer) == NGX_OK); @@ -374,7 +387,7 @@ def function_definition(source: str, name: str) -> str: } else if (strcmp(argv[1], "file-over-legacy-limit") == 0) { conf.phase4_body_limit = 65536U; for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; read_calls = 0; append_calls = 0; allocation_calls = 0; appended_bytes = 0; read_max = 0; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, @@ -422,7 +435,7 @@ def function_definition(source: str, name: str) -> str: } else if (strcmp(argv[1], "append-error") == 0) { fail_append = 1; for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, &conf, &buffer) == NGX_ERROR); CHECK(ctx.response_body_bytes_inspected == 0U); @@ -433,7 +446,7 @@ def function_definition(source: str, name: str) -> str: buffer.in_file = 0; buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 10; for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; append_calls = 0; appended_bytes = 0; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, &conf, &buffer) == NGX_OK); @@ -447,7 +460,7 @@ def function_definition(source: str, name: str) -> str: CHECK(ctx.phase4_processed); } else if (strcmp(argv[1], "engine-error") == 0) { for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; request.header_sent = 1; connection.error = 0; engine_result = -1; CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == NGX_ERROR); CHECK(ctx.phase4_processed && ctx.intervention_triggered && connection.error); @@ -503,6 +516,97 @@ def function_definition(source: str, name: str) -> str: CHECK(buffer.pos == buffer.last && !buffer.in_file); CHECK(buffer.file_last == buffer.file_pos); CHECK(append_calls == 0 && process_calls == 0 && forwarded_links == 2); + } else if (strcmp(argv[1], "empty-memory-eos") == 0) { + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; + chain.next = NULL; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + conf.phase4_mode = (unsigned)i; + process_calls = 0; forward_calls = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(process_calls == 1 && append_calls == 0 && forward_calls == 1); + CHECK(ctx.phase4_processed && !ctx.phase4_failed); + CHECK(ctx.response_body_bytes_seen == 0U && !ctx.response_body_seen); + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(process_calls == 1 && forward_calls == 2); + } + } else if (strcmp(argv[1], "memory-missing-end") == 0) { + buffer.in_file = 0; buffer.memory = 1; buffer.pos = payload; + request.header_sent = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(append_calls == 0 && forward_calls == 0 && ctx.phase4_failed); + } else if (strcmp(argv[1], "invalid-append-results") == 0) { + const int invalid[] = {-2, -1, 2, 99}; + size_t invalid_index; + request.header_sent = 1; chain.next = NULL; + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; + for (i = 0; i <= 2; ++i) { + conf.phase4_mode = (unsigned)i; + for (invalid_index = 0; invalid_index < sizeof(invalid) / sizeof(invalid[0]); ++invalid_index) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + append_calls = 0; appended_bytes = 0; connection.error = 0; + append_result = invalid[invalid_index]; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(append_calls == 1 && process_calls == 0 && forward_calls == 0); + CHECK(ctx.phase4_failed && connection.error); + } + } + } else if (strcmp(argv[1], "failure-reentry") == 0) { + request.header_sent = 1; chain.next = NULL; + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; + buffer.pos = payload; buffer.last = payload + 10; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + conf.phase4_mode = (unsigned)i; + process_calls = 0; append_calls = 0; appended_bytes = 0; + engine_result = 0; connection.error = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(ctx.phase4_failed && connection.error); + engine_result = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_ERROR); + CHECK(process_calls == 1 && append_calls == 1 && forward_calls == 0); + } + } else if (strcmp(argv[1], "strict-reentry") == 0 || + strcmp(argv[1], "sink-failure-reentry") == 0) { + request.header_sent = 1; chain.next = NULL; + buffer.in_file = 0; buffer.memory = 1; + buffer.pos = payload; buffer.last = payload + 10; + native_result = -1; native_late = 1; ctx.last_intervention_status = 403; + if (strcmp(argv[1], "strict-reentry") == 0) conf.phase4_mode = 2; + else log_result = NGX_ERROR; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(ctx.phase4_failed && connection.error && event_calls == 1); + native_result = 0; log_result = NGX_OK; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_ERROR); + CHECK(append_calls == 1 && process_calls == 0 && forward_calls == 0); + CHECK(event_calls == 1); + } else if (strcmp(argv[1], "core-error-once") == 0) { + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; + buffer.pos = payload; buffer.last = payload + 10; + chain.next = NULL; final.last_buf = 1; + engine_result = 0; emulate_core_response = 1; core_response = &tail; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == 500); + CHECK(finalize_calls == 1 && core_response_result == NGX_OK); + CHECK(process_calls == 1 && append_calls == 1 && forward_calls == 1); + CHECK(ctx.phase4_failed && !ctx.phase4_error_emitting); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_ERROR); + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(finalize_calls == 1 && forward_calls == 1); + } else if (strcmp(argv[1], "file-chunk-rejection") == 0) { + request.header_sent = 1; + native_result = 403; native_failed = 1; native_body_limit = 1; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + conf.phase4_mode = (unsigned)i; + read_calls = 0; append_calls = 0; appended_bytes = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(ctx.phase4_failed && ctx.intervention_body_limit); + CHECK(read_calls == 1 && append_calls == 1 && appended_bytes == 32768U); + CHECK(process_calls == 0 && forward_calls == 0 && event_calls == 0); + } } else if (strcmp(argv[1], "late-safe") == 0 || strcmp(argv[1], "late-strict") == 0 || strcmp(argv[1], "late-off") == 0) { request.header_sent = 1; native_result = -1; native_late = 1; ctx.last_intervention_status = 403; @@ -556,20 +660,24 @@ def unit_program(body: str, module: str) -> str: "phase4_handle_intervention", "process_response_intervention", "process_final_response_body", ) - production = "".join( + definitions = [ "\nstatic ngx_int_t\n" + function_definition(body, "ngx_http_modsecurity_" + name) + "\n" for name in functions - ) - production += "\nstatic void\n" + function_definition( + ] + gate = "ngx_http_modsecurity_phase4_error_gate" + if re.search(rf"\b{gate}\s*\(", body): + definitions.insert(0, "\nstatic ngx_int_t\n" + function_definition(body, gate) + "\n") + definitions.append("\nstatic void\n" + function_definition( body, "ngx_http_modsecurity_discard_replaced_response_body" - ) - production += "\nngx_int_t\n" + function_definition( + )) + definitions.append("\nngx_int_t\n" + function_definition( body, "ngx_http_modsecurity_body_filter" - ) + )) + declarations = "\n".join(definition[:definition.index("{")] + ";" for definition in definitions) context_lookup = "\nstatic ngx_http_modsecurity_ctx_t *\n" + function_definition( module, "ngx_http_modsecurity_get_module_ctx" ) + "\n" - return PREAMBLE + context_lookup + production + CASES + return PREAMBLE + declarations + context_lookup + "".join(definitions) + CASES class Phase4RuntimeTests(unittest.TestCase): @@ -613,6 +721,9 @@ def test(self: Phase4RuntimeTests) -> None: "engine-error", "process-zero", "main-eos", "subrequest-eos", "chain-restored", "forward-again", "terminal-trailing", "replaced-body", + "empty-memory-eos", "memory-missing-end", "invalid-append-results", + "failure-reentry", "strict-reentry", "sink-failure-reentry", + "core-error-once", "file-chunk-rejection", "late-safe", "late-strict", "late-off", "native-failure-safe", "native-failure-uncommitted", "uncommitted-deny", "log-failure-safe", ): From 776de3a06b63700dd6d9f499397c9365d709ae23 Mon Sep 17 00:00:00 2001 From: Easton97-Jens <66330090+Easton97-Jens@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:07:25 +0100 Subject: [PATCH 35/40] Refactor intervention initialization for Sonar complexity --- src/ngx_http_modsecurity_module.c | 23 +++++++++++++++-------- tests/test_intervention_runtime.py | 3 +++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index efc24d4b..0c5e4ba0 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -293,6 +293,20 @@ ngx_http_modsecurity_defer_late_phase4_intervention(ngx_http_request_t *r, return 1; } +static void +ngx_http_modsecurity_reset_intervention(ngx_http_modsecurity_ctx_t *ctx) +{ + if (ctx == NULL) { + return; + } + ctx->intervention_late = 0; + ctx->intervention_failed = 0; + ctx->intervention_body_limit = 0; + ctx->intervention_redirect_location_installed = 0; + ctx->last_intervention_status = 0; + ngx_str_null(&ctx->last_intervention_log); +} + int ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_request_t *r, ngx_int_t early_log) @@ -305,14 +319,7 @@ ngx_http_modsecurity_process_intervention (Transaction *transaction, ctx = ngx_http_modsecurity_get_module_ctx(r); mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); - if (ctx != NULL) { - ctx->intervention_late = 0; - ctx->intervention_failed = 0; - ctx->intervention_body_limit = 0; - ctx->intervention_redirect_location_installed = 0; - ctx->last_intervention_status = 0; - ngx_str_null(&ctx->last_intervention_log); - } + ngx_http_modsecurity_reset_intervention(ctx); if (ctx == NULL || mcf == NULL || transaction == NULL) { if (ctx != NULL) ctx->intervention_failed = 1; return r->header_sent ? NGX_ERROR : NGX_HTTP_INTERNAL_SERVER_ERROR; diff --git a/tests/test_intervention_runtime.py b/tests/test_intervention_runtime.py index dcd4848c..714e1825 100644 --- a/tests/test_intervention_runtime.py +++ b/tests/test_intervention_runtime.py @@ -491,6 +491,9 @@ def unit_program(module: str) -> str: ): if re.search(rf"\b{helper}\s*\(", module): helpers += "\nstatic ngx_int_t\n" + function_definition(module, helper) + reset = "ngx_http_modsecurity_reset_intervention" + if re.search(rf"\b{reset}\s*\(", module): + helpers += "\nstatic void\n" + function_definition(module, reset) helpers += "\nint\n" + function_definition( module, "ngx_http_modsecurity_process_intervention" ) From 166907739461927848abfd6d36a288179d97e97a Mon Sep 17 00:00:00 2001 From: Easton97-Jens Date: Sun, 15 Nov 2026 00:09:24 +0100 Subject: [PATCH 36/40] Port connector review fixes for audit, Safe delivery and native limits --- CHANGES | 28 +- README.md | 3 +- docs/phase4-handling.en.md | 40 +- src/ngx_http_modsecurity_access.c | 166 +++-- src/ngx_http_modsecurity_body_filter.c | 245 ++++---- src/ngx_http_modsecurity_common.h | 29 +- src/ngx_http_modsecurity_header_filter.c | 26 +- src/ngx_http_modsecurity_log.c | 26 +- src/ngx_http_modsecurity_log_metadata.h | 362 +++++++++++ src/ngx_http_modsecurity_module.c | 273 ++++++++- tests/README.md | 6 +- tests/modsecurity-config-audit-redirect.t | 80 +++ tests/modsecurity-config-custom-error-page.t | 112 +++- tests/test_header_protocol_runtime.py | 198 ++++++ tests/test_intervention_runtime.py | 109 +++- tests/test_lifecycle_runtime.py | 603 +++++++++++++++---- tests/test_log_metadata_runtime.py | 346 +++++++++++ tests/test_phase4_runtime.py | 256 ++++++-- 18 files changed, 2510 insertions(+), 398 deletions(-) create mode 100644 src/ngx_http_modsecurity_log_metadata.h create mode 100644 tests/modsecurity-config-audit-redirect.t create mode 100644 tests/test_header_protocol_runtime.py create mode 100644 tests/test_log_metadata_runtime.py diff --git a/CHANGES b/CHANGES index addeff15..96b29de9 100644 --- a/CHANGES +++ b/CHANGES @@ -6,7 +6,7 @@ Unreleased leaves ModSecurity response inspection and native interventions active. [PR #377 - @Easton97-Jens] - feat: handle late deny and redirect interventions explicitly. After - headers are sent, safe logs and continues when event logging succeeds; + headers are sent, safe diagnoses optional event-log failures and continues; strict terminates the response. A late intervention cannot replace an HTTP status or body already sent. [PR #377 - @Easton97-Jens] @@ -27,8 +27,9 @@ Unreleased [PR #377 - @Easton97-Jens] - fix: inspect memory and file-backed response buffers, reading file data in bounded 32 KiB chunks while preserving the outgoing chain. Check byte - accounting, engine results, allocations and file reads; genuine errors - remain failures in every mode, including safe. + ranges, engine results, allocations and file reads. Saturate cumulative + counters without imposing a SIZE_MAX response limit; technical body + processing errors remain failures in every mode, including safe. [PR #377 - @Easton97-Jens] - fix: finalize response-body processing once per transaction and respect main-request/subrequest ownership and end-of-stream markers. Correct @@ -53,8 +54,22 @@ Unreleased valid empty memory/EOS buffers without repeating phase completion. [PR #377 - @Easton97-Jens] - fix: preserve request-phase deny/redirect decisions across error_page - routing and start a fresh transaction for an internal target location. - Check request-phase, file-ingestion and append results independently. + routing. Reuse the original transaction for inherited identical policy, + retire it when target policy changes, and audit every transaction owner. + Preserve only the captured local block error page without reinspection; + ordinary internal targets still run their own rules. + [PR #377 - @Easton97-Jens] + - fix: stream file-backed request bodies through bounded 32 KiB reads and + the same native 0/1 append contract as memory bodies. Preserve ProcessPartial + and pending native Reject through P2 dispatch without reopening temp paths. + [PR #377 - @Easton97-Jens] + - fix: identify native response-limit Reject at the actual append boundary, + independent of English log text. Suppress drained replacement data chains + and forward only fresh control markers, with once-only terminal delivery. + [PR #377 - @Easton97-Jens] + - fix: parse complete, unambiguous engine metadata records for rule IDs and + redacted presence summaries. Preserve JSON control characters and valid + UTF-8, escape invalid bytes, and avoid sanitizer buffer truncation. [PR #377 - @Easton97-Jens] - fix: attempt native audit logging once and retain failed native results across re-entry without attempting another client response. @@ -63,6 +78,9 @@ Unreleased partially created native owners, make cleanup idempotent and terminate length-delimited custom transaction IDs before passing them to the SDK. [PR #377 - @Easton97-Jens] + - fix: avoid synthetic Transfer-Encoding inspection headers on HTTP/2 and + HTTP/3, and validate response status before narrowing to the native int. + [PR #377 - @Easton97-Jens] - test: extend regressions for modes, log inheritance, engine MIME selection, Reject/ProcessPartial limits, large responses, file buffers, overflow, subrequests, redirects, processing failures and proxy/HTTP/2 fixtures. diff --git a/README.md b/README.md index fd5c4530..f10df0c9 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,8 @@ modsecurity_phase4_mode **default:** *off* Controls the connector's additional phase 4 intervention policy. `safe` logs a -late intervention as `log_only`; `strict` aborts the connection once headers +late rule intervention as `log_only` and continues even if optional JSON +logging fails (with an nginx diagnostic); `strict` aborts the connection once headers are sent. Before headers are sent, both modes can apply the intervention's HTTP status. Neither mode guarantees a clean 403 after headers have been sent. diff --git a/docs/phase4-handling.en.md b/docs/phase4-handling.en.md index f1719de5..6faea1c7 100644 --- a/docs/phase4-handling.en.md +++ b/docs/phase4-handling.en.md @@ -13,6 +13,8 @@ The response-limit ownership update follows [commit 820b6975495bdf0f90aca67eee86e27a3b7d329b](https://github.com/Easton97-Jens/ModSecurity-conector/blob/820b6975495bdf0f90aca67eee86e27a3b7d329b/docs/phase4-mode-budget.md). Native-result, response completion, and lifecycle corrections follow [commit f5523da040bd6325b369269ada074be6ba069c4f](https://github.com/Easton97-Jens/ModSecurity-conector/tree/f5523da040bd6325b369269ada074be6ba069c4f/connectors/nginx/src). +The review follow-up follows +[commit d420a12c4155a4bc2b9ea63ee5cef633085705e7](https://github.com/Easton97-Jens/ModSecurity-conector/tree/d420a12c4155a4bc2b9ea63ee5cef633085705e7/connectors/nginx/src). These changes are adapted to this standalone module and preserve its phase 4 JSON-lines schema. The source repository's multi-connector runtime and run evidence are not imported. @@ -60,11 +62,13 @@ For the additional policy in `safe` and `strict`: | Header state | `safe` | `strict` | | --- | --- | --- | | Headers not sent | Apply the intervention's status/redirect through the normal path; logged as `deny_status` | Same | -| Headers already sent | Log the rule intervention as `log_only` and continue if logging succeeds | Log `connection_abort` and terminate the response | +| Headers already sent | Attempt `log_only` logging and continue; diagnose optional event-log failures | Log `connection_abort` and terminate the response | `safe` only downgrades late **rule interventions**. An engine API failure, invalid -buffer, failed file read, allocation failure, counter overflow, or final -processing failure remains a failure. It is not converted into successful +buffer, failed file read, body-processing allocation failure, or final +processing failure remains a failure. After headers are sent, optional phase 4 event-log allocation and write +failures are diagnosed but do not abort Safe delivery. Before commitment, +event-log failures retain the existing error path. It is not converted into successful forwarding. Native response-limit Reject is not a rule-policy downgrade. Once a response has failed, later data callbacks and NULL-input flushes cannot @@ -95,7 +99,8 @@ The 1 MiB value is an explicit example setting, not a new engine default. `ProcessPartial` selects inspection of the portion within the engine limit; `Reject` is the alternative engine action when rejection is required. Mode selection does not override that engine policy. Native Reject is recognized -by its exact rule-free engine signature while processing a response body. It +at a valid append boundary before EOS: a rule-free, disruptive native 403 +without a redirect. Recognition does not depend on the diagnostic language. It produces 403 before headers are sent, or fails the transport after commitment, in all modes including `safe`. It does not produce a `phase4_intervention` rule-policy event; a fixed diagnostic is written to the nginx error log. @@ -105,8 +110,9 @@ value has no enforcement effect in any valid mode. A successful configuration load therefore does not establish the old connector inspection limit. Migrate that policy to the engine settings above. -Checked cumulative byte accounting still rejects overflow beyond `SIZE_MAX` -in all modes. Genuine processing and file-read failures remain failures; +Cumulative byte counters saturate at `SIZE_MAX` without limiting delivery. +Large file buffers are streamed without converting their entire length to +`size_t`, including on 32-bit hosts. Genuine processing and file-read failures remain failures; nonfatal engine `ProcessPartial` ingestion is not a connector failure. The connector does not globally buffer responses or reorder nginx body @@ -175,7 +181,11 @@ The `actual_action` values remain `deny_status`, `log_only`, and `connection_abort`; `mode` uses the current mode names. The intervention message is redacted rather than copied from the engine, and response-body payload is not added to this event. nginx's error log may also contain -diagnostics. +diagnostics. Rule IDs and presence flags come only from complete, unambiguous +engine metadata records; reflected delimiter-like payloads are not trusted. +JSON preserves control characters through escapes and copies valid UTF-8; +invalid bytes are escaped individually. Redacted summaries have fixed complete +representations rather than variable-sized truncated buffers. `off` uses native intervention handling and does not produce these dedicated policy events. A dedicated intervention event is not a complete inventory of @@ -183,9 +193,19 @@ all engine, buffer, or I/O errors. Native audit logging is attempted once, including re-entry, and its return value is checked. A failed audit attempt is retained as a failure without -trying to generate a new HTTP response. Request inspection also preserves -deny/redirect decisions across `error_page` routing; a new internal target -location starts its own transaction rather than reusing processed state. +trying to generate a new HTTP response. Every request-owned transaction gets +one audit attempt, including superseded owners and cleanup fallback after +NGINX clears the request pool pointer. An internal redirect reuses a transaction +only for identical inherited policy; a changed policy starts a fresh owner and +retires the old response scope. A native request block can render its captured +local `error_page` target without reblocking that same error response. Merely +setting `error_page` does not exempt an unrelated target from inspection. + +File-backed request bodies use their existing NGINX file descriptor and bounded +32 KiB reads, with the same 0/1 append contract as memory bodies. ProcessPartial +remains valid, and collected native request-limit Reject is retained until P2 +dispatch. Replaced response data is consumed without forwarding empty producer +buffers to the writer; terminal control markers are emitted once. ## 7) Configuration examples and verification diff --git a/src/ngx_http_modsecurity_access.c b/src/ngx_http_modsecurity_access.c index 6c2c1f4b..0a860a83 100644 --- a/src/ngx_http_modsecurity_access.c +++ b/src/ngx_http_modsecurity_access.c @@ -14,6 +14,7 @@ */ #include +#include #ifndef MODSECURITY_DDEBUG #define MODSECURITY_DDEBUG 0 @@ -36,6 +37,91 @@ ngx_http_modsecurity_request_failure(ngx_http_request_t *r, return NGX_HTTP_INTERNAL_SERVER_ERROR; } + +static int +ngx_http_modsecurity_request_intervention(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_int_t early_log) +{ + int result; + + ctx->request_intervention_active = 1; + result = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, + r, early_log); + ctx->request_intervention_active = 0; + return result; +} + + +static ngx_int_t +ngx_http_modsecurity_append_request_chunk(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, const u_char *data, size_t size) +{ + int result; + + result = msc_append_request_body(ctx->modsec_transaction, data, size); + /* Zero is the engine's ProcessPartial result, for memory and file data. */ + if ((result != 0 && result != 1) || + ngx_http_modsecurity_capture_request_body_intervention(r) != NGX_OK) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request body append failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } + return NGX_OK; +} + + +static ngx_int_t +ngx_http_modsecurity_inspect_request_body_file(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) +{ + ngx_file_t *file = &r->request_body->temp_file->file; + ngx_file_info_t info; + off_t size = r->request_body->temp_file->offset; + off_t offset = 0; + size_t amount; + ssize_t read_size; + ngx_int_t result; + + /* Inspect the descriptor NGINX owns. The file-path SDK helper cannot + * distinguish ProcessPartial zero from its file I/O/allocation failures. */ + if (file->fd == NGX_INVALID_FILE || size < 0 || + ngx_fd_info(file->fd, &info) == NGX_FILE_ERROR || + !ngx_is_file(&info) || ngx_file_size(&info) != size) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: invalid request body file descriptor or size"); + return ngx_http_modsecurity_request_failure(r, ctx); + } + if (size > 0 && ctx->request_file_scratch == NULL) { + ctx->request_file_scratch = ngx_pnalloc(r->pool, 32768U); + if (ctx->request_file_scratch == NULL) { + return ngx_http_modsecurity_request_failure(r, ctx); + } + } + while (offset < size) { + amount = (uintmax_t) (size - offset) > 32768U + ? 32768U : (size_t) (size - offset); + read_size = ngx_read_file(file, ctx->request_file_scratch, amount, offset); + if (read_size < 0 || (size_t) read_size != amount) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request body file read is short or failed"); + return ngx_http_modsecurity_request_failure(r, ctx); + } + result = ngx_http_modsecurity_append_request_chunk(r, ctx, + ctx->request_file_scratch, amount); + if (result != NGX_OK) { + return result; + } + offset += (off_t) amount; + } + if (ngx_fd_info(file->fd, &info) == NGX_FILE_ERROR || + !ngx_is_file(&info) || ngx_file_size(&info) != size) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: request body temp file size changed unexpectedly"); + return ngx_http_modsecurity_request_failure(r, ctx); + } + return NGX_OK; +} + void ngx_http_modsecurity_request_read(ngx_http_request_t *r) { @@ -47,7 +133,7 @@ ngx_http_modsecurity_request_read(ngx_http_request_t *r) r->main->count--; #endif - if (ctx->waiting_more_body) + if (ctx != NULL && ctx->waiting_more_body) { ctx->waiting_more_body = 0; r->write_event_handler = ngx_http_core_run_phases; @@ -65,6 +151,9 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) ngx_http_modsecurity_conf_t *mcf; mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + if (ngx_http_modsecurity_prepare_location(r, mcf) != NGX_OK) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } if (mcf == NULL || mcf->enable != 1) { dd("ModSecurity not enabled... returning"); return NGX_DECLINED; @@ -81,9 +170,9 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) dd("catching a new _access_ phase handler"); - /* Internal redirects clear the current location's module context. - * Cleanup recovery belongs to finalization, not access initialization. */ - ctx = ngx_http_get_module_ctx(r, ngx_http_modsecurity_module); + /* Reuse only the same request's equivalent policy, or its own exact + * configured native-block error page. Other targets get a new owner. */ + ctx = ngx_http_modsecurity_get_module_ctx(r); if (ctx != NULL && ctx->r != r) { ctx = NULL; } @@ -201,7 +290,7 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) * */ dd("Processing intervention with the connection information filled in"); - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 1); + ret = ngx_http_modsecurity_request_intervention(r, ctx, 1); if (ret < 0) { return ngx_http_modsecurity_request_failure(r, ctx); } @@ -258,7 +347,7 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) } dd("Processing intervention with the transaction information filled in (uri, method and version)"); - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 1); + ret = ngx_http_modsecurity_request_intervention(r, ctx, 1); if (ret < 0) { return ngx_http_modsecurity_request_failure(r, ctx); } @@ -315,7 +404,7 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) return ngx_http_modsecurity_request_failure(r, ctx); } dd("Processing intervention with the request headers information filled in"); - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 1); + ret = ngx_http_modsecurity_request_intervention(r, ctx, 1); if (ret < 0) { return ngx_http_modsecurity_request_failure(r, ctx); } @@ -416,7 +505,7 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) r->write_event_handler = ngx_http_core_run_phases; - ngx_chain_t *chain = r->request_body->bufs; + ngx_chain_t *chain = r->request_body != NULL ? r->request_body->bufs : NULL; /** * TODO: Speed up the analysis by sending chunk while they arrive. @@ -427,25 +516,11 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) * function. */ - if (r->request_body->temp_file != NULL) { - ngx_str_t file_path = r->request_body->temp_file->file.name; - const char *file_name = ngx_str_to_char(file_path, r->pool); - if (file_name == (char*)-1) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - /* - * Request body was saved to a file, probably we don't have a - * copy of it in memory. - */ - dd("request body inspection: file -- %s", file_name); - - ret = msc_request_body_from_file(ctx->modsec_transaction, file_name); - if (ret != 1) { - ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "ModSecurity: request body file processing failed"); - return ngx_http_modsecurity_request_failure(r, ctx); + if (r->request_body != NULL && r->request_body->temp_file != NULL) { + ret = ngx_http_modsecurity_inspect_request_body_file(r, ctx); + if (ret != NGX_OK) { + return ret; } - already_inspected = 1; } else { dd("inspection request body in memory."); @@ -453,37 +528,24 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) while (chain && !already_inspected) { - u_char *data = chain->buf->pos; - - ret = msc_append_request_body(ctx->modsec_transaction, data, - chain->buf->last - data); - /* Direct append zero is the engine's ProcessPartial result. */ - if (ret != 0 && ret != 1) { - ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "ModSecurity: request body append failed"); + if (chain->buf == NULL) { return ngx_http_modsecurity_request_failure(r, ctx); } - + if (!ngx_buf_special(chain->buf)) { + if (chain->buf->pos == NULL || chain->buf->last == NULL || + chain->buf->last < chain->buf->pos) { + return ngx_http_modsecurity_request_failure(r, ctx); + } + ret = ngx_http_modsecurity_append_request_chunk(r, ctx, + chain->buf->pos, (size_t) (chain->buf->last - chain->buf->pos)); + if (ret != NGX_OK) { + return ret; + } + } if (chain->buf->last_buf) { break; } chain = chain->next; - -/* XXX: chains are processed one-by-one, maybe worth to pass all chains and then call intervention() ? */ - - /** - * ModSecurity may perform stream inspection on this buffer, - * it may ask for a intervention in consequence of that. - * - */ - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); - if (ret < 0) { - return ngx_http_modsecurity_request_failure(r, ctx); - } - if (ret > 0) { - ctx->intervention_triggered = 1; - return ret; - } } /** @@ -505,7 +567,7 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) } ctx->request_body_processed = 1; - ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); + ret = ngx_http_modsecurity_request_intervention(r, ctx, 0); if (ret < 0) { return ngx_http_modsecurity_request_failure(r, ctx); } diff --git a/src/ngx_http_modsecurity_body_filter.c b/src/ngx_http_modsecurity_body_filter.c index bea20963..a9752611 100644 --- a/src/ngx_http_modsecurity_body_filter.c +++ b/src/ngx_http_modsecurity_body_filter.c @@ -22,6 +22,7 @@ #include "ddebug.h" #include "ngx_http_modsecurity_common.h" +#include "ngx_http_modsecurity_log_metadata.h" /* Bounded, reusable materialization for file-only response buffers. */ #define NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK 32768U @@ -30,10 +31,7 @@ static ngx_http_output_body_filter_pt ngx_http_next_body_filter; static ngx_int_t ngx_http_modsecurity_process_response_intervention(ngx_http_request_t *r, ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf); static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason); static ngx_int_t ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf); -static void ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst); -static void ngx_http_modsecurity_extract_rule_id(ngx_pool_t *pool, ngx_str_t *intervention, ngx_str_t *rule_id); static ngx_str_t ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in); -static ngx_str_t ngx_http_modsecurity_sanitize_intervention(ngx_pool_t *pool, ngx_str_t in); /* XXX: check behaviour on few body filters installed */ ngx_int_t @@ -67,6 +65,7 @@ ngx_http_modsecurity_response_body_failure(ngx_http_request_t *r, { ngx_int_t ret; + ctx->phase4_append_active = 0; if (ctx->phase4_failed) { r->connection->error = 1; return NGX_ERROR; @@ -85,9 +84,8 @@ ngx_http_modsecurity_response_body_failure(ngx_http_request_t *r, return ret; } -/* libModSecurity owns response inspection limits in every Phase-4 mode. - * Keep checked accounting without imposing the legacy connector budget; - * independent host, file-buffer and allocation limits remain active. */ +/* These are diagnostic counters, not inspection limits. Saturation must not + * interrupt a valid large response, including on a 32-bit size_t build. */ static ngx_int_t ngx_http_modsecurity_plan_limited_response_body( ngx_http_modsecurity_ctx_t *ctx, ngx_http_modsecurity_conf_t *mcf, @@ -115,10 +113,9 @@ ngx_http_modsecurity_plan_limited_response_body( } if (len > SIZE_MAX - ctx->response_body_bytes_seen) { ctx->response_body_bytes_seen = SIZE_MAX; - ctx->response_body_truncated = 1; - return NGX_ERROR; + } else { + ctx->response_body_bytes_seen += len; } - ctx->response_body_bytes_seen += len; *allowed = len; return NGX_OK; } @@ -129,17 +126,25 @@ ngx_http_modsecurity_append_response_body_chunk( { int ret; + ctx->phase4_append_active = 0; if (bytes == 0U) { return NGX_OK; } - if (data == NULL || ctx->response_body_bytes_inspected > SIZE_MAX - bytes) { + if (data == NULL) { return NGX_ERROR; } ret = msc_append_response_body(ctx->modsec_transaction, data, bytes); if (ret != 0 && ret != 1) { return NGX_ERROR; } - ctx->response_body_bytes_inspected += bytes; + if (bytes > SIZE_MAX - ctx->response_body_bytes_inspected) { + ctx->response_body_bytes_inspected = SIZE_MAX; + } else { + ctx->response_body_bytes_inspected += bytes; + } + /* Keep the accepted append boundary live until its immediate collection. + * The engine evaluates response rules only in process_response_body(). */ + ctx->phase4_append_active = 1; return NGX_OK; } @@ -163,8 +168,7 @@ ngx_http_modsecurity_append_file_response_body(ngx_http_request_t *r, ngx_buf_t *buffer) { uintmax_t file_length; - size_t allowed; - size_t remaining; + uintmax_t remaining; size_t chunk; off_t file_offset; ssize_t read_count; @@ -175,12 +179,10 @@ ngx_http_modsecurity_append_file_response_body(ngx_http_request_t *r, return NGX_ERROR; } file_length = (uintmax_t)buffer->file_last - (uintmax_t)buffer->file_pos; - if (file_length > (uintmax_t)SIZE_MAX || - ngx_http_modsecurity_plan_limited_response_body(ctx, mcf, - (size_t)file_length, &allowed) != NGX_OK) { + if (ctx == NULL || mcf == NULL) { return NGX_ERROR; } - if (allowed == 0U) { + if (file_length == 0U) { return NGX_OK; } if (buffer->file == NULL) { @@ -194,10 +196,10 @@ ngx_http_modsecurity_append_file_response_body(ngx_http_request_t *r, } } file_offset = buffer->file_pos; - remaining = allowed; + remaining = file_length; while (remaining > 0U) { chunk = remaining > NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK - ? NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK : remaining; + ? NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK : (size_t)remaining; read_count = ngx_read_file(buffer->file, ctx->phase4_file_scratch, chunk, file_offset); if (read_count < 0 || (size_t)read_count != chunk) { @@ -206,7 +208,7 @@ ngx_http_modsecurity_append_file_response_body(ngx_http_request_t *r, "ModSecurity: file-backed response body read is short or failed"); return NGX_ERROR; } - if (ngx_http_modsecurity_append_response_body_chunk(ctx, + if (ngx_http_modsecurity_append_limited_response_body(ctx, mcf, ctx->phase4_file_scratch, chunk) != NGX_OK) { return NGX_ERROR; } @@ -257,6 +259,7 @@ ngx_http_modsecurity_process_response_intervention(ngx_http_request_t *r, ret = ngx_http_modsecurity_process_intervention(ctx->modsec_transaction, r, 0); + ctx->phase4_append_active = 0; if (ret == 0) { return NGX_OK; } @@ -289,6 +292,7 @@ ngx_http_modsecurity_process_final_response_body(ngx_http_request_t *r, if (ctx->phase4_processed) { return NGX_OK; } + ctx->phase4_append_active = 0; ctx->phase4_processed = 1; old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); ret = msc_process_response_body(ctx->modsec_transaction); @@ -346,19 +350,61 @@ ngx_http_modsecurity_response_header_sanity(ngx_http_request_t *r, } #endif -static void -ngx_http_modsecurity_discard_replaced_response_body(ngx_chain_t *in) +static ngx_int_t +ngx_http_modsecurity_discard_replaced_response_body(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_chain_t *in) { ngx_chain_t *chain; + ngx_chain_t *out = NULL; + ngx_chain_t **tail = &out; for (chain = in; chain != NULL; chain = chain->next) { - if (chain->buf == NULL) { + ngx_buf_t *source = chain->buf; + ngx_chain_t *control; + + if (source == NULL) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + /* Consume producer-owned bytes without forwarding its now-empty + * memory/file link: NGINX's writer rejects those non-special links. */ + if (ngx_buf_in_memory(source)) { + source->pos = source->last; + } + if (source->in_file) { + source->file_pos = source->file_last; + } + if (ctx->replacement_body_terminal_sent || + !(source->flush || source->sync || source->last_buf || + source->last_in_chain)) { continue; } - chain->buf->pos = chain->buf->last; - chain->buf->in_file = 0; - chain->buf->file_last = chain->buf->file_pos; + control = ngx_alloc_chain_link(r->pool); + if (control == NULL) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + control->buf = ngx_calloc_buf(r->pool); + if (control->buf == NULL) { + return ngx_http_modsecurity_response_body_failure(r, ctx); + } + control->next = NULL; + control->buf->flush = source->flush; + control->buf->last_buf = source->last_buf && r == r->main; + control->buf->last_in_chain = source->last_in_chain || + (source->last_buf && r != r->main); + control->buf->sync = source->sync || control->buf->last_in_chain; + *tail = control; + tail = &control->next; + if (control->buf->last_buf || + (r != r->main && control->buf->last_in_chain)) { + /* Downstream can retain this chain on NGX_AGAIN. Claim EOS + * before dispatch so a later callback cannot duplicate it. */ + ctx->replacement_body_terminal_sent = 1; + } + } + if (out == NULL && !r->buffered && !r->connection->buffered) { + return NGX_OK; } + return ngx_http_next_body_filter(r, out); } ngx_int_t @@ -377,11 +423,10 @@ ngx_http_modsecurity_body_filter(ngx_http_request_t *r, ngx_chain_t *in) if (ret != NGX_OK) { return ret == NGX_DECLINED ? ngx_http_next_body_filter(r, in) : ret; } - if (in == NULL) { - return ngx_http_next_body_filter(r, in); - } if (ctx->response_replaced) { - ngx_http_modsecurity_discard_replaced_response_body(in); + return ngx_http_modsecurity_discard_replaced_response_body(r, ctx, in); + } + if (in == NULL) { return ngx_http_next_body_filter(r, in); } if (ctx->intervention_triggered || ctx->phase4_processed) { @@ -477,15 +522,37 @@ ngx_http_modsecurity_phase4_handle_intervention(ngx_http_request_t *r, log_result = ngx_http_modsecurity_phase4_log_event(r, mcf, wanted, "log_only", "response_committed_safe"); if (log_result != NGX_OK) { - return ngx_http_modsecurity_response_body_failure(r, ctx); + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "modsecurity optional phase4 intervention event could not be written"); } return NGX_OK; } +static ngx_int_t +ngx_http_modsecurity_phase4_log_capacity(const size_t *lengths, size_t count, + size_t *capacity) +{ + size_t total = 512U; + size_t i; + + for (i = 0; i < count; i++) { + if (lengths[i] > SIZE_MAX - total) { + return NGX_ERROR; + } + total += lengths[i]; + } + *capacity = total; + return NGX_OK; +} + static ngx_int_t ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurity_conf_t *mcf, const char *wanted, const char *actual, const char *reason) { u_char *p; + u_char *dbuf; + size_t lengths[9]; + size_t need; + ssize_t n; ngx_str_t euri; ngx_str_t emethod; ngx_str_t ect; @@ -493,13 +560,14 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit ngx_str_t erule; ngx_str_t raw_log; ngx_str_t slog; + ngx_str_t nct; const char *mode = "off"; const char *header_sent = r->header_sent ? "true" : "false"; ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); if (mcf == NULL || mcf->phase4_log_file == NULL || mcf->phase4_log_file->fd == NGX_INVALID_FILE) return NGX_OK; ngx_http_modsecurity_json_escape(r->pool, &r->uri, &euri); ngx_http_modsecurity_json_escape(r->pool, &r->method_name, &emethod); - ngx_str_t nct = ngx_http_modsecurity_normalize_content_type(r->pool, r->headers_out.content_type); + nct = ngx_http_modsecurity_normalize_content_type(r->pool, r->headers_out.content_type); ngx_http_modsecurity_json_escape(r->pool, &nct, &ect); if (ctx) { raw_log = ctx->last_intervention_log; @@ -513,8 +581,23 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit } if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_SAFE) mode = "safe"; else if (mcf->phase4_mode == NGX_HTTP_MODSEC_PHASE4_MODE_STRICT) mode = "strict"; - size_t need = 256 + euri.len + emethod.len + ect.len + elog.len + erule.len + ngx_strlen(mode) + ngx_strlen(wanted) + ngx_strlen(actual) + ngx_strlen(reason); - u_char *dbuf = ngx_pnalloc(r->pool, need); + if (euri.data == NULL || emethod.data == NULL || nct.data == NULL || + ect.data == NULL || elog.data == NULL || erule.data == NULL) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "modsecurity phase4 log metadata allocation failed"); + return NGX_ERROR; + } + lengths[0] = euri.len; lengths[1] = emethod.len; + lengths[2] = ect.len; lengths[3] = elog.len; lengths[4] = erule.len; + lengths[5] = ngx_strlen(mode); lengths[6] = ngx_strlen(wanted); + lengths[7] = ngx_strlen(actual); lengths[8] = ngx_strlen(reason); + if (ngx_http_modsecurity_phase4_log_capacity(lengths, + sizeof(lengths) / sizeof(lengths[0]), &need) != NGX_OK) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "modsecurity phase4 log size overflow"); + return NGX_ERROR; + } + dbuf = ngx_pnalloc(r->pool, need); if (dbuf == NULL) { ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, "modsecurity phase4 log allocation failed"); return NGX_ERROR; @@ -524,7 +607,12 @@ ngx_http_modsecurity_phase4_log_event(ngx_http_request_t *r, ngx_http_modsecurit &euri, &emethod, (ngx_uint_t) r->headers_out.status, ctx ? ctx->last_intervention_status : (ngx_int_t) 0, &ect, header_sent, mode, wanted, actual, reason, &elog, &erule); - ssize_t n = ngx_write_fd(mcf->phase4_log_file->fd, dbuf, p - dbuf); + if (p == dbuf + need || p == dbuf || p[-1] != '\n') { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "modsecurity phase4 log serialization was truncated"); + return NGX_ERROR; + } + n = ngx_write_fd(mcf->phase4_log_file->fd, dbuf, p - dbuf); if (n < 0 || (size_t) n != (size_t) (p - dbuf)) { ngx_log_error(NGX_LOG_WARN, r->connection->log, ngx_errno, "modsecurity phase4 log write failed"); @@ -540,7 +628,9 @@ ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in) size_t i; u_char *semi; out = in; - if (out.data == NULL || out.len == 0) return out; + if (out.data == NULL || out.len == 0) { + out.data = (u_char *)""; out.len = 0; return out; + } semi = (u_char *)ngx_strlchr(out.data, out.data + out.len, ';'); if (semi) out.len = semi - out.data; while (out.len > 0) { @@ -551,88 +641,11 @@ ngx_http_modsecurity_normalize_content_type(ngx_pool_t *pool, ngx_str_t in) } out.len--; } + if (out.len == 0) { + out.data = (u_char *)""; return out; + } out.data = ngx_pnalloc(pool, out.len); if (out.data == NULL) { out.len = 0; return out; } for (i = 0; i < out.len; i++) out.data[i] = ngx_tolower(in.data[i]); return out; } - -static ngx_str_t -ngx_http_modsecurity_sanitize_intervention(ngx_pool_t *pool, ngx_str_t in) -{ - static ngx_str_t redacted = { sizeof("redacted") - 1, (u_char *) "redacted" }; - ngx_str_t out = redacted; - u_char *id, *msg, *op; - size_t len = 0; - if (in.data == NULL || in.len == 0) { - return redacted; - } - id = (u_char *)ngx_strstr(in.data, "id \""); - msg = (u_char *)ngx_strstr(in.data, "msg \""); - op = (u_char *)ngx_strstr(in.data, "Operator"); - if (id == NULL && msg == NULL && op == NULL) { - return redacted; - } - len = 9 + (id ? 10 : 0) + (msg ? 12 : 0) + (op ? 10 : 0); - out.data = ngx_pnalloc(pool, len); - if (out.data == NULL) return redacted; - out.len = ngx_snprintf(out.data, len, "id:%s msg:%s op:%s", - id ? "present" : "-", msg ? "present" : "-", op ? "present" : "-") - out.data; - return out; -} - -static void -ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst) -{ - size_t i; - size_t extra = 0; - u_char *d; - if (src == NULL || src->data == NULL) { dst->len=0; dst->data=(u_char*)""; return; } - for (i = 0; i < src->len; i++) { - if (src->data[i] < 0x20 || src->data[i] == '"' || src->data[i] == '\\') { - extra++; - } - } - dst->data = ngx_pnalloc(pool, src->len + extra + 1); - if (dst->data == NULL) { - dst->len = 0; - dst->data = (u_char *)""; - return; - } - d = dst->data; - for (i = 0; i < src->len; i++) { - u_char c = src->data[i]; - if (c == '"' || c == '\\') { *d++='\\'; *d++=c; } - else if (c < 0x20) { *d++=' '; } - else *d++=c; - } - dst->len = d - dst->data; -} - -static void -ngx_http_modsecurity_extract_rule_id(ngx_pool_t *pool, ngx_str_t *intervention, ngx_str_t *rule_id) -{ - size_t i; - rule_id->data = (u_char *)""; - rule_id->len = 0; - if (intervention == NULL || intervention->data == NULL) return; - for (i = 0; i + 4 < intervention->len; i++) { - size_t j; - - if (ngx_strncasecmp(intervention->data + i, (u_char *)"id \"", 4) != 0) continue; - - j = i + 4; - while (j < intervention->len && intervention->data[j] >= '0' && intervention->data[j] <= '9') j++; - if (j <= i + 4 || j >= intervention->len || intervention->data[j] != '"') continue; - - rule_id->len = j - (i + 4); - rule_id->data = ngx_pnalloc(pool, rule_id->len); - if (rule_id->data == NULL) { - rule_id->len = 0; - rule_id->data = (u_char *)""; - return; - } - ngx_memcpy(rule_id->data, intervention->data + i + 4, rule_id->len); - return; - } -} diff --git a/src/ngx_http_modsecurity_common.h b/src/ngx_http_modsecurity_common.h index 1070c38e..73c81a54 100644 --- a/src/ngx_http_modsecurity_common.h +++ b/src/ngx_http_modsecurity_common.h @@ -82,9 +82,21 @@ typedef struct { } ngx_http_modsecurity_header_t; +typedef struct ngx_http_modsecurity_conf_s ngx_http_modsecurity_conf_t; + typedef struct { ngx_http_request_t *r; + struct { + ngx_pool_t *pool; + const ngx_http_modsecurity_conf_t *conf; + } lifetime; + struct { + ngx_str_t target; + ngx_int_t status; + ngx_int_t response_status; + } error_page; Transaction *modsec_transaction; + ModSecurityIntervention pending_request_intervention; ModSecurityIntervention *delayed_intervention; #if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) @@ -104,12 +116,17 @@ typedef struct { unsigned processed:1; unsigned logged:1; unsigned native_logging_failed:1; + unsigned request_policy_superseded:1; + unsigned request_intervention_active:1; + unsigned request_intervention_pending:1; + unsigned native_request_body_limit_rejection:1; unsigned intervention_triggered:1; unsigned request_body_processed:1; unsigned phase4_headers_checked:1; unsigned phase4_processed:1; unsigned phase4_strict_abort:1; unsigned phase4_started:1; + unsigned phase4_append_active:1; unsigned phase4_failed:1; unsigned phase4_error_emitting:1; unsigned response_body_seen:1; @@ -118,10 +135,12 @@ typedef struct { unsigned intervention_failed:1; unsigned intervention_body_limit:1; unsigned response_replaced:1; + unsigned replacement_body_terminal_sent:1; unsigned intervention_redirect_location_installed:1; size_t response_body_bytes_seen; size_t response_body_bytes_inspected; u_char *phase4_file_scratch; + u_char *request_file_scratch; ngx_str_t last_intervention_log; ngx_int_t last_intervention_status; } ngx_http_modsecurity_ctx_t; @@ -136,10 +155,12 @@ typedef struct { } ngx_http_modsecurity_main_conf_t; -typedef struct { +struct ngx_http_modsecurity_conf_s { void *pool; /* RulesSet or Rules */ void *rules_set; + const void *rules_identity; + unsigned rules_configured:1; ngx_flag_t enable; ngx_flag_t use_error_log; @@ -153,7 +174,7 @@ typedef struct { size_t phase4_body_limit; ngx_open_file_t *phase4_log_file; ngx_str_t phase4_log_path; -} ngx_http_modsecurity_conf_t; +}; typedef ngx_int_t (*ngx_http_modsecurity_resolv_header_pt)(ngx_http_request_t *r, ngx_str_t name, off_t offset); @@ -171,6 +192,9 @@ extern ngx_module_t ngx_http_modsecurity_module; int ngx_http_modsecurity_process_intervention (Transaction *transaction, ngx_http_request_t *r, ngx_int_t early_log); ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_create_ctx(ngx_http_request_t *r); ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r); +ngx_int_t ngx_http_modsecurity_prepare_location(ngx_http_request_t *r, const ngx_http_modsecurity_conf_t *mcf); +ngx_int_t ngx_http_modsecurity_log_request_contexts(ngx_http_request_t *r); +ngx_int_t ngx_http_modsecurity_capture_request_body_intervention(ngx_http_request_t *r); char *ngx_str_to_char(ngx_str_t a, ngx_pool_t *p); #if (NGX_PCRE2) #define ngx_http_modsecurity_pcre_malloc_init(x) NULL @@ -194,6 +218,7 @@ int ngx_http_modsecurity_store_ctx_header(ngx_http_request_t *r, ngx_str_t *name /* ngx_http_modsecurity_log.c */ void ngx_http_modsecurity_log(void *log, const void* data); ngx_int_t ngx_http_modsecurity_log_handler(ngx_http_request_t *r); +ngx_int_t ngx_http_modsecurity_log_context(ngx_http_modsecurity_ctx_t *ctx); /* ngx_http_modsecurity_access.c */ ngx_int_t ngx_http_modsecurity_access_handler(ngx_http_request_t *r); diff --git a/src/ngx_http_modsecurity_header_filter.c b/src/ngx_http_modsecurity_header_filter.c index 6189ead1..4b5272e6 100644 --- a/src/ngx_http_modsecurity_header_filter.c +++ b/src/ngx_http_modsecurity_header_filter.c @@ -14,6 +14,7 @@ */ #include +#include #ifndef MODSECURITY_DDEBUG #define MODSECURITY_DDEBUG 0 @@ -350,6 +351,19 @@ ngx_http_modsecurity_resolv_header_transfer_encoding(ngx_http_request_t *r, ngx_ { ngx_http_modsecurity_ctx_t *ctx = NULL; + (void) offset; +#if (NGX_HTTP_V2) + /* Transfer-Encoding is an HTTP/1.x hop-by-hop header. */ + if (r->stream) { + return 1; + } +#endif +#if defined(nginx_version) && nginx_version >= 1025000 + if (r->http_version == NGX_HTTP_VERSION_30) { + return 1; + } +#endif + if (r->chunked) { ngx_str_t value = ngx_string("chunked"); @@ -514,6 +528,16 @@ ngx_http_modsecurity_header_filter(ngx_http_request_t *r) status = r->headers_out.status; } + /* The native API accepts int; validate the wider NGINX status before + * narrowing it, independently of protocol/header representation. */ + if (status > INT_MAX) { + ctx->intervention_triggered = 1; + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "ModSecurity: response status exceeds the native integer range"); + return ngx_http_filter_finalize_request(r, &ngx_http_modsecurity_module, + NGX_HTTP_INTERNAL_SERVER_ERROR); + } + /* * NGINX always sends HTTP response with HTTP/1.1, except cases when * HTTP V2 module is enabled, and request has been posted with HTTP/2.0. @@ -526,7 +550,7 @@ ngx_http_modsecurity_header_filter(ngx_http_request_t *r) #endif old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); - ret = msc_process_response_headers(ctx->modsec_transaction, status, + ret = msc_process_response_headers(ctx->modsec_transaction, (int) status, http_response_ver); ngx_http_modsecurity_pcre_malloc_done(old_pool); if (ret != 1) { diff --git a/src/ngx_http_modsecurity_log.c b/src/ngx_http_modsecurity_log.c index d699d069..3e6f4499 100644 --- a/src/ngx_http_modsecurity_log.c +++ b/src/ngx_http_modsecurity_log.c @@ -37,24 +37,22 @@ ngx_http_modsecurity_log(void *log, const void* data) ngx_int_t -ngx_http_modsecurity_log_handler(ngx_http_request_t *r) +ngx_http_modsecurity_log_context(ngx_http_modsecurity_ctx_t *ctx) { ngx_pool_t *old_pool; - ngx_http_modsecurity_ctx_t *ctx; + ngx_http_request_t *r; int result; - if (r == NULL || r->connection == NULL) { + if (ctx == NULL || ctx->r == NULL || ctx->r->connection == NULL || + ctx->lifetime.pool == NULL) { return NGX_ERROR; } - ctx = ngx_http_modsecurity_get_module_ctx(r); - if (ctx == NULL) { - return NGX_OK; - } + r = ctx->r; if (ctx->logged) { return ctx->native_logging_failed ? NGX_ERROR : NGX_OK; } - /* Claim the attempt before a native callback can re-enter this handler. */ + /* Claim the attempt before native callbacks can re-enter this owner. */ ctx->logged = 1; ctx->native_logging_failed = 1; if (ctx->modsec_transaction == NULL) { @@ -62,7 +60,8 @@ ngx_http_modsecurity_log_handler(ngx_http_request_t *r) "ModSecurity: native logging transaction is unavailable"); return NGX_ERROR; } - old_pool = ngx_http_modsecurity_pcre_malloc_init(r->pool); + /* Pool cleanup can run after NGINX clears r->pool. Retain its real owner. */ + old_pool = ngx_http_modsecurity_pcre_malloc_init(ctx->lifetime.pool); result = msc_process_logging(ctx->modsec_transaction); ngx_http_modsecurity_pcre_malloc_done(old_pool); if (result != 1) { @@ -74,3 +73,12 @@ ngx_http_modsecurity_log_handler(ngx_http_request_t *r) ctx->native_logging_failed = 0; return NGX_OK; } + + +ngx_int_t +ngx_http_modsecurity_log_handler(ngx_http_request_t *r) +{ + /* Every transaction owner for this request gets one native audit attempt, + * including prior locations and disabled/different target policies. */ + return ngx_http_modsecurity_log_request_contexts(r); +} diff --git a/src/ngx_http_modsecurity_log_metadata.h b/src/ngx_http_modsecurity_log_metadata.h new file mode 100644 index 00000000..5fdec36c --- /dev/null +++ b/src/ngx_http_modsecurity_log_metadata.h @@ -0,0 +1,362 @@ +/* + * ModSecurity-nginx metadata helpers. + * Adapted from common/src/rule_id.c and common/src/json_escape.c in + * Easton97-Jens/ModSecurity-conector at + * d420a12c4155a4bc2b9ea63ee5cef633085705e7. + * Uses bounded NGINX strings and pools without a Common runtime dependency. + */ + +#ifndef NGX_HTTP_MODSECURITY_LOG_METADATA_H +#define NGX_HTTP_MODSECURITY_LOG_METADATA_H + +#include +#include +#include + +typedef struct { + ngx_str_t input; + size_t position; +} ngx_http_modsecurity_metadata_cursor_t; + +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_field_starts(ngx_str_t input, size_t position, + const char *name) +{ + size_t length = strlen(name); + + return position <= input.len && input.len - position >= length + 4 && + input.data[position] == '[' && + memcmp(input.data + position + 1, name, length) == 0 && + input.data[position + length + 1] == ' ' && + input.data[position + length + 2] == '"'; +} + +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_hex_digit(u_char value) +{ + return (value >= '0' && value <= '9') || + (value >= 'a' && value <= 'f') || + (value >= 'A' && value <= 'F'); +} + +/* Engine toHexIfNeeded() uses literal \xHH sequences in the escaped fields. + * Do not decode them while locating delimiters. Raw fields may contain quotes + * without a following ']', but the complete ordered record must still match. */ +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_parse_field( + ngx_http_modsecurity_metadata_cursor_t *cursor, const char *name, + ngx_int_t escaped, ngx_int_t decimal, ngx_str_t *value) +{ + size_t start, position, remaining; + u_char character; + + if (!ngx_http_modsecurity_metadata_field_starts(cursor->input, + cursor->position, name)) { + return 0; + } + start = cursor->position + strlen(name) + 3; + position = start; + while (position < cursor->input.len) { + remaining = cursor->input.len - position; + character = cursor->input.data[position]; + if (character == '"' && remaining >= 2 && + cursor->input.data[position + 1] == ']') { + if (decimal && position == start) { + return 0; + } + if (value != NULL) { + value->data = cursor->input.data + start; + value->len = position - start; + } + cursor->position = position + 2; + return 1; + } + if (character < 0x20 || character > 0x7e || + (decimal && (character < '0' || character > '9'))) { + return 0; + } + if (escaped && character == '\\') { + if (remaining < 4 || cursor->input.data[position + 1] != 'x' || + !ngx_http_modsecurity_metadata_hex_digit( + cursor->input.data[position + 2]) || + !ngx_http_modsecurity_metadata_hex_digit( + cursor->input.data[position + 3])) { + return 0; + } + position += 4; + } else { + if (escaped && character == '"') { + return 0; + } + position++; + } + } + return 0; +} + +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_next_field( + ngx_http_modsecurity_metadata_cursor_t *cursor, const char *name, + ngx_int_t escaped, ngx_int_t decimal) +{ + if (cursor->position >= cursor->input.len || + cursor->input.data[cursor->position] != ' ') { + return 0; + } + cursor->position++; + return ngx_http_modsecurity_metadata_parse_field(cursor, name, escaped, + decimal, NULL); +} + +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_parse_record( + ngx_http_modsecurity_metadata_cursor_t *cursor, ngx_str_t *rule_id) +{ + if (!ngx_http_modsecurity_metadata_parse_field(cursor, "file", 0, 0, NULL) || + !ngx_http_modsecurity_metadata_next_field(cursor, "line", 0, 1) || + cursor->position >= cursor->input.len || + cursor->input.data[cursor->position] != ' ') { + return 0; + } + cursor->position++; + if (!ngx_http_modsecurity_metadata_parse_field(cursor, "id", 0, 1, rule_id) || + !ngx_http_modsecurity_metadata_next_field(cursor, "rev", 1, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "msg", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "data", 1, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "severity", 0, 1) || + !ngx_http_modsecurity_metadata_next_field(cursor, "ver", 1, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "maturity", 0, 1) || + !ngx_http_modsecurity_metadata_next_field(cursor, "accuracy", 0, 1)) { + return 0; + } + while (cursor->position < cursor->input.len && + cursor->input.data[cursor->position] == ' ' && + ngx_http_modsecurity_metadata_field_starts(cursor->input, + cursor->position + 1, "tag")) { + if (!ngx_http_modsecurity_metadata_next_field(cursor, "tag", 1, 0)) { + return 0; + } + } + if (!ngx_http_modsecurity_metadata_next_field(cursor, "hostname", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "uri", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "unique_id", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "ref", 0, 0)) { + return 0; + } + if (cursor->position < cursor->input.len && + cursor->input.data[cursor->position] == ' ' && + (!ngx_http_modsecurity_metadata_next_field(cursor, "hostname", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "uri", 0, 0) || + !ngx_http_modsecurity_metadata_next_field(cursor, "unique_id", 0, 0))) { + return 0; + } + return cursor->position == cursor->input.len; +} + +/* Return 1 for one complete, unambiguous native record, 0 for no record, and + * -1 for malformed/ambiguous framing. The successful ID is a bounded view + * into input; input need not be NUL-terminated and no allocation is made. + * Bare ID snippets and the untrusted matched-data prefix are never metadata. */ +static ngx_inline ngx_int_t +ngx_http_modsecurity_parse_rule_metadata(ngx_str_t input, ngx_str_t *rule_id) +{ + static const char marker[] = "[file \""; + ngx_http_modsecurity_metadata_cursor_t cursor; + ngx_str_t candidate; + size_t position, first; + + if (rule_id == NULL) { + return -1; + } + rule_id->data = (u_char *) ""; + rule_id->len = 0; + if (input.data == NULL) { + return input.len == 0 ? 0 : -1; + } + if (input.len < sizeof(marker) - 1) { + return 0; + } + first = input.len; + for (position = 0; position <= input.len - (sizeof(marker) - 1); + position++) { + if (memcmp(input.data + position, marker, sizeof(marker) - 1) != 0) { + continue; + } + /* Reject extra candidates even if only one suffix would parse: a raw + * URI or reflected prefix can contain a forged complete record. */ + if (first != input.len || + (position != 0 && input.data[position - 1] != ' ')) { + return -1; + } + first = position; + } + if (first == input.len) { + return 0; + } + cursor.input = input; + cursor.position = first; + if (!ngx_http_modsecurity_metadata_parse_record(&cursor, &candidate)) { + return -1; + } + *rule_id = candidate; + return 1; +} + +static ngx_inline void +ngx_http_modsecurity_extract_rule_id(ngx_pool_t *pool, ngx_str_t *intervention, + ngx_str_t *rule_id) +{ + ngx_str_t candidate; + + rule_id->data = (u_char *) ""; + rule_id->len = 0; + if (intervention == NULL || + ngx_http_modsecurity_parse_rule_metadata(*intervention, &candidate) != 1) { + return; + } + rule_id->data = ngx_pnalloc(pool, candidate.len + 1); + if (rule_id->data == NULL) { + return; + } + memcpy(rule_id->data, candidate.data, candidate.len); + rule_id->data[candidate.len] = '\0'; + rule_id->len = candidate.len; +} + +static ngx_inline ngx_str_t +ngx_http_modsecurity_sanitize_intervention(ngx_pool_t *pool, ngx_str_t input) +{ + static ngx_str_t redacted = { + sizeof("redacted") - 1, (u_char *) "redacted" + }; + static ngx_str_t summary = { + sizeof("id:present msg:present op:-") - 1, + (u_char *) "id:present msg:present op:-" + }; + ngx_str_t rule_id; + + (void) pool; + /* A complete record supplies id/msg fields, but no trusted operator + * field. Never derive presence flags from matched data or message prose. + * Static complete strings avoid allocation failure and truncation. */ + return ngx_http_modsecurity_parse_rule_metadata(input, &rule_id) == 1 + ? summary : redacted; +} + +static ngx_inline ngx_int_t +ngx_http_modsecurity_metadata_utf8_continuation(u_char value) +{ + return value >= 0x80 && value <= 0xbf; +} + +static ngx_inline size_t +ngx_http_modsecurity_metadata_utf8_size(const u_char *input, size_t remaining) +{ + u_char first = input[0]; + + if (first >= 0xc2 && first <= 0xdf) { + return remaining >= 2 && + ngx_http_modsecurity_metadata_utf8_continuation(input[1]) ? 2 : 0; + } + if (first >= 0xe0 && first <= 0xef) { + if (remaining < 3 || + !ngx_http_modsecurity_metadata_utf8_continuation(input[1]) || + !ngx_http_modsecurity_metadata_utf8_continuation(input[2]) || + (first == 0xe0 && input[1] < 0xa0) || + (first == 0xed && input[1] > 0x9f)) { + return 0; + } + return 3; + } + if (first >= 0xf0 && first <= 0xf4) { + if (remaining < 4 || + !ngx_http_modsecurity_metadata_utf8_continuation(input[1]) || + !ngx_http_modsecurity_metadata_utf8_continuation(input[2]) || + !ngx_http_modsecurity_metadata_utf8_continuation(input[3]) || + (first == 0xf0 && input[1] < 0x90) || + (first == 0xf4 && input[1] > 0x8f)) { + return 0; + } + return 4; + } + return 0; +} + +/* Preserve valid UTF-8 as-is. JSON controls and invalid UTF-8 bytes are + * represented as \u00xx; each invalid byte retains its unsigned byte value. + * Success is NUL-terminated with an explicit length. Allocation/size failure + * returns data == NULL, len == 0 so the caller can diagnose the optional sink + * failure without confusing it with a valid empty field. */ +static ngx_inline void +ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst) +{ + static const u_char hex[] = "0123456789abcdef"; + size_t position, size, consumed, width; + u_char character, *output; + + dst->data = NULL; + dst->len = 0; + if (src == NULL || src->len == 0) { + dst->data = (u_char *) ""; + return; + } + if (src->data == NULL || src->len == SIZE_MAX) { + return; + } + size = 0; + for (position = 0; position < src->len; position += consumed) { + character = src->data[position]; + consumed = 1; + if (character < 0x20) { + width = 6; + } else if (character == '"' || character == '\\') { + width = 2; + } else if (character < 0x80) { + width = 1; + } else { + consumed = ngx_http_modsecurity_metadata_utf8_size( + src->data + position, src->len - position); + if (consumed == 0) { + consumed = 1; + width = 6; + } else { + width = consumed; + } + } + if (width > SIZE_MAX - 1 - size) { + return; + } + size += width; + } + dst->data = ngx_pnalloc(pool, size + 1); + if (dst->data == NULL) { + return; + } + output = dst->data; + for (position = 0; position < src->len; position += consumed) { + character = src->data[position]; + consumed = 1; + if (character == '"' || character == '\\') { + *output++ = '\\'; + *output++ = character; + } else if (character < 0x20 || + (character >= 0x80 && + (consumed = ngx_http_modsecurity_metadata_utf8_size( + src->data + position, src->len - position)) == 0)) { + *output++ = '\\'; + *output++ = 'u'; + *output++ = '0'; + *output++ = '0'; + *output++ = hex[character >> 4]; + *output++ = hex[character & 0x0f]; + consumed = 1; + } else { + memcpy(output, src->data + position, consumed); + output += consumed; + } + } + *output = '\0'; + dst->len = size; +} + +#endif /* NGX_HTTP_MODSECURITY_LOG_METADATA_H */ diff --git a/src/ngx_http_modsecurity_module.c b/src/ngx_http_modsecurity_module.c index 0c5e4ba0..f77228de 100644 --- a/src/ngx_http_modsecurity_module.c +++ b/src/ngx_http_modsecurity_module.c @@ -21,6 +21,7 @@ #include "ddebug.h" #include "ngx_http_modsecurity_common.h" +#include "ngx_http_modsecurity_log_metadata.h" #include "stdio.h" #include #include @@ -150,10 +151,12 @@ ngx_http_modsecurity_log_intervention(ngx_http_request_t *r, len = ngx_strlen(intervention->log); ctx->last_intervention_log.data = ngx_pnalloc(r->pool, len + 1); if (ctx->last_intervention_log.data == NULL) { - return NGX_ERROR; + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "ModSecurity: optional phase4 metadata allocation failed"); + } else { + ngx_memcpy(ctx->last_intervention_log.data, intervention->log, len + 1); + ctx->last_intervention_log.len = len; } - ngx_memcpy(ctx->last_intervention_log.data, intervention->log, len + 1); - ctx->last_intervention_log.len = len; } if (mcf->use_error_log) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "%s", @@ -214,21 +217,66 @@ ngx_http_modsecurity_process_intervention_redirect(ngx_http_request_t *r, } -/* Native response-limit Reject is not a rule intervention and must never be - * downgraded by Safe. A rule log includes its formatted rule metadata, so - * matching the entire engine-owned message also excludes a same-msg rule. */ +/* appendResponseBody does not evaluate response rules. Only a rule-free + * disruptive 403 collected immediately after a valid append is native Reject. + * Free-text diagnostics are not part of this native API boundary. */ static ngx_int_t ngx_http_modsecurity_is_response_body_limit_rejection( ngx_http_modsecurity_ctx_t *ctx, const ModSecurityIntervention *intervention) { - static const char expected[] = - "Response body limit is marked to reject the request"; + ngx_str_t log; + ngx_str_t rule_id; - return ctx->phase4_started && intervention->disruptive == 1 && + log.data = (u_char *) intervention->log; + log.len = intervention->log != NULL ? ngx_strlen(intervention->log) : 0; + return ctx->phase4_append_active && !ctx->phase4_processed && + intervention->disruptive == 1 && intervention->status == NGX_HTTP_FORBIDDEN && - intervention->url == NULL && intervention->log != NULL && - strncmp(intervention->log, expected, sizeof(expected)) == 0; + intervention->url == NULL && + ngx_http_modsecurity_parse_rule_metadata(log, &rule_id) == 0; +} + +/* A block response may traverse only its captured local error-page target. + * The error_page flag alone is insufficient: ordinary redirects still inspect. */ +static void +ngx_http_modsecurity_capture_error_page(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx, ngx_int_t status) +{ + ngx_http_core_loc_conf_t *clcf; + ngx_http_err_page_t *pages; + ngx_str_t target; + u_char *args; + ngx_uint_t i; + + if (!ctx->request_intervention_active || ctx->native_request_body_limit_rejection || + status < NGX_HTTP_BAD_REQUEST) { + return; + } + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + if (clcf == NULL || clcf->error_pages == NULL) { + return; + } + pages = clcf->error_pages->elts; + for (i = 0; i < clcf->error_pages->nelts; i++) { + if (pages[i].status != status) { + continue; + } + if (ngx_http_complex_value(r, &pages[i].value, &target) != NGX_OK || + target.data == NULL || target.len == 0 || + (target.data[0] != '/' && target.data[0] != '@')) { + return; + } + args = ngx_strlchr(target.data, target.data + target.len, '?'); + if (args != NULL) { + target.len = (size_t) (args - target.data); + } + ctx->error_page.target = target; + ctx->error_page.status = status; + ctx->error_page.response_status = pages[i].overwrite >= 0 ? + pages[i].overwrite : status; + return; + } } static ngx_int_t @@ -267,9 +315,8 @@ ngx_http_modsecurity_collect_native_intervention(ngx_http_request_t *r, "ModSecurity: response body limit rejected the response"); return NGX_ERROR; } - /* Response rules run at EOS. Any other disruptive result while bytes - * are being appended is an engine failure, including an uncorrelated - * Reject whose native message allocation failed. */ + /* Response rules run at EOS. Other disruptive append-time results are + * invalid engine responses and cannot become Safe log-only decisions. */ if (ctx->phase4_started && !ctx->phase4_processed) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ModSecurity: unexpected intervention during response append"); @@ -293,6 +340,62 @@ ngx_http_modsecurity_defer_late_phase4_intervention(ngx_http_request_t *r, return 1; } +/* Drain native request append results immediately. Retain an owned limit + * decision until normal P2 dispatch, even if ProcessPartial returns zero. */ +ngx_int_t +ngx_http_modsecurity_capture_request_body_intervention(ngx_http_request_t *r) +{ + ngx_http_modsecurity_ctx_t *ctx; + ModSecurityIntervention intervention; + ngx_int_t result = NGX_OK; + int native_result; + + ctx = ngx_http_modsecurity_get_module_ctx(r); + if (ctx == NULL || ctx->modsec_transaction == NULL) { + return NGX_ERROR; + } + ngx_memzero(&intervention, sizeof(intervention)); + intervention.status = 200; + native_result = msc_intervention(ctx->modsec_transaction, &intervention); + if (native_result != 0 && native_result != 1) { + result = NGX_ERROR; + } else if (native_result == 1 && intervention.disruptive) { + if ((intervention.status != 403 && intervention.status != 413) || + intervention.url != NULL || intervention.log == NULL || + strcmp(intervention.log, + "Request body limit is marked to reject the request") != 0) { + result = NGX_ERROR; + } else if (!ctx->request_intervention_pending) { + ctx->pending_request_intervention = intervention; + ctx->request_intervention_pending = 1; + ngx_memzero(&intervention, sizeof(intervention)); + } + } + if (intervention.log != NULL) free(intervention.log); + if (intervention.url != NULL) free(intervention.url); + return result; +} + +static ngx_int_t +ngx_http_modsecurity_take_pending_request_intervention( + ngx_http_modsecurity_ctx_t *ctx, ngx_int_t native_result, + ModSecurityIntervention *intervention) +{ + if (native_result == NGX_ERROR || !ctx->request_intervention_active || + !ctx->request_intervention_pending) { + return native_result; + } + if (intervention->log != NULL) free(intervention->log); + if (intervention->url != NULL) free(intervention->url); + *intervention = ctx->pending_request_intervention; + ngx_memzero(&ctx->pending_request_intervention, + sizeof(ctx->pending_request_intervention)); + ctx->request_intervention_pending = 0; + ctx->native_request_body_limit_rejection = 1; + ctx->intervention_failed = 1; + return NGX_OK; +} + static void ngx_http_modsecurity_reset_intervention(ngx_http_modsecurity_ctx_t *ctx) { @@ -302,6 +405,7 @@ ngx_http_modsecurity_reset_intervention(ngx_http_modsecurity_ctx_t *ctx) ctx->intervention_late = 0; ctx->intervention_failed = 0; ctx->intervention_body_limit = 0; + ctx->native_request_body_limit_rejection = 0; ctx->intervention_redirect_location_installed = 0; ctx->last_intervention_status = 0; ngx_str_null(&ctx->last_intervention_log); @@ -329,6 +433,8 @@ ngx_http_modsecurity_process_intervention (Transaction *transaction, ctx->intervention_failed = 1; native_result = ngx_http_modsecurity_collect_native_intervention(r, ctx, transaction, &intervention); + native_result = ngx_http_modsecurity_take_pending_request_intervention(ctx, + native_result, &intervention); result = native_result == NGX_DECLINED ? 0 : ngx_http_modsecurity_intervention_failure_status(r, ctx); if (native_result != NGX_OK) { @@ -348,6 +454,7 @@ ngx_http_modsecurity_process_intervention (Transaction *transaction, &intervention); } else if (intervention.status != 200) { msc_update_status_code(ctx->modsec_transaction, intervention.status); + ngx_http_modsecurity_capture_error_page(r, ctx, intervention.status); if (early_log && ngx_http_modsecurity_log_handler(r) != NGX_OK) { goto done; } @@ -382,6 +489,18 @@ ngx_http_modsecurity_cleanup(void *data) return; } if (ctx->modsec_transaction != NULL) { + /* Redirects may retire this owner before the normal log phase. Its + * retained pool is still valid even after NGINX clears r->pool. */ + (void) ngx_http_modsecurity_log_context(ctx); + if (ctx->pending_request_intervention.log != NULL) { + free(ctx->pending_request_intervention.log); + } + if (ctx->pending_request_intervention.url != NULL) { + free(ctx->pending_request_intervention.url); + } + ngx_memzero(&ctx->pending_request_intervention, + sizeof(ctx->pending_request_intervention)); + ctx->request_intervention_pending = 0; msc_transaction_cleanup(ctx->modsec_transaction); ctx->modsec_transaction = NULL; } @@ -416,9 +535,11 @@ ngx_http_modsecurity_create_ctx(ngx_http_request_t *r) return NULL; } ctx->r = r; + ctx->lifetime.pool = r->pool; mmcf = ngx_http_get_module_main_conf(r, ngx_http_modsecurity_module); mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + ctx->lifetime.conf = mcf; if (mmcf == NULL || mcf == NULL) { return NULL; @@ -471,15 +592,96 @@ ngx_http_modsecurity_create_ctx(ngx_http_request_t *r) return ctx; } -ngx_inline ngx_http_modsecurity_ctx_t * +static ngx_int_t +ngx_http_modsecurity_same_policy(const ngx_http_modsecurity_conf_t *left, + const ngx_http_modsecurity_conf_t *right) +{ + return left != NULL && right != NULL && + left->rules_identity == right->rules_identity && + left->enable == right->enable && + left->use_error_log == right->use_error_log && + left->transaction_id == right->transaction_id && + left->phase4_mode == right->phase4_mode && + left->phase4_log_file == right->phase4_log_file +#if defined(MODSECURITY_SANITY_CHECKS) && (MODSECURITY_SANITY_CHECKS) + && left->sanity_checks_enabled == right->sanity_checks_enabled +#endif + ; +} + +static ngx_int_t +ngx_http_modsecurity_is_block_error_page(const ngx_http_request_t *r, + const ngx_http_modsecurity_ctx_t *ctx) +{ + ngx_http_core_loc_conf_t *clcf; + ngx_str_t target; + + if (!r->internal || ctx->r != r || !ctx->intervention_triggered || + ctx->intervention_failed || ctx->phase4_failed || + ctx->error_page.target.len == 0 || + ctx->error_page.response_status != (ngx_int_t) r->err_status || + ctx->error_page.status != ctx->last_intervention_status) { + return 0; + } + target = r->uri; + if (ctx->error_page.target.data[0] == '@') { + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + if (clcf == NULL || !clcf->named) { + return 0; + } + target = clcf->name; + } + return target.len == ctx->error_page.target.len && + ngx_strncmp(target.data, ctx->error_page.target.data, target.len) == 0; +} + +/* Retire an allowed source policy when an internal target changes policy or + * disables inspection. Its audit owner remains on the pool cleanup chain. */ +ngx_int_t +ngx_http_modsecurity_prepare_location(ngx_http_request_t *r, + const ngx_http_modsecurity_conf_t *mcf) +{ + ngx_pool_cleanup_t *cln; + ngx_http_modsecurity_ctx_t *ctx; + + if (r == NULL || r->pool == NULL) { + return NGX_ERROR; + } + for (cln = r->pool->cleanup; cln != NULL; cln = cln->next) { + if (cln->handler != ngx_http_modsecurity_cleanup || cln->data == NULL) { + continue; + } + ctx = cln->data; + if (ctx->r == r && ctx->modsec_transaction != NULL && + !ctx->request_policy_superseded && !ctx->intervention_triggered && + !ctx->logged && !ctx->intervention_failed && !ctx->phase4_failed && + !ngx_http_modsecurity_same_policy(ctx->lifetime.conf, mcf)) { + ctx->request_policy_superseded = 1; + } + } + return NGX_OK; +} + +ngx_http_modsecurity_ctx_t * ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r) { ngx_http_modsecurity_ctx_t *ctx; ngx_http_modsecurity_ctx_t *candidate; ngx_pool_cleanup_t *cln; + ngx_http_modsecurity_conf_t *mcf; + if (r == NULL || r->pool == NULL) { + return NULL; + } + mcf = ngx_http_get_module_loc_conf(r, ngx_http_modsecurity_module); + if (mcf == NULL || mcf->enable != 1) { + return NULL; + } ctx = ngx_http_get_module_ctx(r, ngx_http_modsecurity_module); - if (ctx != NULL && ctx->r == r) { + if (ctx != NULL && ctx->r == r && ctx->modsec_transaction != NULL && + !ctx->request_policy_superseded && + (ngx_http_modsecurity_same_policy(ctx->lifetime.conf, mcf) || + ngx_http_modsecurity_is_block_error_page(r, ctx))) { return ctx; } /* Internal redirects reset module contexts but retain the same request. @@ -489,13 +691,42 @@ ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r) continue; } candidate = cln->data; - if (candidate != NULL && candidate->r == r) { + if (candidate != NULL && candidate->r == r && + candidate->modsec_transaction != NULL && + ((!candidate->intervention_triggered && !candidate->logged && + !candidate->request_policy_superseded && + !candidate->intervention_failed && !candidate->phase4_failed && + ngx_http_modsecurity_same_policy(candidate->lifetime.conf, mcf)) || + ngx_http_modsecurity_is_block_error_page(r, candidate))) { + ngx_http_set_ctx(r, candidate, ngx_http_modsecurity_module); return candidate; } } return NULL; } +ngx_int_t +ngx_http_modsecurity_log_request_contexts(ngx_http_request_t *r) +{ + ngx_pool_cleanup_t *cln; + ngx_http_modsecurity_ctx_t *ctx; + ngx_int_t result = NGX_OK; + + if (r == NULL || r->pool == NULL) { + return NGX_ERROR; + } + for (cln = r->pool->cleanup; cln != NULL; cln = cln->next) { + if (cln->handler != ngx_http_modsecurity_cleanup || cln->data == NULL) { + continue; + } + ctx = cln->data; + if (ctx->r == r && ngx_http_modsecurity_log_context(ctx) != NGX_OK) { + result = NGX_ERROR; + } + } + return result; +} + char * ngx_conf_set_rules(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { @@ -525,6 +756,7 @@ ngx_conf_set_rules(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) mmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_modsecurity_module); mmcf->rules_inline += res; + mcf->rules_configured = 1; return NGX_CONF_OK; } @@ -559,6 +791,7 @@ ngx_conf_set_rules_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) mmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_modsecurity_module); mmcf->rules_file += res; + mcf->rules_configured = 1; return NGX_CONF_OK; } @@ -598,6 +831,7 @@ ngx_conf_set_rules_remote(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) mmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_modsecurity_module); mmcf->rules_remote += res; + mcf->rules_configured = 1; return NGX_CONF_OK; } @@ -944,6 +1178,7 @@ ngx_http_modsecurity_create_conf(ngx_conf_t *cf) conf->enable = NGX_CONF_UNSET; conf->rules_set = msc_create_rules_set(); + conf->rules_identity = conf->rules_set; conf->pool = cf->pool; conf->transaction_id = NGX_CONF_UNSET_PTR; conf->use_error_log = NGX_CONF_UNSET; @@ -1012,6 +1247,10 @@ ngx_http_modsecurity_merge_conf(ngx_conf_t *cf, void *parent, void *child) return strdup(error); } + if (!c->rules_configured) { + c->rules_identity = p->rules_identity; + } + #if defined(MODSECURITY_DDEBUG) && (MODSECURITY_DDEBUG) dd("NEW CHILD RULES"); msc_rules_dump(c->rules_set); diff --git a/tests/README.md b/tests/README.md index 50052c65..84e6af44 100644 --- a/tests/README.md +++ b/tests/README.md @@ -7,7 +7,7 @@ For more information about those tests, read the subsection "Testing your patch" on the project's README file. For more about nginx tests, check their repository: -http://hg.nginx.org/nginx-tests/ +https://github.com/nginx/nginx-tests ## Standalone Phase4 helper regressions @@ -36,6 +36,10 @@ failed filter re-entry (including NULL-input flushes), a synchronous core error response, and early rejection after a single file chunk. Request tests exercise internal target-location isolation, error-page P1/P2 continuation, once-only audit logging, failed context creation and custom-ID termination. +Review follow-ups cover same-policy redirect reuse, changed-policy audit owners, +exact block-page routing, file-backed request ProcessPartial, saturated counters, +replacement control chains, language-independent append-limit classification, +complete metadata parsing and valid JSON for control/UTF-8/invalid-byte inputs. ## Native Phase4 integration tests diff --git a/tests/modsecurity-config-audit-redirect.t b/tests/modsecurity-config-audit-redirect.t new file mode 100644 index 00000000..f3255d7f --- /dev/null +++ b/tests/modsecurity-config-audit-redirect.t @@ -0,0 +1,80 @@ +#!/usr/bin/perl + +# Preserve native audit owners across same-policy, changed-policy and disabled +# internal targets. These checks require a real NGINX/libModSecurity build. + +use warnings; +use strict; +use Test::More; +BEGIN { use FindBin; chdir($FindBin::Bin); } +use lib 'lib'; +use Test::Nginx; + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http/)->plan(8); + +$t->write_file_expand('nginx.conf', <<'EOF'); +%%TEST_GLOBALS%% +daemon off; +events {} +http { + %%TEST_GLOBALS_HTTP%% + server { + listen 127.0.0.1:8080; + server_name localhost; + modsecurity on; + modsecurity_rules ' + SecRuleEngine On + SecAuditEngine On + SecAuditLogType Serial + SecAuditLogParts ABZ + SecAuditLog %%TESTDIR%%/audit-source.log + '; + location /same/ { + try_files $uri /same/index.html; + } + location = /changed-start { + try_files /nonexistent-audit-source /changed-final; + } + location = /changed-final { + modsecurity_rules ' + SecAuditLog %%TESTDIR%%/audit-target.log + '; + } + location = /disabled-start { + try_files /nonexistent-audit-source /disabled-final; + } + location = /disabled-final { + modsecurity off; + } + } +} +EOF + +mkdir($t->testdir() . '/same'); +$t->write_file('same/index.html', 'same-policy-response'); +$t->write_file('changed-final', 'changed-policy-response'); +$t->write_file('disabled-final', 'disabled-policy-response'); +$t->run(); + +my $same = http_get('/same/missing'); +like($same, qr/200 OK/, 'same-policy target succeeds'); +like($same, qr/same-policy-response/, 'same-policy response preserved'); +my $changed = http_get('/changed-start'); +like($changed, qr/200 OK/, 'changed-policy target succeeds'); +like($changed, qr/changed-policy-response/, 'changed-policy response preserved'); +my $disabled = http_get('/disabled-start'); +like($disabled, qr/200 OK/, 'disabled target succeeds'); +like($disabled, qr/disabled-policy-response/, 'disabled response preserved'); + +# Stop first so the native serial audit epilogues are fully written. +$t->stop(); +my $source = $t->read_file('audit-source.log'); +my $target = $t->read_file('audit-target.log'); +my $source_records = () = $source =~ /^GET /mg; +my $target_records = () = $target =~ /^GET /mg; +is($source_records, 3, + 'one source audit per request, including changed and disabled targets'); +is($target_records, 1, 'only the changed policy creates a second audit owner'); diff --git a/tests/modsecurity-config-custom-error-page.t b/tests/modsecurity-config-custom-error-page.t index f4bc076f..be73b528 100644 --- a/tests/modsecurity-config-custom-error-page.t +++ b/tests/modsecurity-config-custom-error-page.t @@ -130,6 +130,54 @@ http { '; } } + + server { + listen 127.0.0.1:8080; + server_name s4; + root %%TESTDIR%%/http; + modsecurity on; + modsecurity_rules ' + SecRuleEngine On + SecRule REQUEST_HEADERS:X-Scanner "@streq yes" "id:910510,phase:1,deny,status:403,log" + SecRule REQUEST_HEADERS:X-Body-Deny "@streq yes" "id:910511,phase:2,deny,status:403,log" + SecAuditEngine On + SecAuditLogParts ABHZ + SecAuditLog %%TESTDIR%%/auditlog-redirect-all.txt + SecAuditLogType Serial + '; + error_page 403 /block.html; + + location = /attack.html { } + location = /block.html { internal; } + location = /same-origin { try_files /missing-same-origin /same-target.html; } + location = /same-target.html { } + location = /changed-origin { try_files /missing-changed-origin /changed-target.html; } + location = /changed-target.html { + modsecurity_rules 'SecAction "id:910514,phase:1,pass,nolog"'; + } + } + + server { + listen 127.0.0.1:8080; + server_name s5; + root %%TESTDIR%%/http; + client_body_buffer_size 64k; + modsecurity on; + modsecurity_rules ' + SecRuleEngine On + SecRequestBodyAccess On + SecRequestBodyLimit 11 + SecRequestBodyLimitAction ProcessPartial + SecRule REQUEST_BODY "@beginsWith partial-hit" "id:910520,phase:2,deny,status:409,nolog" + SecRule REQUEST_BODY "@contains beyond-limit" "id:910521,phase:2,deny,status:418,nolog" + SecAuditEngine Off + '; + error_page 405 =200 /partial-accepted.txt; + location = /file-partial { client_body_in_file_only clean; } + location = /memory-partial { } + location = /partial-accepted.txt { internal; modsecurity off; } + } + } EOF @@ -141,9 +189,15 @@ mkdir($t->testdir() . '/http'); $t->write_file("/http/403.html", $custom_txt); my $protected_txt = "This is a protected error target."; $t->write_file("/http/protected-error.txt", $protected_txt); +my $block_txt = "Configured WAF block page."; +$t->write_file('/http/block.html', $block_txt); +for my $name (qw/attack.html same-target.html changed-target.html file-partial memory-partial/) { + $t->write_file('/http/' . $name, 'allowed static response'); +} +$t->write_file('/http/partial-accepted.txt', 'accepted partial request'); $t->run(); -$t->plan(14); +$t->plan(29); ############################################################################### @@ -199,8 +253,64 @@ like($target_allow, qr/\Q$protected_txt\E/, 'error-page target delivers legitima like($target_p1, qr/^HTTP\/1\.[01] 451/, 'error-page target preserves P1 deny'); like($target_p2, qr/^HTTP\/1\.[01] 403/, 'error-page target continues from P1 allow to P2 deny'); +# A configured block page must not run the original scanner/header rules again. +my $blocked_p1 = http_get_host('s4', '/attack.html?marker=p1-scan', 'X-Scanner: yes'); +my $blocked_p2 = http_get_host('s4', '/attack.html?marker=p2-scan', 'X-Body-Deny: yes'); +like($blocked_p1, qr/^HTTP\/1\.[01] 403/, 'P1 denial keeps its status through the configured block page'); +like($blocked_p1, qr/\Q$block_txt\E/, 'P1 denial serves the configured block page'); +like($blocked_p2, qr/^HTTP\/1\.[01] 403/, 'P2 denial keeps its status through the configured block page'); +like($blocked_p2, qr/\Q$block_txt\E/, 'P2 denial serves the configured block page'); + +my $same_policy = http_get_host('s4', '/same-origin?marker=same-policy'); +my $changed_policy = http_get_host('s4', '/changed-origin?marker=changed-policy'); +like($same_policy, qr/^HTTP\/1\.[01] 200/, 'equivalent inherited policy permits the internal redirect'); +like($changed_policy, qr/^HTTP\/1\.[01] 200/, 'changed target policy permits its legitimate control'); +my $redirect_audit = do { + local $/ = undef; + open my $fh, '<', "$d/auditlog-redirect-all.txt" or die "could not open: $!"; + <$fh>; +}; +# Part B has one exact HTTP request line per serial native audit transaction. +for my $observation ( + ['/attack.html?marker=p1-scan', 1, 'P1 block is audited once'], + ['/attack.html?marker=p2-scan', 1, 'original P2 block is audited once'], + ['/same-origin?marker=same-policy', 1, 'equivalent inherited policy does not duplicate audits'], + ['/changed-origin?marker=changed-policy', 2, 'both changed-policy transaction owners are audited'], +) { + my ($uri, $expected, $name) = @$observation; + my $count = () = $redirect_audit =~ /^GET \Q$uri\E HTTP\/1\.[01]\r?$/mg; + is($count, $expected, $name); +} + +# The retained prefix is inspected identically in memory and a forced temp file. +# The allowed control's forbidden tail must stay beyond the engine-owned limit. +my $partial_hit = 'partial-hit-beyond-limit'; +my $partial_allow = 'allowed-xxxbeyond-limit'; +my $file_hit = http_post_host('s5', '/file-partial', $partial_hit); +my $memory_hit = http_post_host('s5', '/memory-partial', $partial_hit); +my $file_allow = http_post_host('s5', '/file-partial', $partial_allow); +my $memory_allow = http_post_host('s5', '/memory-partial', $partial_allow); +like($file_hit, qr/^HTTP\/1\.[01] 409/, 'file-backed ProcessPartial runs P2 on the retained prefix'); +like($memory_hit, qr/^HTTP\/1\.[01] 409/, 'memory ProcessPartial runs P2 on the retained prefix'); +like($file_allow, qr/^HTTP\/1\.[01] 200/, 'file-backed ProcessPartial permits its retained-prefix control'); +like($memory_allow, qr/^HTTP\/1\.[01] 200/, 'memory ProcessPartial permits its retained-prefix control'); +like($file_allow, qr/accepted partial request/, 'file-backed partial control delivers the expected response'); + ############################################################################### +sub http_post_host { + my ($host, $url, $body) = @_; + my $length = length($body); + return http(< str: + name = "ngx_http_modsecurity_resolv_header_transfer_encoding" + for match in re.finditer(rf"\b{name}\s*\(", source): + cursor = source.index(")", match.end()) + 1 + while source[cursor].isspace(): + cursor += 1 + if source[cursor] != "{": + continue + start = source.rfind("\n", 0, match.start()) + 1 + start = source.rfind("\n", 0, start - 1) + 1 + depth = 0 + for index in range(cursor, len(source)): + if source[index] == "{": + depth += 1 + elif source[index] == "}": + depth -= 1 + if depth == 0: + return source[start:index + 1] + raise AssertionError("production Transfer-Encoding resolver was not found") + + +PREAMBLE = r""" +#include +#include +#include +#include +#include +#include + +typedef unsigned char u_char; +typedef intptr_t ngx_int_t; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct { + int chunked; + int http_version; +#if (NGX_HTTP_V2) + void *stream; +#endif +} ngx_http_request_t; +typedef struct { void *modsec_transaction; } ngx_http_modsecurity_ctx_t; +#define ngx_string(value) { sizeof(value) - 1, (u_char *) value } +#if nginx_version >= 1025000 +#define NGX_HTTP_VERSION_30 3000 +#endif + +static int transaction; +static ngx_http_modsecurity_ctx_t context = { &transaction }; +static unsigned context_calls, native_calls, sanity_calls; +static int native_result; + +static ngx_http_modsecurity_ctx_t * +ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r) +{ + (void) r; + context_calls++; + return &context; +} + +#if (MODSECURITY_SANITY_CHECKS) +static int ngx_http_modsecurity_store_ctx_header(ngx_http_request_t *r, + ngx_str_t *name, ngx_str_t *value) +{ + (void) r; + if (name->len != 17 || value->len != 7) abort(); + sanity_calls++; + return 1; +} +#endif + +static int msc_add_n_response_header(void *owner, const unsigned char *name, + size_t name_size, const unsigned char *value, size_t value_size) +{ + if (owner != &transaction || name_size != 17 || value_size != 7 || + memcmp(name, "Transfer-Encoding", 17) != 0 || + memcmp(value, "chunked", 7) != 0) abort(); + native_calls++; + return native_result; +} +""" + +MAIN = r""" +int main(int argc, char **argv) +{ + ngx_http_request_t request; + ngx_str_t name = ngx_string("Transfer-Encoding"); + ngx_int_t result; + + if (argc != 5) return 2; + memset(&request, 0, sizeof(request)); + request.http_version = atoi(argv[1]); + request.chunked = atoi(argv[2]); +#if (NGX_HTTP_V2) + request.stream = atoi(argv[3]) ? &transaction : NULL; +#endif + native_result = atoi(argv[4]); + result = ngx_http_modsecurity_resolv_header_transfer_encoding(&request, + name, 0); + printf("%ld %u %u %u\n", (long) result, native_calls, sanity_calls, + context_calls); + return 0; +} +""" + + +class HeaderProtocolRuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler or shutil.which(compiler[0]) is None: + raise unittest.SkipTest("A C compiler is needed for header protocol regressions") + cls.directory = tempfile.TemporaryDirectory(prefix="modsec-header-protocol-") + cls.addClassCleanup(cls.directory.cleanup) + source = PREAMBLE + resolver_definition(HEADER_FILTER.read_text(encoding="utf-8")) + MAIN + path = Path(cls.directory.name) / "header-protocol.c" + path.write_text(source, encoding="utf-8") + cls.binaries = {} + for v2, sanity, version in itertools.product((0, 1), (0, 1), (1024000, 1031000)): + key = v2, sanity, version + binary = Path(cls.directory.name) / f"header-{v2}-{sanity}-{version}" + completed = subprocess.run( + compiler + ["-std=c99", "-Wall", "-Wextra", "-Werror", + f"-DNGX_HTTP_V2={v2}", + f"-DMODSECURITY_SANITY_CHECKS={sanity}", + f"-Dnginx_version={version}", str(path), "-o", str(binary)], + capture_output=True, text=True, check=False, + ) + if completed.returncode != 0: + raise AssertionError(completed.stdout + completed.stderr) + cls.binaries[key] = binary + + def execute(self, key, protocol, chunked, stream=0, result=73): + completed = subprocess.run( + [str(self.binaries[key]), str(protocol), str(chunked), str(stream), str(result)], + capture_output=True, text=True, check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + return tuple(map(int, completed.stdout.split())) + + def test_http1_chunked_preserves_native_and_sanity_callbacks(self): + for key in self.binaries: + with self.subTest(build=key): + self.assertEqual(self.execute(key, 1001, 1), (73, 1, key[1], 1)) + + def test_http1_unchunked_never_synthesizes_transfer_encoding(self): + for key in self.binaries: + with self.subTest(build=key): + self.assertEqual(self.execute(key, 1001, 0), (1, 0, 0, 0)) + + def test_http2_stream_bypasses_even_a_stale_chunked_flag(self): + for key in self.binaries: + if not key[0]: + continue + for chunked in (0, 1): + with self.subTest(build=key, chunked=chunked): + self.assertEqual(self.execute(key, 2000, chunked, stream=1), (1, 0, 0, 0)) + + def test_http3_bypasses_without_requiring_an_http2_stream_field(self): + for key in self.binaries: + if key[2] < 1025000: + continue + for chunked in (0, 1): + with self.subTest(build=key, chunked=chunked): + self.assertEqual(self.execute(key, 3000, chunked), (1, 0, 0, 0)) + + def test_http1_propagates_native_header_result(self): + for key in self.binaries: + for native_result in (-1, 0, 1): + with self.subTest(build=key, result=native_result): + self.assertEqual(self.execute(key, 1001, 1, result=native_result), + (native_result, 1, key[1], 1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_intervention_runtime.py b/tests/test_intervention_runtime.py index 714e1825..ced15e32 100644 --- a/tests/test_intervention_runtime.py +++ b/tests/test_intervention_runtime.py @@ -56,18 +56,27 @@ int disruptive; double pause; } ModSecurityIntervention; +typedef struct ngx_http_modsecurity_conf_s ngx_http_modsecurity_conf_t; +typedef struct { int named; ngx_str_t name; } ngx_http_core_loc_conf_t; typedef struct { ngx_http_request_t *r; + struct { const ngx_http_modsecurity_conf_t *conf; } lifetime; + struct { ngx_str_t target; ngx_int_t status, response_status; } error_page; Transaction *modsec_transaction; + ModSecurityIntervention pending_request_intervention; ngx_str_t last_intervention_log; int last_intervention_status, intervention_late, intervention_failed; int intervention_redirect_location_installed, logged; int phase4_failed, phase4_error_emitting, phase4_started, phase4_processed, intervention_body_limit; + int phase4_append_active, request_policy_superseded, intervention_triggered; + int request_intervention_active, request_intervention_pending, native_request_body_limit_rejection; } ngx_http_modsecurity_ctx_t; -typedef struct { +struct ngx_http_modsecurity_conf_s { void *phase4_log_file; - int use_error_log, phase4_mode; -} ngx_http_modsecurity_conf_t; + const void *rules_identity; + void *transaction_id; + int enable, use_error_log, phase4_mode, sanity_checks_enabled; +}; typedef struct { void *log; } ngx_connection_t; struct ngx_http_request_s { int header_sent; @@ -75,6 +84,9 @@ ngx_connection_t *connection; ngx_http_modsecurity_ctx_t *ctx; ngx_http_modsecurity_conf_t *conf; + ngx_http_core_loc_conf_t *core; + int internal, err_status; + ngx_str_t uri; struct { ngx_list_t headers; ngx_table_elt_t *location; } headers_out; }; #define NGX_OK 0 @@ -85,8 +97,14 @@ #define NGX_HTTP_FORBIDDEN 403 #define NGX_HTTP_INTERNAL_SERVER_ERROR 500 #define NGX_LOG_ERR 3 +#define NGX_LOG_WARN 4 +#define ngx_inline inline +#define ngx_http_modsecurity_module 1 +#define ngx_http_core_module 2 #define ngx_http_get_module_ctx(r, module) ((r)->ctx) -#define ngx_http_get_module_loc_conf(r, module) ((r)->conf) +#define ngx_http_get_module_loc_conf(r, module) ((module) == 2 ? (void *)(r)->core : (void *)(r)->conf) +#define ngx_http_set_ctx(r, c, module) ((r)->ctx = (c)) +#define ngx_strncmp(a, b, n) strncmp((const char *)(a), (const char *)(b), (n)) #define ngx_memzero(data, size) memset((data), 0, (size)) #define ngx_memcpy(dst, src, size) memcpy((dst), (src), (size)) #define ngx_strlen(data) strlen((const char *)(data)) @@ -98,7 +116,7 @@ static int native_available = 1, native_status = 403, native_calls; static int native_return = 1, native_disruptive = 1; static int native_has_log = 1, native_has_url, native_initialized; -static char native_log[256] = "native intervention message"; +static char native_log[4096] = "native intervention message"; static char native_url[256] = "https://example.test/new-location"; static int freed_log, freed_url, bad_free, allocation_calls, fail_allocation_at; static int list_calls, fail_list, clear_calls, error_log_calls; @@ -107,6 +125,8 @@ static char captured_error[256]; static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r); static void ngx_http_modsecurity_cleanup(void *data) { (void)data; } +static void ngx_http_modsecurity_capture_error_page(ngx_http_request_t *r, ngx_http_modsecurity_ctx_t *ctx, ngx_int_t status) +{ (void)r; (void)ctx; (void)status; } static int msc_intervention(Transaction *transaction, ModSecurityIntervention *out) { @@ -209,7 +229,8 @@ request.pool = &pool; request.connection = &connection; request.ctx = &ctx; request.conf = &conf; ctx.r = &request; ctx.modsec_transaction = &transaction; - conf.phase4_log_file = &transaction; conf.use_error_log = 1; + conf.phase4_log_file = &transaction; conf.use_error_log = 1; conf.enable = 1; + ctx.lifetime.conf = &conf; previous_location.hash = 1; request.headers_out.location = &previous_location; ctx.intervention_late = 1; ctx.intervention_failed = 1; @@ -275,7 +296,7 @@ conf.phase4_mode = mode; for (committed = 0; committed <= 1; ++committed) { strcpy(native_log, limit_message); freed_log = 0; - request.header_sent = committed; ctx.phase4_started = 1; + request.header_sent = committed; ctx.phase4_started = 1; ctx.phase4_append_active = 1; result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); CHECK(result == (committed ? NGX_ERROR : 403)); CHECK(ctx.intervention_failed && ctx.intervention_body_limit && !ctx.intervention_late); @@ -284,6 +305,51 @@ CHECK(status_calls == 0 && list_calls == 0 && early_log_calls == 0); } } + } else if (strcmp(argv[1], "body-limit-language-independent") == 0) { + const char *messages[] = {"Andere Diagnose", "", NULL}; + int mode, committed; + size_t i; + ctx.phase4_started = 1; ctx.phase4_append_active = 1; + for (mode = 0; mode <= 2; ++mode) { + conf.phase4_mode = mode; + for (committed = 0; committed <= 1; ++committed) { + request.header_sent = committed; + for (i = 0; i < sizeof(messages) / sizeof(messages[0]); ++i) { + native_has_log = messages[i] != NULL; freed_log = 0; + if (messages[i]) strcpy(native_log, messages[i]); + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == (committed ? NGX_ERROR : 403)); + CHECK(ctx.intervention_body_limit && ctx.intervention_failed); + CHECK(freed_log == native_has_log && status_calls == 0 && list_calls == 0); + } + } + } + } else if (strcmp(argv[1], "body-limit-boundary-controls") == 0) { + ctx.phase4_started = 1; ctx.phase4_processed = 1; ctx.phase4_append_active = 1; + strcpy(native_log, limit_message); + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 403 && !ctx.intervention_body_limit && !ctx.intervention_failed); + CHECK(status_calls == 1 && freed_log == 1); + status_calls = 0; freed_log = 0; ctx.phase4_processed = 0; + strcpy(native_log, "[file \"/rule\"] [line \"1\"] [id \"123\"] [rev \"\"] [msg \"limit\"] [data \"\"] [severity \"2\"] [ver \"\"] [maturity \"0\"] [accuracy \"0\"] [hostname \"host\"] [uri \"/\"] [unique_id \"x\"] [ref \"\"]"); + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 500 && !ctx.intervention_body_limit && ctx.intervention_failed); + CHECK(status_calls == 0 && freed_log == 1); + } else if (strcmp(argv[1], "request-append-retains-limit") == 0) { + strcpy(native_log, "Request body limit is marked to reject the request"); + native_status = 413; + CHECK(ngx_http_modsecurity_capture_request_body_intervention(&request) == NGX_OK); + CHECK(ctx.request_intervention_pending && freed_log == 0); + native_available = 0; ctx.request_intervention_active = 1; + result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); + CHECK(result == 413 && ctx.native_request_body_limit_rejection); + CHECK(!ctx.request_intervention_pending && freed_log == 1 && status_calls == 1); + CHECK(!ctx.intervention_failed && ctx.last_intervention_status == 413); + } else if (strcmp(argv[1], "request-append-invalid-result") == 0) { + native_return = 2; + CHECK(ngx_http_modsecurity_capture_request_body_intervention(&request) == NGX_ERROR); + CHECK(!ctx.request_intervention_pending && freed_log == 1); + CHECK(status_calls == 0 && early_log_calls == 0); } else if (strcmp(argv[1], "body-limit-other-phase") == 0 || strcmp(argv[1], "body-limit-rule-lookalike") == 0) { if (strcmp(argv[1], "body-limit-other-phase") == 0) strcpy(native_log, limit_message); @@ -297,7 +363,7 @@ CHECK(freed_log == 1 && freed_url == 0 && early_log_calls == 0); } else if (strcmp(argv[1], "uncorrelated-append-intervention") == 0) { int mode, committed; - ctx.phase4_started = 1; native_has_log = 0; + ctx.phase4_started = 1; ctx.phase4_append_active = 1; native_status = 451; native_has_log = 0; for (mode = 0; mode <= 2; ++mode) { conf.phase4_mode = mode; for (committed = 0; committed <= 1; ++committed) { @@ -424,11 +490,12 @@ request.header_sent = strcmp(argv[1], "log-allocation-committed") == 0; native_has_url = 1; native_status = 302; fail_allocation_at = 1; result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); - CHECK(result == (request.header_sent ? NGX_ERROR : 500)); - CHECK(ctx.intervention_failed && !ctx.intervention_late); - CHECK(!ctx.intervention_redirect_location_installed && error_log_calls == 0); + CHECK(result == (request.header_sent ? NGX_ERROR : 302)); + CHECK(!ctx.intervention_failed && ctx.intervention_late == request.header_sent); + CHECK(ctx.intervention_redirect_location_installed == !request.header_sent); + CHECK(error_log_calls == 2); CHECK(ctx.last_intervention_log.len == 0U && ctx.last_intervention_log.data == NULL); - CHECK(freed_log == 1 && freed_url == 1 && list_calls == 0); + CHECK(freed_log == 1 && freed_url == 1 && list_calls == !request.header_sent); } else if (strcmp(argv[1], "disabled-log-retention") == 0) { conf.phase4_log_file = NULL; conf.use_error_log = 0; ctx.last_intervention_log.data = (u_char *)"old message"; @@ -451,10 +518,11 @@ request.header_sent = strcmp(argv[1], "missing-config-committed") == 0; result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); CHECK(result == (request.header_sent ? NGX_ERROR : 500)); - CHECK(ctx.intervention_failed && !ctx.intervention_late); + /* No policy owner is selected without configuration: its state stays untouched. */ + CHECK(ctx.intervention_failed && ctx.intervention_late); CHECK(native_calls == 0 && freed_log == 0 && freed_url == 0); } else if (strcmp(argv[1], "redirect-context-recovery") == 0) { - pool.cleanup = &cleanup; request.ctx = NULL; + pool.cleanup = &cleanup; request.ctx = NULL; ctx.intervention_failed = 0; result = ngx_http_modsecurity_process_intervention(&transaction, &request, 0); CHECK(result == 403 && native_calls == 1 && !ctx.intervention_failed); CHECK(freed_log == 1 && ctx.last_intervention_status == 403); @@ -474,7 +542,10 @@ def unit_program(module: str) -> str: - helpers = "\nstatic ngx_http_modsecurity_ctx_t *\n" + function_definition( + helpers = "" + for name in ("ngx_http_modsecurity_same_policy", "ngx_http_modsecurity_is_block_error_page"): + helpers += "\nstatic ngx_int_t\n" + function_definition(module, name) + helpers += "\nstatic ngx_http_modsecurity_ctx_t *\n" + function_definition( module, "ngx_http_modsecurity_get_module_ctx" ) helpers += "\nstatic ngx_int_t\n" + function_definition( @@ -484,6 +555,7 @@ def unit_program(module: str) -> str: module, "ngx_http_modsecurity_process_intervention_redirect" ) for helper in ( + "ngx_http_modsecurity_take_pending_request_intervention", "ngx_http_modsecurity_is_response_body_limit_rejection", "ngx_http_modsecurity_intervention_failure_status", "ngx_http_modsecurity_collect_native_intervention", @@ -491,13 +563,15 @@ def unit_program(module: str) -> str: ): if re.search(rf"\b{helper}\s*\(", module): helpers += "\nstatic ngx_int_t\n" + function_definition(module, helper) + helpers += "\nngx_int_t\n" + function_definition(module, "ngx_http_modsecurity_capture_request_body_intervention") reset = "ngx_http_modsecurity_reset_intervention" if re.search(rf"\b{reset}\s*\(", module): helpers += "\nstatic void\n" + function_definition(module, reset) helpers += "\nint\n" + function_definition( module, "ngx_http_modsecurity_process_intervention" ) - return PREAMBLE + helpers + CASES + metadata = (ROOT / "src" / "ngx_http_modsecurity_log_metadata.h").read_text(encoding="utf-8") + return PREAMBLE + metadata + helpers + CASES class InterventionRuntimeTests(unittest.TestCase): @@ -545,7 +619,8 @@ def test(self: InterventionRuntimeTests) -> None: "late-phase4-policy-deferral", "invalid-native-results", "nondisruptive-result", "no-result-owned-outputs", "body-limit-rejection", "body-limit-other-phase", "body-limit-rule-lookalike", - "early-log-failure", + "early-log-failure", "body-limit-language-independent", "body-limit-boundary-controls", + "request-append-retains-limit", "request-append-invalid-result", "uncorrelated-append-intervention", "redirect-owned", "redirect-committed", "redirect-crlf", "redirect-list-failure", "redirect-allocation-failure", "redirect-sanity-failure", diff --git a/tests/test_lifecycle_runtime.py b/tests/test_lifecycle_runtime.py index a87d546d..51744a1a 100644 --- a/tests/test_lifecycle_runtime.py +++ b/tests/test_lifecycle_runtime.py @@ -23,38 +23,38 @@ #include #include #include -#include +#include typedef unsigned char u_char; typedef long ngx_int_t; typedef unsigned long ngx_uint_t; +typedef long off_t; +typedef long ssize_t; typedef struct { size_t len; u_char *data; } ngx_str_t; typedef struct { int unused; } ngx_pool_t; typedef struct { int unused; } Transaction; typedef struct ngx_http_request_s ngx_http_request_t; -typedef struct ngx_list_part_s { - void *elts; - ngx_uint_t nelts; - struct ngx_list_part_s *next; -} ngx_list_part_t; +typedef struct ngx_list_part_s { void *elts; ngx_uint_t nelts; struct ngx_list_part_s *next; } ngx_list_part_t; typedef struct { ngx_list_part_t part; } ngx_list_t; typedef struct { ngx_str_t key, value; } ngx_table_elt_t; typedef struct { u_char *pos, *last; int last_buf; } ngx_buf_t; typedef struct ngx_chain_s { ngx_buf_t *buf; struct ngx_chain_s *next; } ngx_chain_t; -typedef struct { struct { ngx_str_t name; } file; } ngx_temp_file_t; +typedef struct { int fd; ngx_str_t name; } ngx_file_t; +typedef struct { off_t size; int regular; } ngx_file_info_t; +typedef struct { ngx_file_t file; off_t offset; } ngx_temp_file_t; typedef struct { ngx_temp_file_t *temp_file; ngx_chain_t *bufs; } ngx_http_request_body_t; typedef struct { void *sockaddr, *local_sockaddr, *log; ngx_str_t addr_text; int error; } ngx_connection_t; typedef struct { ngx_http_request_t *r; + struct { ngx_pool_t *pool; const void *conf; } lifetime; Transaction *modsec_transaction; int waiting_more_body, body_requested, request_body_processed; - int intervention_triggered, intervention_failed; - int logged, native_logging_failed; + int intervention_triggered, intervention_failed, request_intervention_active; + int request_intervention_pending, logged, native_logging_failed; + u_char *request_file_scratch; } ngx_http_modsecurity_ctx_t; typedef struct { int enable; } ngx_http_modsecurity_conf_t; struct ngx_http_request_s { - ngx_pool_t *pool; - ngx_connection_t *connection; - ngx_http_request_t *main; + ngx_pool_t *pool; ngx_connection_t *connection; ngx_http_request_t *main; int count, method, http_version, header_sent, error_page; ngx_str_t http_protocol, unparsed_uri, method_name; struct { ngx_str_t server; ngx_list_t headers; } headers_in; @@ -77,35 +77,48 @@ #define NGX_HTTP_VERSION_11 1001 #define NGX_SOCKADDR_STRLEN 64 #define NGX_LOG_ERR 3 +#define NGX_INVALID_FILE (-1) +#define NGX_FILE_ERROR (-1) +#define ngx_is_file(info) ((info)->regular) +#define ngx_file_size(info) ((info)->size) +#define ngx_buf_special(buf) ((buf)->pos == NULL && (buf)->last == NULL) #define dd(...) ((void)0) #define ngx_http_get_module_ctx(r, module) ((r)->ctx) #define ngx_http_get_module_loc_conf(r, module) ((r)->conf) #define CHECK(condition) do { if (!(condition)) { \ fprintf(stderr, "line %d: %s\n", __LINE__, #condition); return 1; \ } } while (0) - static Transaction transaction; static ngx_http_modsecurity_ctx_t created; -static int created_calls, recovered_calls, error_calls; +static int created_calls, recovered_calls, error_calls, prepare_calls, prepare_failure, recover_allowed; static int native_results[4] = {1, 1, 1, 1}; static int native_calls[4], intervention_results[5], intervention_calls[5]; -static int active_phase, append_result = 1, append_calls, file_result = 1; +static int active_phase, append_result = 1, append_calls, capture_calls, pending_limit; static int log_result = 1, log_calls, log_reenter, nested_log_result; +static int stat_calls, read_calls, short_read, fail_read, bad_stat, regular_file = 1, changed_file, fail_scratch; +static size_t max_read; +static off_t actual_file_size = 3; +static u_char file_bytes[65537], scratch[32768]; static ngx_http_request_t *active_request; static ngx_int_t ngx_http_modsecurity_log_handler(ngx_http_request_t *r); +static ngx_int_t ngx_http_modsecurity_log_context(ngx_http_modsecurity_ctx_t *ctx); static void ngx_http_core_run_phases(ngx_http_request_t *r) { (void)r; } +static ngx_int_t ngx_http_modsecurity_prepare_location(ngx_http_request_t *r, const ngx_http_modsecurity_conf_t *conf) +{ + (void)r; (void)conf; ++prepare_calls; + return prepare_failure ? NGX_ERROR : NGX_OK; +} static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r) { ++recovered_calls; if (r->ctx != NULL && r->ctx->r == r) return r->ctx; - return r->recovered != NULL && r->recovered->r == r ? r->recovered : NULL; + return recover_allowed && r->recovered != NULL && r->recovered->r == r ? r->recovered : NULL; } static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_create_ctx(ngx_http_request_t *r) { - ++created_calls; - memset(&created, 0, sizeof(created)); - created.r = r; - created.modsec_transaction = &transaction; + ++created_calls; memset(&created, 0, sizeof(created)); + created.r = r; created.modsec_transaction = &transaction; + created.lifetime.pool = r->pool; created.lifetime.conf = r->conf; r->ctx = &created; return &created; } @@ -117,71 +130,87 @@ static int ngx_inet_get_port(void *address) { (void)address; return 80; } static ngx_int_t ngx_connection_local_sockaddr(ngx_connection_t *connection, ngx_str_t *out, int port) { - (void)connection; (void)port; - out->data = (u_char *)"127.0.0.1"; out->len = 9; + (void)connection; (void)port; out->data = (u_char *)"127.0.0.1"; out->len = 9; return NGX_OK; } static ngx_pool_t *ngx_http_modsecurity_pcre_malloc_init(ngx_pool_t *pool) { return pool; } static void ngx_http_modsecurity_pcre_malloc_done(ngx_pool_t *pool) { (void)pool; } static void ngx_log_error(int level, void *log, int error, const char *format, ...) { - (void)level; (void)log; (void)error; (void)format; - ++error_calls; -} -static int phase_result(int phase) -{ - active_phase = phase; - ++native_calls[phase]; - return native_results[phase]; + (void)level; (void)log; (void)error; (void)format; ++error_calls; } +static int phase_result(int phase) { active_phase = phase; ++native_calls[phase]; return native_results[phase]; } static int msc_process_connection(Transaction *t, const char *client, int cp, const char *server, int sp) { - (void)t; (void)client; (void)cp; (void)server; (void)sp; - return phase_result(0); + (void)t; (void)client; (void)cp; (void)server; (void)sp; return phase_result(0); } static int msc_process_uri(Transaction *t, const char *uri, const char *method, const char *version) { - (void)t; (void)uri; (void)method; (void)version; - return phase_result(1); + (void)t; (void)uri; (void)method; (void)version; return phase_result(1); } static int msc_add_n_request_header(Transaction *t, const u_char *name, size_t nl, const u_char *value, size_t vl) { - (void)t; (void)name; (void)nl; (void)value; (void)vl; - return 1; + (void)t; (void)name; (void)nl; (void)value; (void)vl; return 1; } static int msc_process_request_headers(Transaction *t) { (void)t; return phase_result(2); } static int msc_process_request_body(Transaction *t) { (void)t; return phase_result(3); } static int msc_append_request_body(Transaction *t, const u_char *data, size_t size) { - (void)t; (void)data; (void)size; - active_phase = 4; - ++append_calls; - return append_result; + (void)t; (void)data; (void)size; active_phase = 4; ++append_calls; return append_result; } -static int msc_request_body_from_file(Transaction *t, const char *name) +static ngx_int_t ngx_http_modsecurity_capture_request_body_intervention(ngx_http_request_t *r) { - (void)t; (void)name; - return file_result; + ++capture_calls; + if (intervention_results[4] < 0) return NGX_ERROR; + if (pending_limit) r->ctx->request_intervention_pending = 1; + return NGX_OK; } static int ngx_http_modsecurity_process_intervention(Transaction *t, ngx_http_request_t *r, int early) { - (void)t; (void)r; (void)early; - ++intervention_calls[active_phase]; + (void)t; (void)early; ++intervention_calls[active_phase]; + if (!r->ctx->request_intervention_active) abort(); + if (active_phase == 3 && r->ctx->request_intervention_pending) { + r->ctx->request_intervention_pending = 0; + return 413; + } return intervention_results[active_phase]; } static ngx_int_t ngx_http_read_client_request_body(ngx_http_request_t *r, void (*callback)(ngx_http_request_t *)) { - (void)r; (void)callback; - return NGX_OK; + (void)r; (void)callback; return NGX_OK; +} +static void *ngx_pnalloc(ngx_pool_t *pool, size_t bytes) +{ + (void)pool; return !fail_scratch && bytes == sizeof(scratch) ? scratch : NULL; +} +static int ngx_fd_info(int fd, ngx_file_info_t *info) +{ + ++stat_calls; + if (fd != 7 || bad_stat) return NGX_FILE_ERROR; + info->size = actual_file_size - (changed_file && stat_calls > 1 ? 1 : 0); + info->regular = regular_file; + return 0; +} +static ssize_t ngx_read_file(ngx_file_t *file, u_char *out, size_t amount, off_t offset) +{ + (void)file; ++read_calls; + if (amount > max_read) max_read = amount; + if (fail_read || offset < 0 || (uintmax_t)offset + amount > sizeof(file_bytes)) return -1; + if (short_read) return (ssize_t)amount - 1; + memcpy(out, file_bytes + offset, amount); + return (ssize_t)amount; } static int msc_process_logging(Transaction *t) { - (void)t; - ++log_calls; - if (log_reenter && log_calls == 1) - nested_log_result = ngx_http_modsecurity_log_handler(active_request); + (void)t; ++log_calls; + if (log_reenter && log_calls == 1) nested_log_result = ngx_http_modsecurity_log_handler(active_request); return log_result; } +static ngx_int_t ngx_http_modsecurity_log_request_contexts(ngx_http_request_t *r) +{ + if (r == NULL || r->pool == NULL) return NGX_ERROR; + return r->ctx != NULL ? ngx_http_modsecurity_log_context(r->ctx) : NGX_OK; +} """ CASES = r""" @@ -197,8 +226,8 @@ u_char bytes[] = "abc"; ngx_buf_t buffer = {bytes, bytes + 3, 0}; ngx_chain_t chain = {&buffer, NULL}; - ngx_temp_file_t file = {{{0, NULL}}}; - int phase, before_calls, after_calls; + ngx_temp_file_t file = {{7, {8, (u_char *)"body.tmp"}}, 3}; + int phase, before_calls, after_calls, allowed = 0; ngx_int_t result; request.pool = &pool; request.connection = &connection; request.main = &request; request.conf = &conf; request.request_body = &body; @@ -207,73 +236,88 @@ request.method_name.data = (u_char *)"POST"; request.method_name.len = 4; connection.addr_text.data = (u_char *)"127.0.0.1"; connection.addr_text.len = 9; request.error_page = 1; - - if (strcmp(scenario, "fresh-error-page") == 0 || - strcmp(scenario, "foreign-direct") == 0) { + if (strcmp(scenario, "body-callback-no-context") == 0) { + ngx_http_modsecurity_request_read(&request); + CHECK(created_calls == 0 && native_calls[3] == 0); + return 0; + } + if (strcmp(scenario, "fresh-error-page") == 0 || strcmp(scenario, "foreign-direct") == 0) { previous.r = strcmp(scenario, "foreign-direct") == 0 ? &other_request : &request; previous.modsec_transaction = &transaction; previous.request_body_processed = 1; previous.intervention_triggered = 1; if (strcmp(scenario, "foreign-direct") == 0) request.ctx = &previous; else request.recovered = &previous; intervention_results[3] = parameter; - } else if (strcmp(scenario, "p1-status") == 0) { - intervention_results[2] = parameter; - } else if (strcmp(scenario, "p2-status") == 0) { - intervention_results[3] = parameter; - } else if (strcmp(scenario, "intervention-error") == 0) { + } else if (strcmp(scenario, "same-policy") == 0 || strcmp(scenario, "block-page") == 0) { + previous.r = &request; previous.modsec_transaction = &transaction; + previous.request_body_processed = 1; + previous.intervention_triggered = strcmp(scenario, "block-page") == 0; + request.recovered = &previous; recover_allowed = 1; + } else if (strcmp(scenario, "p1-status") == 0) intervention_results[2] = parameter; + else if (strcmp(scenario, "p2-status") == 0) intervention_results[3] = parameter; + else if (strcmp(scenario, "intervention-error") == 0) { intervention_results[parameter] = -1; if (parameter == 4) body.bufs = &chain; - } else if (strcmp(scenario, "native-error") == 0) { - phase = parameter / 10; - native_results[phase] = parameter % 10 - 1; - } else if (strcmp(scenario, "append") == 0) { - body.bufs = &chain; append_result = parameter; - } else if (strcmp(scenario, "file") == 0) { - file.file.name.data = (u_char *)"body.tmp"; file.file.name.len = 8; - body.temp_file = &file; file_result = parameter; - } else { - return 2; - } - + } else if (strcmp(scenario, "native-error") == 0) native_results[parameter / 10] = parameter % 10 - 1; + else if (strcmp(scenario, "append") == 0) { + body.bufs = &chain; append_result = parameter; allowed = parameter == 0 || parameter == 1; + } else if (strcmp(scenario, "file") == 0 || strcmp(scenario, "file-large") == 0 || strcmp(scenario, "file-reject") == 0) { + body.temp_file = &file; append_result = parameter; allowed = parameter == 0 || parameter == 1; + if (strcmp(scenario, "file-large") == 0) file.offset = actual_file_size = sizeof(file_bytes); + if (strcmp(scenario, "file-reject") == 0) pending_limit = 1; + } else if (strcmp(scenario, "file-error") == 0) { + body.temp_file = &file; + if (parameter == 0) file.file.fd = NGX_INVALID_FILE; + if (parameter == 1) file.offset = -1; + if (parameter == 2) bad_stat = 1; + if (parameter == 3) regular_file = 0; + if (parameter == 4) actual_file_size = 2; + if (parameter == 5) short_read = 1; + if (parameter == 6) fail_read = 1; + if (parameter == 7) changed_file = 1; + if (parameter == 8) fail_scratch = 1; + } else if (strcmp(scenario, "disabled") == 0) conf.enable = 0; + else if (strcmp(scenario, "prepare-error") == 0) prepare_failure = 1; + else return 2; result = ngx_http_modsecurity_access_handler(&request); - if (strcmp(scenario, "fresh-error-page") == 0 || - strcmp(scenario, "foreign-direct") == 0) { - CHECK(created_calls == 1 && recovered_calls == 0); + CHECK(prepare_calls == 1); + CHECK(created.request_intervention_active == 0); + if (strcmp(scenario, "fresh-error-page") == 0 || strcmp(scenario, "foreign-direct") == 0) { + CHECK(created_calls == 1 && recovered_calls == 1); CHECK(native_calls[2] == 1 && native_calls[3] == 1); CHECK(previous.request_body_processed && previous.intervention_triggered); CHECK(result == (parameter > 0 ? parameter : NGX_DECLINED)); + } else if (strcmp(scenario, "same-policy") == 0 || strcmp(scenario, "block-page") == 0) { + CHECK(result == NGX_DECLINED && created_calls == 0); + CHECK(native_calls[0] == 0 && native_calls[2] == 0 && native_calls[3] == 0); + } else if (strcmp(scenario, "disabled") == 0 || strcmp(scenario, "prepare-error") == 0) { + CHECK(result == (prepare_failure ? 500 : NGX_DECLINED) && created_calls == 0); } else if (strcmp(scenario, "p1-status") == 0) { - CHECK(result == parameter && created.intervention_triggered); - CHECK(native_calls[3] == 0); - } else if (strcmp(scenario, "p2-status") == 0) { - CHECK(result == parameter && created.intervention_triggered); + CHECK(result == parameter && created.intervention_triggered && native_calls[3] == 0); + } else if (strcmp(scenario, "p2-status") == 0 || strcmp(scenario, "file-reject") == 0) { + CHECK(result == (pending_limit ? 413 : parameter) && created.intervention_triggered); CHECK(native_calls[3] == 1 && created.request_body_processed); - } else if ((strcmp(scenario, "append") == 0 && (parameter == 0 || parameter == 1)) || - (strcmp(scenario, "file") == 0 && parameter == 1)) { - CHECK(result == NGX_DECLINED && native_calls[3] == 1); - CHECK(!created.intervention_failed && created.request_body_processed); + CHECK(!created.request_intervention_pending); + } else if (allowed) { + CHECK(result == NGX_DECLINED && native_calls[3] == 1 && created.request_body_processed); + CHECK(!created.intervention_failed); + if (body.temp_file != NULL) CHECK(stat_calls == 2 && read_calls == append_calls && capture_calls == append_calls); + if (strcmp(scenario, "file-large") == 0) CHECK(append_calls == 3 && max_read == 32768U); } else { - CHECK(result == 500 && created.intervention_failed); - CHECK(created.intervention_triggered); - if (strcmp(scenario, "native-error") == 0) { - phase = parameter / 10; - CHECK(intervention_calls[phase] == 0); - } - if (strcmp(scenario, "append") == 0 || strcmp(scenario, "file") == 0) - CHECK(native_calls[3] == 0); + CHECK(result == 500 && created.intervention_failed && created.intervention_triggered); + if (strcmp(scenario, "native-error") == 0) CHECK(intervention_calls[parameter / 10] == 0); + if (body.temp_file != NULL || strcmp(scenario, "append") == 0) CHECK(native_calls[3] == 0); before_calls = append_calls + log_calls; for (phase = 0; phase < 4; ++phase) before_calls += native_calls[phase]; - result = ngx_http_modsecurity_access_handler(&request); + CHECK(ngx_http_modsecurity_access_handler(&request) == 500); after_calls = append_calls + log_calls; for (phase = 0; phase < 4; ++phase) after_calls += native_calls[phase]; - CHECK(result == 500 && before_calls == after_calls); + CHECK(before_calls == after_calls); request.header_sent = 1; - CHECK(ngx_http_modsecurity_access_handler(&request) == NGX_ERROR); - CHECK(connection.error); + CHECK(ngx_http_modsecurity_access_handler(&request) == NGX_ERROR && connection.error); } return 0; } - static int run_logging(const char *scenario, int parameter) { ngx_pool_t pool = {0}; @@ -282,18 +326,14 @@ ngx_http_request_t request = {0}; ngx_int_t result; request.pool = &pool; request.connection = &connection; request.ctx = &ctx; - ctx.r = &request; ctx.modsec_transaction = &transaction; + ctx.r = &request; ctx.modsec_transaction = &transaction; ctx.lifetime.pool = &pool; active_request = &request; log_result = parameter; if (strcmp(scenario, "log-missing") == 0) ctx.modsec_transaction = NULL; if (strcmp(scenario, "log-no-context") == 0) request.ctx = NULL; if (strcmp(scenario, "log-reenter") == 0) log_reenter = 1; result = ngx_http_modsecurity_log_handler(&request); - if (strcmp(scenario, "log-no-context") == 0) { - CHECK(result == NGX_OK && log_calls == 0); - return 0; - } - CHECK(ctx.logged); - CHECK(result == (parameter == 1 && ctx.modsec_transaction != NULL ? NGX_OK : NGX_ERROR)); + if (strcmp(scenario, "log-no-context") == 0) { CHECK(result == NGX_OK && log_calls == 0); return 0; } + CHECK(ctx.logged && result == (parameter == 1 && ctx.modsec_transaction != NULL ? NGX_OK : NGX_ERROR)); CHECK(ctx.native_logging_failed == (result != NGX_OK)); CHECK(log_calls == (ctx.modsec_transaction != NULL ? 1 : 0)); if (log_reenter) CHECK(nested_log_result == NGX_ERROR); @@ -302,32 +342,27 @@ CHECK(!connection.error && !request.header_sent); return 0; } - int main(int argc, char **argv) { CHECK(argc == 3); - return strncmp(argv[1], "log-", 4) == 0 - ? run_logging(argv[1], atoi(argv[2])) - : run_access(argv[1], atoi(argv[2])); + return strncmp(argv[1], "log-", 4) == 0 ? run_logging(argv[1], atoi(argv[2])) : run_access(argv[1], atoi(argv[2])); } """ def unit_program(access: str, logging: str) -> str: helpers = "" - if "ngx_http_modsecurity_request_failure(" in access: - helpers += "\nstatic ngx_int_t\n" + function_definition( - access, "ngx_http_modsecurity_request_failure" - ) - helpers += "\nvoid\n" + function_definition( - access, "ngx_http_modsecurity_request_read" - ) - helpers += "\nngx_int_t\n" + function_definition( - access, "ngx_http_modsecurity_access_handler" - ) - helpers += "\nstatic ngx_int_t\n" + function_definition( - logging, "ngx_http_modsecurity_log_handler" - ) + for name, result in ( + ("ngx_http_modsecurity_request_failure", "static ngx_int_t"), + ("ngx_http_modsecurity_request_intervention", "static int"), + ("ngx_http_modsecurity_append_request_chunk", "static ngx_int_t"), + ("ngx_http_modsecurity_inspect_request_body_file", "static ngx_int_t"), + ("ngx_http_modsecurity_request_read", "void"), + ("ngx_http_modsecurity_access_handler", "ngx_int_t"), + ): + helpers += "\n" + result + "\n" + function_definition(access, name) + for name in ("ngx_http_modsecurity_log_context", "ngx_http_modsecurity_log_handler"): + helpers += "\nstatic ngx_int_t\n" + function_definition(logging, name) return PREAMBLE + helpers + CASES @@ -339,12 +374,16 @@ def unit_program(access: str, logging: str) -> str: typedef unsigned char u_char; typedef struct { size_t len; u_char *data; } ngx_str_t; typedef struct { int unused; } Transaction; +typedef long ngx_int_t; typedef struct { int unused; } ngx_array_t; typedef struct { int unused; } ngx_http_modsecurity_header_t; typedef struct ngx_http_request_s ngx_http_request_t; typedef struct { ngx_http_request_t *r; Transaction *modsec_transaction; + struct { void *pool; const void *conf; } lifetime; + struct { char *log, *url; } pending_request_intervention; + int request_intervention_pending, logged; ngx_array_t *sanity_headers_out; } ngx_http_modsecurity_ctx_t; typedef struct { void (*handler)(void *); void *data; } ngx_pool_cleanup_t; @@ -366,6 +405,7 @@ def unit_program(access: str, logging: str) -> str: #define ngx_http_get_module_loc_conf(r, module) ((r)->conf) #define ngx_http_set_ctx(r, value, module) ((r)->ctx = (value)) #define ngx_memcpy(dst, src, size) memcpy((dst), (src), (size)) +#define ngx_memzero(dst, size) memset((dst), 0, (size)) #define CHECK(condition) do { if (!(condition)) { \ fprintf(stderr, "line %d: %s\n", __LINE__, #condition); return 1; \ } } while (0) @@ -373,9 +413,14 @@ def unit_program(access: str, logging: str) -> str: static int allocation_calls, fail_allocation_at, complex_failure, native_failure; static int registration_failure, registration_calls, native_calls, cleanup_calls; static int sanity_failure, sanity_allocations, sanity_destroys; -static int custom_id_copied, custom_id_terminated; +static int custom_id_copied, custom_id_terminated, audit_attempts; static char custom_id[4] = {'i', 'd', '7', '!'}; static void ngx_http_modsecurity_cleanup(void *data); +static ngx_int_t ngx_http_modsecurity_log_context(ngx_http_modsecurity_ctx_t *ctx) +{ + if (!ctx->logged) { ctx->logged = 1; ++audit_attempts; } + return NGX_OK; +} static void *ngx_pnalloc(ngx_pool_t *pool, size_t bytes) { void *result; @@ -469,19 +514,20 @@ def unit_program(access: str, logging: str) -> str: if (expected_failure) { CHECK(ctx == NULL && request.ctx == NULL); CHECK(pool.cleanup.handler == NULL && pool.cleanup.data == NULL); - if (registration_failure || sanity_failure) CHECK(cleanup_calls == 1); + if (registration_failure || sanity_failure) CHECK(cleanup_calls == 1 && audit_attempts == 1); else CHECK(cleanup_calls == 0); if (fail_allocation_at || complex_failure) CHECK(native_calls == 0); return 0; } CHECK(ctx != NULL && ctx != NGX_CONF_ERROR && ctx == request.ctx); CHECK(native_calls == 1 && registration_calls == 1 && cleanup_calls == 0); + CHECK(ctx->lifetime.pool == &pool && ctx->lifetime.conf == &conf); CHECK(pool.cleanup.handler == ngx_http_modsecurity_cleanup && pool.cleanup.data == ctx); if (conf.transaction_id != NULL) CHECK(custom_id_copied && custom_id_terminated); ngx_http_modsecurity_cleanup(NULL); pool.cleanup.handler(pool.cleanup.data); pool.cleanup.handler(pool.cleanup.data); - CHECK(ctx->modsec_transaction == NULL && cleanup_calls == 1); + CHECK(ctx->modsec_transaction == NULL && cleanup_calls == 1 && audit_attempts == 1); CHECK(sanity_allocations == MODSECURITY_SANITY_CHECKS); CHECK(sanity_destroys == MODSECURITY_SANITY_CHECKS); return 0; @@ -499,6 +545,224 @@ def ownership_program(module: str) -> str: return OWNERSHIP_PREAMBLE + helpers + OWNERSHIP_CASES +POLICY_PREAMBLE = r""" +#include +#include +#include +#include +typedef unsigned char u_char; +typedef long ngx_int_t; +typedef unsigned long ngx_uint_t; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct { int result, logs, cleanups; } Transaction; +typedef struct ngx_http_request_s ngx_http_request_t; +typedef struct ngx_pool_cleanup_s { + void (*handler)(void *); void *data; struct ngx_pool_cleanup_s *next; +} ngx_pool_cleanup_t; +typedef struct { ngx_pool_cleanup_t *cleanup; } ngx_pool_t; +typedef struct { void *elts; ngx_uint_t nelts; } ngx_array_t; +typedef struct { ngx_str_t literal; int failed; } ngx_http_complex_value_t; +typedef struct { ngx_int_t status, overwrite; ngx_http_complex_value_t value; } ngx_http_err_page_t; +typedef struct { ngx_array_t *error_pages; int named; ngx_str_t name; } ngx_http_core_loc_conf_t; +typedef struct ngx_http_modsecurity_conf_s { + void *rules_set; const void *rules_identity; + int enable, use_error_log, sanity_checks_enabled; + void *transaction_id, *phase4_log_file; ngx_uint_t phase4_mode; +} ngx_http_modsecurity_conf_t; +typedef struct { + ngx_http_request_t *r; + struct { ngx_pool_t *pool; const ngx_http_modsecurity_conf_t *conf; } lifetime; + struct { ngx_str_t target; ngx_int_t status, response_status; } error_page; + Transaction *modsec_transaction; + struct { char *log, *url; } pending_request_intervention; + int logged, native_logging_failed, request_policy_superseded; + int request_intervention_active, request_intervention_pending, native_request_body_limit_rejection; + int intervention_triggered, intervention_failed, phase4_failed; + ngx_int_t last_intervention_status; + ngx_array_t *sanity_headers_out; +} ngx_http_modsecurity_ctx_t; +typedef struct { void *log; int error; } ngx_connection_t; +struct ngx_http_request_s { + ngx_pool_t *pool; ngx_connection_t *connection; + ngx_http_modsecurity_ctx_t *ctx; + ngx_http_modsecurity_conf_t *conf; + ngx_http_core_loc_conf_t *core; + int internal, error_page, err_status, header_sent; ngx_str_t uri; +}; +enum { ngx_http_modsecurity_module = 1, ngx_http_core_module = 2 }; +#define NGX_OK 0 +#define NGX_ERROR (-1) +#define NGX_HTTP_BAD_REQUEST 400 +#define NGX_LOG_ERR 3 +#define ngx_http_get_module_ctx(r, module) ((r)->ctx) +#define ngx_http_set_ctx(r, value, module) ((r)->ctx = (value)) +#define ngx_http_get_module_loc_conf(r, module) \ + ((module) == ngx_http_core_module ? (void *)(r)->core : (void *)(r)->conf) +#define ngx_strncmp(a, b, n) strncmp((const char *)(a), (const char *)(b), (n)) +#define ngx_memzero(dst, size) memset((dst), 0, (size)) +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "line %d: %s\n", __LINE__, #condition); return 1; \ +} } while (0) +static ngx_pool_t *last_pool_seen; +static void ngx_http_modsecurity_cleanup(void *data); +static ngx_pool_t *ngx_http_modsecurity_pcre_malloc_init(ngx_pool_t *pool) +{ + last_pool_seen = pool; + return pool; +} +static void ngx_http_modsecurity_pcre_malloc_done(ngx_pool_t *pool) { (void)pool; } +static void ngx_log_error(int level, void *log, int error, const char *format, ...) +{ + (void)level; (void)log; (void)error; (void)format; +} +static int msc_process_logging(Transaction *transaction) +{ + ++transaction->logs; + return transaction->result; +} +static void msc_transaction_cleanup(Transaction *transaction) +{ + ++transaction->cleanups; + if (transaction->cleanups != 1) abort(); +} +static int ngx_http_complex_value(ngx_http_request_t *r, ngx_http_complex_value_t *value, ngx_str_t *out) +{ + (void)r; *out = value->literal; return value->failed ? NGX_ERROR : NGX_OK; +} +static u_char *ngx_strlchr(u_char *first, u_char *last, u_char value) +{ + return memchr(first, value, (size_t)(last - first)); +} +#if MODSECURITY_SANITY_CHECKS +static void ngx_array_destroy(ngx_array_t *array) { (void)array; } +#endif +""" + +POLICY_CASES = r""" +int main(int argc, char **argv) +{ + Transaction first = {1, 0, 0}, second = {1, 0, 0}, foreign_transaction = {1, 0, 0}; + ngx_connection_t connection = {0}; + ngx_http_modsecurity_conf_t original = {0}, destination; + ngx_http_core_loc_conf_t core = {0}; + ngx_http_request_t request = {0}, other = {0}; + ngx_http_modsecurity_ctx_t origin = {0}, target = {0}, foreign = {0}; + ngx_pool_cleanup_t source_cleanup = {ngx_http_modsecurity_cleanup, &origin, NULL}; + ngx_pool_cleanup_t target_cleanup = {ngx_http_modsecurity_cleanup, &target, &source_cleanup}; + ngx_pool_cleanup_t foreign_cleanup = {ngx_http_modsecurity_cleanup, &foreign, NULL}; + ngx_pool_t pool = {&source_cleanup}; + ngx_http_err_page_t error_page = {403, -1, {{24, (u_char *)"/block.html?observation=1"}, 0}}; + ngx_array_t pages = {&error_page, 1}; + ngx_http_modsecurity_ctx_t *selected; + int parameter, expected; + CHECK(argc == 3); + parameter = atoi(argv[2]); + original.enable = 1; original.rules_set = &first; original.rules_identity = &first; + destination = original; destination.rules_set = &second; + request.pool = &pool; request.connection = &connection; + request.conf = &destination; request.core = &core; + request.uri.data = (u_char *)"/block.html"; request.uri.len = 11; + origin.r = &request; origin.lifetime.pool = &pool; origin.lifetime.conf = &original; + origin.modsec_transaction = &first; + target.r = &request; target.lifetime.pool = &pool; target.lifetime.conf = &destination; + target.modsec_transaction = &second; + foreign.r = &other; foreign.lifetime.pool = &pool; foreign.lifetime.conf = &original; + foreign.modsec_transaction = &foreign_transaction; + source_cleanup.next = &foreign_cleanup; + + if (strcmp(argv[1], "same-policy") == 0) { + CHECK(ngx_http_modsecurity_same_policy(&original, &destination)); + CHECK(ngx_http_modsecurity_prepare_location(&request, &destination) == NGX_OK); + CHECK(!origin.request_policy_superseded); + CHECK(ngx_http_modsecurity_get_module_ctx(&request) == &origin && request.ctx == &origin); + CHECK(ngx_http_modsecurity_log_handler(&request) == NGX_OK); + CHECK(ngx_http_modsecurity_log_handler(&request) == NGX_OK); + CHECK(first.logs == 1 && second.logs == 0 && foreign_transaction.logs == 0); + } else if (strcmp(argv[1], "policy-key") == 0) { + if (parameter == 0) destination.rules_identity = &second; + if (parameter == 1) destination.enable = 0; + if (parameter == 2) destination.use_error_log = 1; + if (parameter == 3) destination.transaction_id = &second; + if (parameter == 4) destination.phase4_mode = 1; + if (parameter == 5) destination.phase4_log_file = &second; + if (parameter == 6) destination.sanity_checks_enabled = 1; + expected = parameter != 6 || MODSECURITY_SANITY_CHECKS; + CHECK(ngx_http_modsecurity_prepare_location(&request, &destination) == NGX_OK); + CHECK(origin.request_policy_superseded == expected); + CHECK(ngx_http_modsecurity_get_module_ctx(&request) == (expected ? NULL : &origin)); + } else if (strcmp(argv[1], "changed-policy") == 0 || strcmp(argv[1], "failed-audit") == 0 || strcmp(argv[1], "disabled-target") == 0) { + destination.rules_identity = &second; + if (strcmp(argv[1], "disabled-target") == 0) destination.enable = 0; + CHECK(ngx_http_modsecurity_prepare_location(&request, &destination) == NGX_OK); + CHECK(origin.request_policy_superseded && ngx_http_modsecurity_get_module_ctx(&request) == NULL); + if (destination.enable) { pool.cleanup = &target_cleanup; request.ctx = ⌖ } + if (strcmp(argv[1], "failed-audit") == 0) first.result = 0; + expected = first.result == 1 ? NGX_OK : NGX_ERROR; + CHECK(ngx_http_modsecurity_log_handler(&request) == expected); + CHECK(ngx_http_modsecurity_log_handler(&request) == expected); + CHECK(first.logs == 1 && second.logs == (destination.enable ? 1 : 0)); + CHECK(foreign_transaction.logs == 0 && !connection.error && !request.header_sent); + } else if (strcmp(argv[1], "cleanup-fallback") == 0) { + request.pool = NULL; + ngx_http_modsecurity_cleanup(&origin); + ngx_http_modsecurity_cleanup(&origin); + CHECK(first.logs == 1 && first.cleanups == 1 && origin.modsec_transaction == NULL); + CHECK(last_pool_seen == &pool && foreign_transaction.logs == 0); + } else if (strcmp(argv[1], "block-page") == 0) { + destination.rules_identity = &second; + core.error_pages = &pages; + origin.request_intervention_active = 1; + origin.intervention_triggered = 1; origin.last_intervention_status = 403; + request.internal = 1; request.error_page = 1; request.err_status = 403; + if (parameter == 6) origin.native_request_body_limit_rejection = 1; + if (parameter == 7) origin.request_intervention_active = 0; + if (parameter == 8 || parameter == 9) { + error_page.value.literal.data = (u_char *)"@block"; error_page.value.literal.len = 6; + core.named = parameter == 8; core.name = error_page.value.literal; + } + if (parameter == 10 || parameter == 11) { + error_page.overwrite = parameter == 10 ? 200 : 0; + request.err_status = error_page.overwrite; + } + if (parameter == 14) { + error_page.value.literal.data = (u_char *)"https://example.test/error"; + error_page.value.literal.len = 26; + } + if (parameter == 15) error_page.value.failed = 1; + if (parameter != 13) ngx_http_modsecurity_capture_error_page(&request, &origin, 403); + origin.request_intervention_active = 0; + if (parameter == 1) { request.uri.data = (u_char *)"/different"; request.uri.len = 10; } + if (parameter == 2) request.err_status = 451; + if (parameter == 3) request.internal = 0; + if (parameter == 4) origin.intervention_failed = 1; + if (parameter == 5) origin.last_intervention_status = 451; + if (parameter == 12) origin.r = &other; + expected = parameter == 0 || parameter == 8 || parameter == 10 || parameter == 11; + selected = ngx_http_modsecurity_get_module_ctx(&request); + CHECK(selected == (expected ? &origin : NULL)); + CHECK(first.logs == 0 && second.logs == 0 && foreign_transaction.logs == 0); + } else return 2; + return 0; +} +""" + + +def policy_program(module: str, logging: str) -> str: + helpers = "\nngx_int_t\n" + function_definition(logging, "ngx_http_modsecurity_log_context") + for name, result in ( + ("ngx_http_modsecurity_same_policy", "static int"), + ("ngx_http_modsecurity_capture_error_page", "static void"), + ("ngx_http_modsecurity_is_block_error_page", "static int"), + ("ngx_http_modsecurity_prepare_location", "ngx_int_t"), + ("ngx_http_modsecurity_get_module_ctx", "ngx_http_modsecurity_ctx_t *"), + ("ngx_http_modsecurity_log_request_contexts", "ngx_int_t"), + ("ngx_http_modsecurity_cleanup", "static void"), + ): + helpers += "\n" + result + "\n" + function_definition(module, name) + helpers += "\nngx_int_t\n" + function_definition(logging, "ngx_http_modsecurity_log_handler") + return POLICY_PREAMBLE + helpers + POLICY_CASES + + class LifecycleRuntimeTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: @@ -537,6 +801,9 @@ def test_internal_error_page_uses_fresh_location_context(self): def test_foreign_direct_context_does_not_skip_access_processing(self): self.case("foreign-direct", 403) + def test_body_callback_without_a_recoverable_context_is_safe(self): + self.case("body-callback-no-context", 0) + def test_error_page_preserves_p1_and_p2_status_decisions(self): for scenario in ("p1-status", "p2-status"): for status in (302, 403, 451): @@ -559,11 +826,32 @@ def test_native_append_partial_is_not_phase_failure(self): with self.subTest(result=result): self.case("append", result) - def test_native_file_result_is_not_append_partial(self): + def test_file_and_memory_partial_results_have_equal_semantics(self): for result in (-1, 0, 1, 2): with self.subTest(result=result): self.case("file", result) + def test_file_chunks_are_bounded_and_partial_inspection_continues(self): + self.case("file-large", 0) + + def test_file_descriptor_read_and_allocation_failures_remain_errors(self): + for failure in range(9): + with self.subTest(failure=failure): + self.case("file-error", failure) + + def test_file_native_reject_is_dispatched_after_successful_eos(self): + self.case("file-reject", 1) + + def test_same_policy_and_native_block_page_do_not_repeat_request_phases(self): + for scenario in ("same-policy", "block-page"): + with self.subTest(scenario=scenario): + self.case(scenario, 0) + + def test_location_preparation_runs_before_disabled_or_failed_entry(self): + for scenario in ("disabled", "prepare-error"): + with self.subTest(scenario=scenario): + self.case(scenario, 0) + def test_audit_result_is_checked_and_attempted_once(self): for result in (-1, 0, 1, 2): with self.subTest(result=result): @@ -635,5 +923,70 @@ def test_cleanup_is_null_safe_and_idempotent(self): self.case("cleanup-once") +class LifecyclePolicyTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler or shutil.which(compiler[0]) is None: + raise RuntimeError("set CC to a GCC- or Clang-compatible C17 compiler") + workspace = tempfile.TemporaryDirectory(prefix="nginx-policy-") + cls.addClassCleanup(workspace.cleanup) + directory = Path(workspace.name) + source = directory / "policy.c" + source.write_text(policy_program( + (ROOT / "src/ngx_http_modsecurity_module.c").read_text(encoding="utf-8"), + (ROOT / "src/ngx_http_modsecurity_log.c").read_text(encoding="utf-8"), + ), encoding="utf-8") + cls.binaries = {} + for sanity in (0, 1): + binary = directory / (f"policy-{sanity}.exe" if os.name == "nt" else f"policy-{sanity}") + build = subprocess.run( + [*compiler, "-std=c17", "-Wall", "-Wextra", "-Werror", + f"-DMODSECURITY_SANITY_CHECKS={sanity}", str(source), "-o", str(binary)], + capture_output=True, text=True, check=False, timeout=60, + ) + if build.returncode: + raise AssertionError(f"Policy helper build failed:\n{build.stderr}") + cls.binaries[sanity] = binary + + def case(self, name: str, parameter: int = 0) -> None: + for sanity, binary in self.binaries.items(): + with self.subTest(sanity=sanity): + result = subprocess.run( + [str(binary), name, str(parameter)], capture_output=True, + text=True, check=False, timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_inherited_equivalent_policy_reuses_one_audited_owner(self): + self.case("same-policy") + + def test_semantic_policy_changes_require_a_new_owner(self): + for parameter in range(7): + with self.subTest(parameter=parameter): + self.case("policy-key", parameter) + + def test_changed_or_disabled_target_retains_source_audit(self): + for name in ("changed-policy", "disabled-target"): + with self.subTest(name=name): + self.case(name) + + def test_owner_audit_failure_is_sticky_without_skipping_other_owners(self): + self.case("failed-audit") + + def test_cleanup_audits_once_with_the_retained_pool(self): + self.case("cleanup-fallback") + + def test_exact_native_block_destination_can_reuse_its_origin(self): + for parameter in (0, 8, 10, 11): + with self.subTest(parameter=parameter): + self.case("block-page", parameter) + + def test_unrelated_routes_cannot_claim_the_block_page_exemption(self): + for parameter in (1, 2, 3, 4, 5, 6, 7, 9, 12, 13, 14, 15): + with self.subTest(parameter=parameter): + self.case("block-page", parameter) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_log_metadata_runtime.py b/tests/test_log_metadata_runtime.py new file mode 100644 index 00000000..e22b3777 --- /dev/null +++ b/tests/test_log_metadata_runtime.py @@ -0,0 +1,346 @@ +"""Compile the real standalone metadata/JSON header with small NGINX doubles. + +Expected records follow the complete native RuleMessage framing contract. +Python's strict UTF-8 decoder and JSON parser independently check the escaped +output. Invalid input bytes are represented as U+00xx; valid UTF-8 is retained. +These are helper regressions, not native HTTP or audit-log integration tests. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + +PREAMBLE = r""" +#include +#include +#include +#include +#include + +typedef unsigned char u_char; +typedef intptr_t ngx_int_t; +typedef struct { size_t len; u_char *data; } ngx_str_t; +typedef struct { int fail; unsigned calls; } ngx_pool_t; +#define ngx_inline inline + +static void *allocations[8]; +static unsigned allocation_count; + +static void *ngx_pnalloc(ngx_pool_t *pool, size_t size) +{ + void *result; + + pool->calls++; + if (pool->fail || allocation_count == 8) { + return NULL; + } + result = malloc(size == 0 ? 1 : size); + if (result != NULL) { + allocations[allocation_count++] = result; + } + return result; +} + +#include "ngx_http_modsecurity_log_metadata.h" +""" + +PROGRAM = PREAMBLE + r""" +static int from_hex(char value) +{ + if (value >= '0' && value <= '9') return value - '0'; + if (value >= 'a' && value <= 'f') return value - 'a' + 10; + if (value >= 'A' && value <= 'F') return value - 'A' + 10; + return -1; +} + +int main(int argc, char **argv) +{ + ngx_pool_t pool = {0, 0}; + ngx_str_t input, output = {5, (u_char *) "stale"}; + u_char *bytes; + size_t size, index; + ngx_int_t result; + int high, low; + + if (argc < 3 || strlen(argv[2]) % 2 != 0) return 2; + size = strlen(argv[2]) / 2; + /* Deliberately do not append a NUL: the production API is len-bounded. */ + bytes = malloc(size == 0 ? 1 : size); + if (bytes == NULL) return 2; + for (index = 0; index < size; index++) { + high = from_hex(argv[2][index * 2]); + low = from_hex(argv[2][index * 2 + 1]); + if (high < 0 || low < 0) return 2; + bytes[index] = (u_char) (high * 16 + low); + } + input.data = bytes; + input.len = size; + if (argc > 3) pool.fail = atoi(argv[3]); + if (argc > 4) { + input.len = (size_t) strtoull(argv[4], NULL, 10); + if (input.len > size) return 2; + } + if (strcmp(argv[1], "json-oversize") == 0) input.len = SIZE_MAX; + if (strcmp(argv[1], "json-null") == 0 || + strcmp(argv[1], "metadata-null") == 0) input.data = NULL; + + if (strncmp(argv[1], "json", 4) == 0) { + ngx_http_modsecurity_json_escape(&pool, &input, &output); + if (output.data == NULL) { + if (output.len != 0) return 3; + fputs("NULL", stdout); + } else { + if (output.data[output.len] != '\0') return 3; + fwrite(output.data, 1, output.len, stdout); + } + } else if (strncmp(argv[1], "metadata", 8) == 0) { + result = ngx_http_modsecurity_parse_rule_metadata(input, &output); + if (pool.calls != 0) return 3; + printf("%ld\n", (long) result); + if (result != 1 && output.len != 0) return 3; + fwrite(output.data, 1, output.len, stdout); + } else if (strcmp(argv[1], "extract") == 0) { + ngx_http_modsecurity_extract_rule_id(&pool, &input, &output); + /* The copied ID must survive overwriting the temporary input. */ + memset(bytes, 'x', size); + if (output.data == NULL) { + if (output.len != 0) return 3; + fputs("NULL", stdout); + } else { + if (output.data[output.len] != '\0') return 3; + fwrite(output.data, 1, output.len, stdout); + } + } else if (strcmp(argv[1], "summary") == 0) { + output = ngx_http_modsecurity_sanitize_intervention(&pool, input); + if (pool.calls != 0) return 3; + fwrite(output.data, 1, output.len, stdout); + } else { + return 2; + } + free(bytes); + for (index = 0; index < allocation_count; index++) free(allocations[index]); + return 0; +} +""" + +PARSER_ONLY = PREAMBLE + r""" +int main(void) +{ + ngx_str_t input = {0, (u_char *) ""}; + ngx_str_t output; + + return ngx_http_modsecurity_parse_rule_metadata(input, &output) != 0; +} +""" + + +def engine_record(*, rule_id: str = "942100", changes=None, tags=(), tail=False) -> bytes: + fields = [ + ("file", "/rules/response.conf"), + ("line", "12"), + ("id", rule_id), + ("rev", ""), + ("msg", "response rule"), + ("data", r"Matched \x22data\x22"), + ("severity", "2"), + ("ver", "test"), + ("maturity", "0"), + ("accuracy", "0"), + ] + fields += [("tag", value) for value in tags] + fields += [ + ("hostname", "example.invalid"), + ("uri", "/response"), + ("unique_id", "transaction"), + ("ref", "o0,4v0,4"), + ] + if changes: + fields = [(name, changes.get(name, value)) for name, value in fields] + if tail: + fields += [("hostname", "example.invalid"), ("uri", "/response"), + ("unique_id", "transaction")] + return " ".join(f'[{name} "{value}"]' for name, value in fields).encode("ascii") + + +class LogMetadataRuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.compiler = shlex.split(os.environ.get("CC", "cc")) + if not cls.compiler or shutil.which(cls.compiler[0]) is None: + raise unittest.SkipTest("A C compiler is needed for metadata helper regressions") + cls.directory = tempfile.TemporaryDirectory(prefix="modsec-log-metadata-") + cls.addClassCleanup(cls.directory.cleanup) + cls.binary = Path(cls.directory.name) / "metadata" + cls.parser_binary = Path(cls.directory.name) / "parser-only" + for name, source, binary in ( + ("metadata.c", PROGRAM, cls.binary), + ("parser-only.c", PARSER_ONLY, cls.parser_binary), + ): + path = Path(cls.directory.name) / name + path.write_text(source, encoding="utf-8") + completed = subprocess.run( + cls.compiler + ["-std=c99", "-Wall", "-Wextra", "-Werror", + "-I", str(ROOT / "src"), str(path), "-o", str(binary)], + capture_output=True, text=True, check=False, + ) + if completed.returncode != 0: + raise AssertionError(completed.stdout + completed.stderr) + + def execute(self, operation: str, data: bytes, *, fail=False, length=None) -> bytes: + command = [str(self.binary), operation, data.hex(), str(int(fail))] + if length is not None: + command.append(str(length)) + completed = subprocess.run(command, capture_output=True, check=False) + self.assertEqual(completed.returncode, 0, completed.stderr.decode("utf-8", "replace")) + return completed.stdout + + def parse(self, data: bytes, **kwargs): + result, rule_id = self.execute("metadata", data, **kwargs).split(b"\n", 1) + return int(result), rule_id + + def escaped(self, data: bytes, **kwargs): + output = self.execute("json", data, **kwargs) + text = output.decode("utf-8", "strict") + return output, json.loads('"' + text + '"') + + def test_header_can_be_included_for_only_the_parser_without_unused_warnings(self): + completed = subprocess.run([str(self.parser_binary)], capture_output=True, check=False) + self.assertEqual(completed.returncode, 0, completed.stderr) + + def test_complete_record_and_optional_error_log_tail(self): + for tail in (False, True): + with self.subTest(tail=tail): + self.assertEqual(self.parse(engine_record(tail=tail)), (1, b"942100")) + + def test_decimal_ids_are_preserved_without_integer_conversion(self): + value = "000" + "9" * 100 + self.assertEqual(self.parse(engine_record(rule_id=value)), (1, value.encode("ascii"))) + + def test_reflected_bare_id_before_actual_metadata_does_not_win(self): + data = b'Matched data: id "1" Operator msg "reflected" ' + engine_record() + self.assertEqual(self.parse(data), (1, b"942100")) + self.assertEqual(self.execute("summary", data), b"id:present msg:present op:-") + + def test_bare_presence_snippets_remain_redacted(self): + for data in (b'id "1"', b'Operator', b'msg "body"', b'id "1" Operator'): + with self.subTest(data=data): + self.assertEqual(self.parse(data), (0, b"")) + self.assertEqual(self.execute("summary", data), b"redacted") + + def test_forged_or_duplicate_records_are_unknown(self): + real = engine_record() + for data in ( + engine_record(rule_id="1") + b" " + real, + b'[file "malformed" ' + real, + engine_record(changes={"uri": '/[file "forged'}), + b"prefix" + real, + ): + with self.subTest(data=data): + self.assertEqual(self.parse(data), (-1, b"")) + self.assertEqual(self.execute("summary", data), b"redacted") + + def test_complete_order_and_terminal_framing_are_required(self): + record = engine_record() + invalid = ( + record[:-1], + record + b" trailing", + record.replace(b'[rev ""] ', b""), + record.replace(b'[line "12"] [id "942100"]', b'[id "942100"] [line "12"]'), + record + b' [hostname "example.invalid"]', + ) + for data in invalid: + with self.subTest(data=data): + self.assertEqual(self.parse(data), (-1, b"")) + + def test_native_numeric_fields_must_be_nonempty_decimal(self): + for field in ("line", "id", "severity", "maturity", "accuracy"): + for value in ("", "-1", "1x"): + with self.subTest(field=field, value=value): + self.assertEqual(self.parse(engine_record(changes={field: value})), (-1, b"")) + + def test_raw_quotes_and_hex_escaped_fields_follow_engine_framing(self): + record = engine_record(changes={"file": '/rules/quoted"name.conf'}, + tags=(r"attack-\x22tag\x22", "second")) + self.assertEqual(self.parse(record), (1, b"942100")) + for value in (r"bad\z", r"bad\x2", r"bad\xGG", 'raw"quote'): + with self.subTest(value=value): + self.assertEqual(self.parse(engine_record(changes={"data": value})), (-1, b"")) + + def test_metadata_is_length_bounded_and_rejects_embedded_controls(self): + record = engine_record() + self.assertEqual(self.parse(record + b' [file "outside"]', length=len(record)), + (1, b"942100")) + for character in (b"\0", b"\n", b"\x80"): + data = record.replace(b"response rule", b"response" + character + b"rule") + self.assertEqual(self.parse(data), (-1, b"")) + self.assertEqual(self.execute("metadata-null", b""), b"0\n") + self.assertEqual(self.execute("metadata-null", b"x"), b"-1\n") + + def test_extracted_id_is_an_owned_nul_terminated_copy(self): + self.assertEqual(self.execute("extract", engine_record()), b"942100") + self.assertEqual(self.execute("extract", b'id "1"'), b"") + self.assertEqual(self.execute("extract", engine_record(), fail=True), b"NULL") + + def test_summary_is_complete_redacted_and_needs_no_allocation(self): + record = engine_record(changes={"msg": "Operator private matched-data"}) + expected = b"id:present msg:present op:-" + self.assertEqual(self.execute("summary", record), expected) + self.assertEqual(self.execute("summary", record, fail=True), expected) + self.assertNotIn(b"private", self.execute("summary", record)) + + def test_all_bytes_produce_strict_json_without_losing_controls(self): + data = bytes(range(256)) + escaped, value = self.escaped(data) + self.assertEqual(value, "".join(chr(character) for character in range(256))) + self.assertIn(b"\\u0000", escaped) + self.assertIn(b"\\u001f", escaped) + self.assertIn(b"\\u00ff", escaped) + + def test_valid_utf8_is_preserved_byte_for_byte(self): + text = "Grüße 日本語 😀\u2028\u2029" + data = text.encode("utf-8") + escaped, value = self.escaped(data) + self.assertEqual(escaped, data) + self.assertEqual(value, text) + + def test_valid_utf8_boundary_sequences_are_retained(self): + data = b"\xc2\x80\xdf\xbf\xe0\xa0\x80\xed\x9f\xbf\xee\x80\x80\xef\xbf\xbf\xf0\x90\x80\x80\xf4\x8f\xbf\xbf" + escaped, value = self.escaped(data) + self.assertEqual(escaped, data) + self.assertEqual(value, data.decode("utf-8", "strict")) + + def test_invalid_and_truncated_utf8_bytes_are_explicitly_escaped(self): + invalid = ( + b"\x80\xbf", b"\xc0\xaf", b"\xc1\xbf", b"\xe0\x80\xaf", + b"\xed\xa0\x80", b"\xf0\x80\x80\x80", b"\xf4\x90\x80\x80", + b"\xf5\x80\x80\x80", b"\xff", b"\xc2", b"\xe1\x80", b"\xf1\x80\x80", + ) + for data in invalid: + with self.subTest(data=data): + escaped, value = self.escaped(data) + self.assertTrue(escaped.isascii()) + self.assertEqual(value, "".join(chr(character) for character in data)) + + def test_json_respects_declared_length(self): + self.assertEqual(self.escaped(b"ok\xff", length=2), (b"ok", "ok")) + + def test_json_allocation_failure_is_distinct_from_valid_empty(self): + self.assertEqual(self.execute("json", b"ordinary", fail=True), b"NULL") + self.assertEqual(self.execute("json", b"", fail=True), b"") + self.assertEqual(self.execute("json-null", b""), b"") + self.assertEqual(self.execute("json-null", b"x"), b"NULL") + self.assertEqual(self.execute("json-oversize", b"x"), b"NULL") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_phase4_runtime.py b/tests/test_phase4_runtime.py index 9777a356..2d06710d 100644 --- a/tests/test_phase4_runtime.py +++ b/tests/test_phase4_runtime.py @@ -49,6 +49,9 @@ def function_definition(source: str, name: str) -> str: PREAMBLE = r""" +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 64 +#endif #include #include #include @@ -60,20 +63,24 @@ def function_definition(source: str, name: str) -> str: #endif typedef unsigned char u_char; typedef int ngx_int_t; +typedef struct { size_t len; u_char *data; } ngx_str_t; typedef struct ngx_http_request_s ngx_http_request_t; +typedef struct ngx_http_modsecurity_conf_s ngx_http_modsecurity_conf_t; +typedef struct { int named; ngx_str_t name; } ngx_http_core_loc_conf_t; typedef struct ngx_pool_cleanup_s { void (*handler)(void *); void *data; struct ngx_pool_cleanup_s *next; } ngx_pool_cleanup_t; typedef struct { ngx_pool_cleanup_t *cleanup; } ngx_pool_t; -typedef struct { int error; void *log; } ngx_connection_t; +typedef struct { int error; void *log; int buffered; } ngx_connection_t; typedef struct { u_char *data; size_t length; } ngx_file_t; typedef struct { u_char *pos, *last; off_t file_pos, file_last; ngx_file_t *file; unsigned memory, in_file, last_buf, last_in_chain; + unsigned flush, sync; } ngx_buf_t; typedef struct ngx_chain_s { ngx_buf_t *buf; @@ -85,16 +92,25 @@ def function_definition(source: str, name: str) -> str: int phase4_strict_abort, phase4_headers_checked, intervention_triggered; int intervention_late, intervention_failed, response_replaced, last_intervention_status; int phase4_failed, phase4_error_emitting, phase4_started, intervention_body_limit; + int phase4_append_active, replacement_body_terminal_sent; + int logged, request_policy_superseded; + struct { ngx_pool_t *pool; const ngx_http_modsecurity_conf_t *conf; } lifetime; + struct { ngx_str_t target; ngx_int_t status, response_status; } error_page; ngx_http_request_t *r; void *modsec_transaction; u_char *phase4_file_scratch; } ngx_http_modsecurity_ctx_t; -typedef struct { +struct ngx_http_modsecurity_conf_s { unsigned phase4_mode; size_t phase4_body_limit; -} ngx_http_modsecurity_conf_t; + int enable, use_error_log; + void *rules_identity, *transaction_id, *phase4_log_file; +}; struct ngx_http_request_s { int header_sent, filter_finalize; + int buffered, internal, err_status; + ngx_str_t uri; + ngx_http_core_loc_conf_t *core_conf; struct ngx_http_request_s *main; ngx_pool_t *pool; ngx_connection_t *connection; @@ -113,8 +129,12 @@ def function_definition(source: str, name: str) -> str: #define NGX_HTTP_MODSECURITY_PHASE4_FILE_READ_CHUNK 32768U #define ngx_buf_in_memory(buffer) ((buffer)->memory) #define ngx_http_get_module_ctx(r, module) ((r)->ctx) -#define ngx_http_get_module_loc_conf(r, module) ((r)->conf) -#define ngx_log_error(...) ((void)0) +#define ngx_http_core_module 1 +#define ngx_http_get_module_loc_conf(r, module) \ + ((module) == ngx_http_core_module ? (void *)(r)->core_conf : (void *)(r)->conf) +#define ngx_http_set_ctx(r, value, module) ((r)->ctx = (value)) +#define ngx_strncmp(a, b, n) strncmp((const char *)(a), (const char *)(b), (n)) +#define ngx_log_error(...) (++diagnostic_calls) static int ngx_http_modsecurity_module; static ngx_http_modsecurity_ctx_t *ngx_http_modsecurity_get_module_ctx(ngx_http_request_t *r); static void ngx_http_modsecurity_cleanup(void *data) { (void)data; } @@ -122,7 +142,12 @@ def function_definition(source: str, name: str) -> str: static int fail_allocation, fail_read, short_read, fail_append, append_result = 1; static int engine_result = 1, native_result, native_late, native_failed, native_body_limit; static int log_result, event_calls, pcre_enter, pcre_leave; +static int diagnostic_calls, native_only_after_eos, append_collections, eos_with_append_active; static int forward_calls, forwarded_links, forward_result; +static int verify_writer, writer_rejected_zero, forwarded_controls, forwarded_eos; +static int control_link_calls, control_buffer_calls, fail_control_link, fail_control_buffer; +static ngx_chain_t control_links[16]; +static ngx_buf_t control_buffers[16]; static int finalize_calls, emulate_core_response, core_response_result; static ngx_chain_t *core_response; ngx_int_t ngx_http_modsecurity_body_filter(ngx_http_request_t *, ngx_chain_t *); @@ -130,6 +155,18 @@ def function_definition(source: str, name: str) -> str: static u_char scratch[32768], inspected[200000]; static const char *last_action, *last_reason; +static ngx_chain_t *ngx_alloc_chain_link(ngx_pool_t *pool) +{ + (void)pool; + if (fail_control_link || control_link_calls >= 16) return NULL; + return &control_links[control_link_calls++]; +} +static ngx_buf_t *ngx_calloc_buf(ngx_pool_t *pool) +{ + (void)pool; + if (fail_control_buffer || control_buffer_calls >= 16) return NULL; + return &control_buffers[control_buffer_calls++]; +} static void *ngx_pnalloc(ngx_pool_t *pool, size_t bytes) { (void)pool; @@ -159,7 +196,8 @@ def function_definition(source: str, name: str) -> str: } static int msc_process_response_body(void *transaction) { - (void)transaction; + ngx_http_modsecurity_ctx_t *ctx = transaction; + if (ctx->phase4_append_active) ++eos_with_append_active; ++process_calls; return engine_result; } @@ -179,6 +217,8 @@ def function_definition(source: str, name: str) -> str: ngx_http_modsecurity_ctx_t *ctx = ngx_http_modsecurity_get_module_ctx(r); (void)transaction; (void)early; if (ctx == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; + if (ctx->phase4_append_active) ++append_collections; + if (native_only_after_eos && !ctx->phase4_processed) return 0; ctx->intervention_late = native_late; ctx->intervention_failed = native_failed; ctx->intervention_body_limit = native_body_limit; @@ -210,7 +250,22 @@ def function_definition(source: str, name: str) -> str: { (void)r; ++forward_calls; - for (; in != NULL; in = in->next) ++forwarded_links; + for (; in != NULL; in = in->next) { + ngx_buf_t *buffer = in->buf; + int special = buffer != NULL && !buffer->memory && !buffer->in_file && + (buffer->flush || buffer->sync || buffer->last_buf); + int empty = buffer == NULL || (buffer->memory + ? buffer->pos == buffer->last + : (!buffer->in_file || buffer->file_pos == buffer->file_last)); + if (verify_writer && empty && !special) { + ++writer_rejected_zero; + return NGX_ERROR; + } + ++forwarded_links; + if (special) ++forwarded_controls; + if (buffer != NULL && (buffer->last_buf || + (r != r->main && buffer->last_in_chain))) ++forwarded_eos; + } return forward_result; } #define CHECK(condition) do { if (!(condition)) { \ @@ -222,7 +277,10 @@ def function_definition(source: str, name: str) -> str: int main(int argc, char **argv) { ngx_http_modsecurity_ctx_t ctx = {0}; - ngx_http_modsecurity_conf_t conf = {NGX_HTTP_MODSEC_PHASE4_MODE_SAFE, 1048576U}; + ngx_http_modsecurity_conf_t conf = { + .phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_SAFE, + .phase4_body_limit = 1048576U, .enable = 1 + }; ngx_pool_t pool = {0}; ngx_connection_t connection = {0}; ngx_http_request_t request = {0}; @@ -238,7 +296,7 @@ def function_definition(source: str, name: str) -> str: CHECK(argc == 2); memset(payload, 'A', sizeof(payload)); request.pool = &pool; request.connection = &connection; - request.ctx = &ctx; request.conf = &conf; request.main = &request; ctx.r = &request; ctx.modsec_transaction = &ctx; + request.ctx = &ctx; request.conf = &conf; request.main = &request; ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; buffer.file = &file; buffer.in_file = 1; buffer.file_last = sizeof(payload); if (strcmp(argv[1], "request-ownership") == 0) { @@ -246,7 +304,9 @@ def function_definition(source: str, name: str) -> str: ngx_http_request_t subrequest = {0}; ngx_pool_cleanup_t owner_cleanup = {ngx_http_modsecurity_cleanup, &ctx, NULL}; ngx_pool_cleanup_t other_cleanup = {ngx_http_modsecurity_cleanup, &other_ctx, &owner_cleanup}; - other_ctx.r = &subrequest; + other_ctx.r = &subrequest; other_ctx.modsec_transaction = &other_ctx; + other_ctx.lifetime.pool = &pool; other_ctx.lifetime.conf = &conf; + subrequest.conf = &conf; subrequest.pool = &pool; subrequest.main = &request; pool.cleanup = &other_cleanup; CHECK(ngx_http_modsecurity_get_module_ctx(&request) == &ctx); @@ -256,6 +316,7 @@ def function_definition(source: str, name: str) -> str: subrequest.ctx = &ctx; /* Reject a foreign direct context too. */ CHECK(ngx_http_modsecurity_get_module_ctx(&subrequest) == &other_ctx); pool.cleanup = &owner_cleanup; /* Auth subrequest has no own transaction. */ + subrequest.ctx = NULL; CHECK(ngx_http_modsecurity_get_module_ctx(&subrequest) == NULL); subrequest.ctx = NULL; subrequest.connection = &connection; final.last_in_chain = 1; @@ -266,7 +327,7 @@ def function_definition(source: str, name: str) -> str: CHECK(process_calls == 1 && ctx.phase4_processed && forward_calls == 2); } else if (strcmp(argv[1], "all-modes-large") == 0) { for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, 1048577U, &allowed) == NGX_OK); CHECK(allowed == 1048577U && ctx.response_body_bytes_seen == allowed); @@ -276,7 +337,7 @@ def function_definition(source: str, name: str) -> str: } else if (strcmp(argv[1], "all-modes-multiple") == 0) { size_t chunk; for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; for (chunk = 0; chunk < 3; ++chunk) { CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, 1048576U, &allowed) == NGX_OK); @@ -289,7 +350,7 @@ def function_definition(source: str, name: str) -> str: } } else if (strcmp(argv[1], "legacy-boundary") == 0) { for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, 1048575U, &allowed) == NGX_OK); ctx.response_body_bytes_inspected += allowed; @@ -308,7 +369,7 @@ def function_definition(source: str, name: str) -> str: buffer.in_file = 0; buffer.memory = 1; buffer.pos = payload; buffer.last = payload + sizeof(payload); for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; append_calls = 0; appended_bytes = 0; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, &conf, &buffer) == NGX_OK); @@ -322,21 +383,22 @@ def function_definition(source: str, name: str) -> str: } } else if (strcmp(argv[1], "overflow") == 0) { for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; ctx.response_body_bytes_seen = SIZE_MAX - 1U; ctx.response_body_bytes_inspected = SIZE_MAX - 1U; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, - 2U, &allowed) == NGX_ERROR); - CHECK(allowed == 0U && ctx.response_body_truncated); + 2U, &allowed) == NGX_OK); + CHECK(allowed == 2U && !ctx.response_body_truncated); CHECK(ctx.response_body_bytes_seen == SIZE_MAX); CHECK(ctx.response_body_bytes_inspected == SIZE_MAX - 1U); CHECK(ngx_http_modsecurity_append_response_body_chunk(&ctx, - payload, 2U) == NGX_ERROR); - CHECK(append_calls == 0); + payload, 2U) == NGX_OK); + CHECK(ctx.response_body_bytes_inspected == SIZE_MAX); + CHECK(append_calls == (int)i + 1 && ctx.phase4_append_active); } } else if (strcmp(argv[1], "invalid-accounting") == 0) { for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; ctx.response_body_bytes_inspected = 1U; CHECK(ngx_http_modsecurity_plan_limited_response_body(&ctx, &conf, 1U, &allowed) == NGX_ERROR); @@ -371,7 +433,7 @@ def function_definition(source: str, name: str) -> str: conf.phase4_body_limit = 65536U; buffer.memory = 1; buffer.pos = payload; buffer.last = payload + sizeof(payload); for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; append_calls = 0; appended_bytes = 0; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, &conf, &buffer) == NGX_OK); @@ -387,7 +449,7 @@ def function_definition(source: str, name: str) -> str: } else if (strcmp(argv[1], "file-over-legacy-limit") == 0) { conf.phase4_body_limit = 65536U; for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; read_calls = 0; append_calls = 0; allocation_calls = 0; appended_bytes = 0; read_max = 0; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, @@ -435,7 +497,7 @@ def function_definition(source: str, name: str) -> str: } else if (strcmp(argv[1], "append-error") == 0) { fail_append = 1; for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, &conf, &buffer) == NGX_ERROR); CHECK(ctx.response_body_bytes_inspected == 0U); @@ -446,7 +508,7 @@ def function_definition(source: str, name: str) -> str: buffer.in_file = 0; buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 10; for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; append_calls = 0; appended_bytes = 0; CHECK(ngx_http_modsecurity_append_response_body_buffer(&request, &ctx, &conf, &buffer) == NGX_OK); @@ -460,7 +522,7 @@ def function_definition(source: str, name: str) -> str: CHECK(ctx.phase4_processed); } else if (strcmp(argv[1], "engine-error") == 0) { for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; conf.phase4_mode = (unsigned)i; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; request.header_sent = 1; connection.error = 0; engine_result = -1; CHECK(ngx_http_modsecurity_process_final_response_body(&request, &ctx, &conf) == NGX_ERROR); CHECK(ctx.phase4_processed && ctx.intervention_triggered && connection.error); @@ -512,15 +574,105 @@ def function_definition(source: str, name: str) -> str: } else if (strcmp(argv[1], "replaced-body") == 0) { ctx.response_replaced = 1; buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 10; + final.last_buf = 1; verify_writer = 1; CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); - CHECK(buffer.pos == buffer.last && !buffer.in_file); + CHECK(buffer.pos == buffer.last && buffer.in_file); CHECK(buffer.file_last == buffer.file_pos); - CHECK(append_calls == 0 && process_calls == 0 && forwarded_links == 2); + CHECK(append_calls == 0 && process_calls == 0 && forwarded_links == 1); + CHECK(forwarded_controls == 1 && forwarded_eos == 1 && writer_rejected_zero == 0); + CHECK(ctx.replacement_body_terminal_sent && chain.next == &tail); + } else if (strcmp(argv[1], "replaced-data-only") == 0) { + ctx.response_replaced = 1; verify_writer = 1; chain.next = NULL; + buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 10; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(buffer.pos == buffer.last && buffer.file_pos == buffer.file_last); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_OK); + CHECK(forward_calls == 0 && control_link_calls == 0 && writer_rejected_zero == 0); + CHECK(append_calls == 0 && process_calls == 0 && !ctx.replacement_body_terminal_sent); + } else if (strcmp(argv[1], "replaced-buffered-drain") == 0) { + ctx.response_replaced = 1; verify_writer = 1; chain.next = NULL; + request.buffered = 1; forward_result = NGX_AGAIN; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_AGAIN); + CHECK(forward_calls == 1 && forwarded_links == 0 && writer_rejected_zero == 0); + request.buffered = 0; connection.buffered = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_AGAIN); + CHECK(forward_calls == 2 && forwarded_links == 0); + connection.buffered = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_OK); + CHECK(forward_calls == 2 && control_link_calls == 0); + } else if (strcmp(argv[1], "replaced-terminal-again") == 0) { + ctx.response_replaced = 1; verify_writer = 1; chain.next = NULL; + buffer.memory = 1; buffer.pos = payload; buffer.last = payload + 10; + buffer.last_buf = 1; request.buffered = 1; forward_result = NGX_AGAIN; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_AGAIN); + CHECK(ctx.replacement_body_terminal_sent && forwarded_eos == 1); + CHECK(control_link_calls == 1 && forwarded_controls == 1 && writer_rejected_zero == 0); + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_AGAIN); + CHECK(forward_calls == 2 && forwarded_eos == 1 && forwarded_links == 1); + request.buffered = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_OK); + CHECK(forward_calls == 2 && control_link_calls == 1 && chain.next == NULL); + } else if (strcmp(argv[1], "replaced-subrequest-eos") == 0) { + ctx.response_replaced = 1; verify_writer = 1; request.main = &main_request; + chain.next = NULL; buffer.last_in_chain = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(forwarded_controls == 1 && forwarded_eos == 1 && writer_rejected_zero == 0); + CHECK(ctx.replacement_body_terminal_sent && control_buffers[0].sync); + CHECK(!control_buffers[0].last_buf && control_buffers[0].last_in_chain); + } else if (strcmp(argv[1], "replaced-main-chain-marker") == 0) { + ctx.response_replaced = 1; verify_writer = 1; chain.next = NULL; + buffer.last_in_chain = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(!ctx.replacement_body_terminal_sent && forwarded_eos == 0); + CHECK(control_buffers[0].sync && control_buffers[0].last_in_chain); + final.last_buf = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &tail) == NGX_OK); + CHECK(ctx.replacement_body_terminal_sent && forwarded_eos == 1); + CHECK(forwarded_controls == 2 && writer_rejected_zero == 0); + } else if (strcmp(argv[1], "replaced-control-allocation-failure") == 0) { + for (i = 0; i < 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; + ctx.response_replaced = 1; request.header_sent = 1; connection.error = 0; + buffer.file_pos = 0; buffer.last_buf = 1; chain.next = NULL; + fail_control_link = i == 0; fail_control_buffer = i != 0; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); + CHECK(ctx.phase4_failed && connection.error && forward_calls == 0); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_ERROR); + } + } else if (strcmp(argv[1], "counter-saturation-stream") == 0) { + chain.next = NULL; request.header_sent = 1; + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; + buffer.pos = payload; buffer.last = payload + 10; + for (i = 0; i <= 2; ++i) { + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; + ctx.response_body_bytes_seen = SIZE_MAX - 1U; + ctx.response_body_bytes_inspected = SIZE_MAX - 1U; + append_calls = 0; appended_bytes = 0; process_calls = 0; forward_calls = 0; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(ctx.response_body_bytes_seen == SIZE_MAX && ctx.response_body_bytes_inspected == SIZE_MAX); + CHECK(append_calls == 1 && appended_bytes == 10U && process_calls == 1 && forward_calls == 1); + CHECK(!ctx.phase4_failed && !ctx.response_body_truncated && !connection.error); + } + } else if (strcmp(argv[1], "large-file-first-chunk") == 0) { + if (sizeof(off_t) < 8) return 0; + buffer.file_last = (off_t)UINT32_MAX + (off_t)sizeof(payload); + native_result = 403; native_failed = 1; native_body_limit = 1; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == 403); + CHECK(read_calls == 1 && append_calls == 1 && appended_bytes == 32768U); + CHECK(ctx.intervention_body_limit && process_calls == 0 && forward_calls == 0); + } else if (strcmp(argv[1], "append-boundary") == 0) { + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; + buffer.pos = payload; buffer.last = payload + 10; chain.next = NULL; + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); + CHECK(append_collections == 1 && eos_with_append_active == 0); + CHECK(!ctx.phase4_append_active && ctx.phase4_processed && forward_calls == 1); } else if (strcmp(argv[1], "empty-memory-eos") == 0) { buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; chain.next = NULL; for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; process_calls = 0; forward_calls = 0; CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_OK); @@ -544,7 +696,7 @@ def function_definition(source: str, name: str) -> str: for (i = 0; i <= 2; ++i) { conf.phase4_mode = (unsigned)i; for (invalid_index = 0; invalid_index < sizeof(invalid) / sizeof(invalid[0]); ++invalid_index) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; append_calls = 0; appended_bytes = 0; connection.error = 0; append_result = invalid[invalid_index]; CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); @@ -557,7 +709,7 @@ def function_definition(source: str, name: str) -> str: buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; buffer.pos = payload; buffer.last = payload + 10; for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; process_calls = 0; append_calls = 0; appended_bytes = 0; engine_result = 0; connection.error = 0; @@ -571,17 +723,23 @@ def function_definition(source: str, name: str) -> str: } else if (strcmp(argv[1], "strict-reentry") == 0 || strcmp(argv[1], "sink-failure-reentry") == 0) { request.header_sent = 1; chain.next = NULL; - buffer.in_file = 0; buffer.memory = 1; + buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; buffer.pos = payload; buffer.last = payload + 10; native_result = -1; native_late = 1; ctx.last_intervention_status = 403; + native_only_after_eos = 1; if (strcmp(argv[1], "strict-reentry") == 0) conf.phase4_mode = 2; else log_result = NGX_ERROR; - CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); - CHECK(ctx.phase4_failed && connection.error && event_calls == 1); + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == + (conf.phase4_mode == 2 ? NGX_ERROR : NGX_OK)); + CHECK(ctx.phase4_failed == (conf.phase4_mode == 2)); + CHECK(connection.error == (conf.phase4_mode == 2) && event_calls == 1); native_result = 0; log_result = NGX_OK; - CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); - CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == NGX_ERROR); - CHECK(append_calls == 1 && process_calls == 0 && forward_calls == 0); + CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == + (conf.phase4_mode == 2 ? NGX_ERROR : NGX_OK)); + CHECK(ngx_http_modsecurity_body_filter(&request, NULL) == + (conf.phase4_mode == 2 ? NGX_ERROR : NGX_OK)); + CHECK(append_calls == 1 && process_calls == 1); + CHECK(forward_calls == (conf.phase4_mode == 2 ? 0 : 3)); CHECK(event_calls == 1); } else if (strcmp(argv[1], "core-error-once") == 0) { buffer.in_file = 0; buffer.memory = 1; buffer.last_buf = 1; @@ -599,7 +757,7 @@ def function_definition(source: str, name: str) -> str: request.header_sent = 1; native_result = 403; native_failed = 1; native_body_limit = 1; for (i = 0; i <= 2; ++i) { - memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; + memset(&ctx, 0, sizeof(ctx)); ctx.r = &request; ctx.modsec_transaction = &ctx; ctx.lifetime.pool = &pool; ctx.lifetime.conf = &conf; conf.phase4_mode = (unsigned)i; read_calls = 0; append_calls = 0; appended_bytes = 0; CHECK(ngx_http_modsecurity_body_filter(&request, &chain) == NGX_ERROR); @@ -641,9 +799,15 @@ def function_definition(source: str, name: str) -> str: CHECK(strcmp(last_reason, "response_not_committed") == 0); } } else if (strcmp(argv[1], "log-failure-safe") == 0) { + request.header_sent = 1; native_result = -1; native_late = 1; log_result = NGX_ERROR; + CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == NGX_OK); + CHECK(!ctx.intervention_triggered && !ctx.phase4_failed && !connection.error); + CHECK(event_calls == 1 && diagnostic_calls == 1); + } else if (strcmp(argv[1], "log-failure-strict") == 0) { + conf.phase4_mode = NGX_HTTP_MODSEC_PHASE4_MODE_STRICT; request.header_sent = 1; native_result = -1; native_late = 1; log_result = NGX_ERROR; CHECK(ngx_http_modsecurity_process_response_intervention(&request, &ctx, &conf) == NGX_ERROR); - CHECK(ctx.intervention_triggered && connection.error); + CHECK(ctx.phase4_failed && ctx.phase4_strict_abort && connection.error && event_calls == 1); } else { return 2; } @@ -667,14 +831,21 @@ def unit_program(body: str, module: str) -> str: gate = "ngx_http_modsecurity_phase4_error_gate" if re.search(rf"\b{gate}\s*\(", body): definitions.insert(0, "\nstatic ngx_int_t\n" + function_definition(body, gate) + "\n") - definitions.append("\nstatic void\n" + function_definition( + discard_type = "void" if re.search( + r"static\s+void\s+ngx_http_modsecurity_discard_replaced_response_body\s*\(", body + ) else "ngx_int_t" + definitions.append("\nstatic " + discard_type + "\n" + function_definition( body, "ngx_http_modsecurity_discard_replaced_response_body" )) definitions.append("\nngx_int_t\n" + function_definition( body, "ngx_http_modsecurity_body_filter" )) declarations = "\n".join(definition[:definition.index("{")] + ";" for definition in definitions) - context_lookup = "\nstatic ngx_http_modsecurity_ctx_t *\n" + function_definition( + context_lookup = "".join( + "\nstatic ngx_int_t\n" + function_definition(module, name) + "\n" + for name in ("ngx_http_modsecurity_same_policy", "ngx_http_modsecurity_is_block_error_page") + ) + context_lookup += "\nstatic ngx_http_modsecurity_ctx_t *\n" + function_definition( module, "ngx_http_modsecurity_get_module_ctx" ) + "\n" return PREAMBLE + declarations + context_lookup + "".join(definitions) + CASES @@ -721,11 +892,14 @@ def test(self: Phase4RuntimeTests) -> None: "engine-error", "process-zero", "main-eos", "subrequest-eos", "chain-restored", "forward-again", "terminal-trailing", "replaced-body", + "replaced-data-only", "replaced-buffered-drain", "replaced-terminal-again", + "replaced-subrequest-eos", "replaced-main-chain-marker", "replaced-control-allocation-failure", + "counter-saturation-stream", "large-file-first-chunk", "append-boundary", "empty-memory-eos", "memory-missing-end", "invalid-append-results", "failure-reentry", "strict-reentry", "sink-failure-reentry", "core-error-once", "file-chunk-rejection", "late-safe", "late-strict", "late-off", "native-failure-safe", - "native-failure-uncommitted", "uncommitted-deny", "log-failure-safe", + "native-failure-uncommitted", "uncommitted-deny", "log-failure-safe", "log-failure-strict", ): setattr(Phase4RuntimeTests, "test_" + case.replace("-", "_"), case_test(case)) From f9a3ef033976cb7f5c2de1d4b456e1e172d7b010 Mon Sep 17 00:00:00 2001 From: Easton97-Jens Date: Sun, 15 Nov 2026 00:17:38 +0100 Subject: [PATCH 37/40] Refine parser structure and preserve native test boundaries --- src/ngx_http_modsecurity_access.c | 62 +++++---- src/ngx_http_modsecurity_log_metadata.h | 161 +++++++++++++--------- tests/modsecurity-config-audit-redirect.t | 27 +++- tests/test_intervention_runtime.py | 3 +- tests/test_lifecycle_runtime.py | 1 + 5 files changed, 157 insertions(+), 97 deletions(-) diff --git a/src/ngx_http_modsecurity_access.c b/src/ngx_http_modsecurity_access.c index 0a860a83..d55c4a2b 100644 --- a/src/ngx_http_modsecurity_access.c +++ b/src/ngx_http_modsecurity_access.c @@ -70,6 +70,37 @@ ngx_http_modsecurity_append_request_chunk(ngx_http_request_t *r, } +static ngx_int_t +ngx_http_modsecurity_append_request_body(ngx_http_request_t *r, + ngx_http_modsecurity_ctx_t *ctx) +{ + ngx_chain_t *chain = r->request_body != NULL ? r->request_body->bufs : NULL; + ngx_int_t result; + + while (chain != NULL) { + if (chain->buf == NULL) { + return ngx_http_modsecurity_request_failure(r, ctx); + } + if (!ngx_buf_special(chain->buf)) { + if (chain->buf->pos == NULL || chain->buf->last == NULL || + chain->buf->last < chain->buf->pos) { + return ngx_http_modsecurity_request_failure(r, ctx); + } + result = ngx_http_modsecurity_append_request_chunk(r, ctx, + chain->buf->pos, (size_t) (chain->buf->last - chain->buf->pos)); + if (result != NGX_OK) { + return result; + } + } + if (chain->buf->last_buf) { + break; + } + chain = chain->next; + } + return NGX_OK; +} + + static ngx_int_t ngx_http_modsecurity_inspect_request_body_file(ngx_http_request_t *r, ngx_http_modsecurity_ctx_t *ctx) @@ -499,14 +530,11 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) if (ctx->waiting_more_body == 0) { int ret = 0; - int already_inspected = 0; dd("request body is ready to be processed"); r->write_event_handler = ngx_http_core_run_phases; - ngx_chain_t *chain = r->request_body != NULL ? r->request_body->bufs : NULL; - /** * TODO: Speed up the analysis by sending chunk while they arrive. * @@ -518,34 +546,12 @@ ngx_http_modsecurity_access_handler(ngx_http_request_t *r) if (r->request_body != NULL && r->request_body->temp_file != NULL) { ret = ngx_http_modsecurity_inspect_request_body_file(r, ctx); - if (ret != NGX_OK) { - return ret; - } - already_inspected = 1; } else { dd("inspection request body in memory."); + ret = ngx_http_modsecurity_append_request_body(r, ctx); } - - while (chain && !already_inspected) - { - if (chain->buf == NULL) { - return ngx_http_modsecurity_request_failure(r, ctx); - } - if (!ngx_buf_special(chain->buf)) { - if (chain->buf->pos == NULL || chain->buf->last == NULL || - chain->buf->last < chain->buf->pos) { - return ngx_http_modsecurity_request_failure(r, ctx); - } - ret = ngx_http_modsecurity_append_request_chunk(r, ctx, - chain->buf->pos, (size_t) (chain->buf->last - chain->buf->pos)); - if (ret != NGX_OK) { - return ret; - } - } - if (chain->buf->last_buf) { - break; - } - chain = chain->next; + if (ret != NGX_OK) { + return ret; } /** diff --git a/src/ngx_http_modsecurity_log_metadata.h b/src/ngx_http_modsecurity_log_metadata.h index 5fdec36c..005c7805 100644 --- a/src/ngx_http_modsecurity_log_metadata.h +++ b/src/ngx_http_modsecurity_log_metadata.h @@ -42,13 +42,39 @@ ngx_http_modsecurity_metadata_hex_digit(u_char value) /* Engine toHexIfNeeded() uses literal \xHH sequences in the escaped fields. * Do not decode them while locating delimiters. Raw fields may contain quotes * without a following ']', but the complete ordered record must still match. */ +static ngx_inline size_t +ngx_http_modsecurity_metadata_field_character_size(ngx_str_t input, + size_t position, ngx_int_t escaped, ngx_int_t decimal) +{ + u_char character = input.data[position]; + size_t remaining = input.len - position; + + if (character < 0x20 || character > 0x7e || + (decimal && (character < '0' || character > '9'))) { + return 0; + } + if (!escaped) { + return 1; + } + if (character == '\\') { + if (remaining < 4 || input.data[position + 1] != 'x' || + !ngx_http_modsecurity_metadata_hex_digit(input.data[position + 2]) || + !ngx_http_modsecurity_metadata_hex_digit(input.data[position + 3])) { + return 0; + } + return 4; + } + return character == '"' ? 0 : 1; +} + static ngx_inline ngx_int_t ngx_http_modsecurity_metadata_parse_field( ngx_http_modsecurity_metadata_cursor_t *cursor, const char *name, ngx_int_t escaped, ngx_int_t decimal, ngx_str_t *value) { - size_t start, position, remaining; - u_char character; + size_t start; + size_t position; + size_t consumed; if (!ngx_http_modsecurity_metadata_field_starts(cursor->input, cursor->position, name)) { @@ -57,9 +83,8 @@ ngx_http_modsecurity_metadata_parse_field( start = cursor->position + strlen(name) + 3; position = start; while (position < cursor->input.len) { - remaining = cursor->input.len - position; - character = cursor->input.data[position]; - if (character == '"' && remaining >= 2 && + if (cursor->input.data[position] == '"' && + cursor->input.len - position >= 2 && cursor->input.data[position + 1] == ']') { if (decimal && position == start) { return 0; @@ -71,25 +96,12 @@ ngx_http_modsecurity_metadata_parse_field( cursor->position = position + 2; return 1; } - if (character < 0x20 || character > 0x7e || - (decimal && (character < '0' || character > '9'))) { + consumed = ngx_http_modsecurity_metadata_field_character_size( + cursor->input, position, escaped, decimal); + if (consumed == 0) { return 0; } - if (escaped && character == '\\') { - if (remaining < 4 || cursor->input.data[position + 1] != 'x' || - !ngx_http_modsecurity_metadata_hex_digit( - cursor->input.data[position + 2]) || - !ngx_http_modsecurity_metadata_hex_digit( - cursor->input.data[position + 3])) { - return 0; - } - position += 4; - } else { - if (escaped && character == '"') { - return 0; - } - position++; - } + position += consumed; } return 0; } @@ -163,7 +175,8 @@ ngx_http_modsecurity_parse_rule_metadata(ngx_str_t input, ngx_str_t *rule_id) static const char marker[] = "[file \""; ngx_http_modsecurity_metadata_cursor_t cursor; ngx_str_t candidate; - size_t position, first; + size_t position; + size_t first; if (rule_id == NULL) { return -1; @@ -282,6 +295,56 @@ ngx_http_modsecurity_metadata_utf8_size(const u_char *input, size_t remaining) return 0; } +/* Classify one input byte or complete UTF-8 sequence for both JSON passes. */ +static ngx_inline size_t +ngx_http_modsecurity_metadata_json_character_size(const u_char *input, + size_t remaining, size_t *consumed) +{ + u_char character = input[0]; + size_t sequence_size; + + *consumed = 1; + if (character < 0x20) { + return 6; + } + if (character == '"' || character == '\\') { + return 2; + } + if (character < 0x80) { + return 1; + } + sequence_size = ngx_http_modsecurity_metadata_utf8_size(input, remaining); + if (sequence_size == 0) { + return 6; + } + *consumed = sequence_size; + return sequence_size; +} + +static ngx_inline u_char * +ngx_http_modsecurity_metadata_write_json_character(u_char *output, + const u_char *input, size_t consumed, size_t width) +{ + static const u_char hex[] = "0123456789abcdef"; + u_char character = input[0]; + + if (width == 6) { + *output++ = '\\'; + *output++ = 'u'; + *output++ = '0'; + *output++ = '0'; + *output++ = hex[character >> 4]; + *output++ = hex[character & 0x0f]; + } else if (consumed == 1 && width == 2) { + *output++ = '\\'; + *output++ = character; + } else { + memcpy(output, input, consumed); + output += consumed; + } + return output; +} + /* Preserve valid UTF-8 as-is. JSON controls and invalid UTF-8 bytes are * represented as \u00xx; each invalid byte retains its unsigned byte value. * Success is NUL-terminated with an explicit length. Allocation/size failure @@ -290,9 +353,11 @@ ngx_http_modsecurity_metadata_utf8_size(const u_char *input, size_t remaining) static ngx_inline void ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *dst) { - static const u_char hex[] = "0123456789abcdef"; - size_t position, size, consumed, width; - u_char character, *output; + size_t position; + size_t size; + size_t consumed; + size_t width; + u_char *output; dst->data = NULL; dst->len = 0; @@ -305,24 +370,8 @@ ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *ds } size = 0; for (position = 0; position < src->len; position += consumed) { - character = src->data[position]; - consumed = 1; - if (character < 0x20) { - width = 6; - } else if (character == '"' || character == '\\') { - width = 2; - } else if (character < 0x80) { - width = 1; - } else { - consumed = ngx_http_modsecurity_metadata_utf8_size( - src->data + position, src->len - position); - if (consumed == 0) { - consumed = 1; - width = 6; - } else { - width = consumed; - } - } + width = ngx_http_modsecurity_metadata_json_character_size( + src->data + position, src->len - position, &consumed); if (width > SIZE_MAX - 1 - size) { return; } @@ -334,26 +383,10 @@ ngx_http_modsecurity_json_escape(ngx_pool_t *pool, ngx_str_t *src, ngx_str_t *ds } output = dst->data; for (position = 0; position < src->len; position += consumed) { - character = src->data[position]; - consumed = 1; - if (character == '"' || character == '\\') { - *output++ = '\\'; - *output++ = character; - } else if (character < 0x20 || - (character >= 0x80 && - (consumed = ngx_http_modsecurity_metadata_utf8_size( - src->data + position, src->len - position)) == 0)) { - *output++ = '\\'; - *output++ = 'u'; - *output++ = '0'; - *output++ = '0'; - *output++ = hex[character >> 4]; - *output++ = hex[character & 0x0f]; - consumed = 1; - } else { - memcpy(output, src->data + position, consumed); - output += consumed; - } + width = ngx_http_modsecurity_metadata_json_character_size( + src->data + position, src->len - position, &consumed); + output = ngx_http_modsecurity_metadata_write_json_character(output, + src->data + position, consumed, width); } *output = '\0'; dst->len = size; diff --git a/tests/modsecurity-config-audit-redirect.t b/tests/modsecurity-config-audit-redirect.t index f3255d7f..d2cf89f6 100644 --- a/tests/modsecurity-config-audit-redirect.t +++ b/tests/modsecurity-config-audit-redirect.t @@ -27,23 +27,40 @@ http { modsecurity on; modsecurity_rules ' SecRuleEngine On - SecAuditEngine On - SecAuditLogType Serial - SecAuditLogParts ABZ - SecAuditLog %%TESTDIR%%/audit-source.log '; location /same/ { + modsecurity_rules ' + SecAuditEngine On + SecAuditLogType Serial + SecAuditLogParts ABZ + SecAuditLog %%TESTDIR%%/audit-source.log + '; try_files $uri /same/index.html; } location = /changed-start { + modsecurity_rules ' + SecAuditEngine On + SecAuditLogType Serial + SecAuditLogParts ABZ + SecAuditLog %%TESTDIR%%/audit-source.log + '; try_files /nonexistent-audit-source /changed-final; } location = /changed-final { modsecurity_rules ' + SecAuditEngine On + SecAuditLogType Serial + SecAuditLogParts ABZ SecAuditLog %%TESTDIR%%/audit-target.log '; } location = /disabled-start { + modsecurity_rules ' + SecAuditEngine On + SecAuditLogType Serial + SecAuditLogParts ABZ + SecAuditLog %%TESTDIR%%/audit-source.log + '; try_files /nonexistent-audit-source /disabled-final; } location = /disabled-final { @@ -69,6 +86,8 @@ my $disabled = http_get('/disabled-start'); like($disabled, qr/200 OK/, 'disabled target succeeds'); like($disabled, qr/disabled-policy-response/, 'disabled response preserved'); +# Each location owns its audit settings: the engine's rules merge otherwise +# lets an explicitly configured parent audit path override a child path. # Stop first so the native serial audit epilogues are fully written. $t->stop(); my $source = $t->read_file('audit-source.log'); diff --git a/tests/test_intervention_runtime.py b/tests/test_intervention_runtime.py index ced15e32..b317c7bc 100644 --- a/tests/test_intervention_runtime.py +++ b/tests/test_intervention_runtime.py @@ -570,7 +570,8 @@ def unit_program(module: str) -> str: helpers += "\nint\n" + function_definition( module, "ngx_http_modsecurity_process_intervention" ) - metadata = (ROOT / "src" / "ngx_http_modsecurity_log_metadata.h").read_text(encoding="utf-8") + metadata_path = (ROOT / "src" / "ngx_http_modsecurity_log_metadata.h").as_posix() + metadata = f'\n#include "{metadata_path}"\n' return PREAMBLE + metadata + helpers + CASES diff --git a/tests/test_lifecycle_runtime.py b/tests/test_lifecycle_runtime.py index 51744a1a..74c4d474 100644 --- a/tests/test_lifecycle_runtime.py +++ b/tests/test_lifecycle_runtime.py @@ -356,6 +356,7 @@ def unit_program(access: str, logging: str) -> str: ("ngx_http_modsecurity_request_failure", "static ngx_int_t"), ("ngx_http_modsecurity_request_intervention", "static int"), ("ngx_http_modsecurity_append_request_chunk", "static ngx_int_t"), + ("ngx_http_modsecurity_append_request_body", "static ngx_int_t"), ("ngx_http_modsecurity_inspect_request_body_file", "static ngx_int_t"), ("ngx_http_modsecurity_request_read", "void"), ("ngx_http_modsecurity_access_handler", "ngx_int_t"), From 688cfbd6cee32bcc331456d9e94934261f2469ed Mon Sep 17 00:00:00 2001 From: Easton97-Jens Date: Sun, 15 Nov 2026 18:18:51 +0100 Subject: [PATCH 38/40] Fix queued HTTP/2 resets with versioned NGINX host patch --- .gitattributes | 2 + .github/scripts/apply-nginx-http2-patch.sh | 48 ++++ .github/workflows/test.yml | 4 +- .github/workflows/test_new.yml | 23 +- .oss-scanner/Dockerfile | 5 +- CHANGES | 11 + README.md | 11 +- docs/phase4-handling.en.md | 21 ++ patches/README.md | 33 +++ patches/nginx-http2-close-stream-flush.patch | 15 ++ tests/README.md | 13 + tests/modsecurity-phase4-h2-reset.t | 236 +++++++++++++++++++ tests/test_nginx_patch_runtime.py | 165 +++++++++++++ 13 files changed, 582 insertions(+), 5 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/scripts/apply-nginx-http2-patch.sh create mode 100644 patches/README.md create mode 100644 patches/nginx-http2-close-stream-flush.patch create mode 100644 tests/modsecurity-phase4-h2-reset.t create mode 100644 tests/test_nginx_patch_runtime.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..c3ef7e78 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +/.github/scripts/apply-nginx-http2-patch.sh text eol=lf +/patches/*.patch text eol=lf diff --git a/.github/scripts/apply-nginx-http2-patch.sh b/.github/scripts/apply-nginx-http2-patch.sh new file mode 100644 index 00000000..64dde04e --- /dev/null +++ b/.github/scripts/apply-nginx-http2-patch.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +# Apply the versioned host correction before configuring/building NGINX. +# The connector itself does not own NGINX's physical HTTP/2 connection events. +set -eu + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 /path/to/nginx/source" >&2 + exit 2 +fi + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +patch_file="$script_dir/../../patches/nginx-http2-close-stream-flush.patch" +nginx_dir=$(CDPATH= cd -- "$1" && pwd) + +if [ ! -f "$nginx_dir/src/http/v2/ngx_http_v2.c" ]; then + echo "NGINX HTTP/2 source is missing in $nginx_dir" >&2 + exit 1 +fi + +if ! command -v patch >/dev/null 2>&1; then + echo "The patch utility is required to apply the NGINX HTTP/2 fix" >&2 + exit 1 +fi + +# Match the source checkout's line endings without rewriting NGINX before the +# dry run. Windows checkouts may use CRLF even when this patch is stored as LF. +patch_input=$(mktemp "${TMPDIR:-/tmp}/modsecurity-nginx-h2-patch.XXXXXX") +trap 'rm -f "$patch_input"' 0 HUP INT TERM +cr=$(printf '\r') +if LC_ALL=C grep -qU "${cr}$" "$nginx_dir/src/http/v2/ngx_http_v2.c"; then + sed "s/${cr}\{0,1\}$/${cr}/" "$patch_file" > "$patch_input" +else + sed "s/${cr}$//" "$patch_file" > "$patch_input" +fi + +if patch --binary --dry-run --batch --forward --fuzz=0 -p1 -d "$nginx_dir" \ + -i "$patch_input" >/dev/null 2>&1; then + patch --binary --batch --forward --fuzz=0 --no-backup-if-mismatch -p1 \ + -d "$nginx_dir" -i "$patch_input" + echo "Applied NGINX HTTP/2 close-stream reset flush fix" +elif patch --binary --dry-run --batch --reverse --fuzz=0 -p1 -d "$nginx_dir" \ + -i "$patch_input" >/dev/null 2>&1; then + echo "NGINX HTTP/2 close-stream reset flush fix is already applied" +else + echo "NGINX HTTP/2 source does not match the versioned fix; review it before building" >&2 + exit 1 +fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7e3ac8e..40a51f04 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt-get update -y -qq - sudo apt-get install -y make autoconf automake make libyajl-dev libxml2-dev libmaxminddb-dev libcurl4-gnutls-dev $COMPDEPS + sudo apt-get install -y patch make autoconf automake make libyajl-dev libxml2-dev libmaxminddb-dev libcurl4-gnutls-dev $COMPDEPS - name: Get libModSecurity source uses: actions/checkout@v4 with: @@ -50,6 +50,8 @@ jobs: repository: nginx/nginx path: nginx fetch-depth: 1 + - name: Apply NGINX HTTP/2 reset delivery fix + run: sh ModSecurity-nginx/.github/scripts/apply-nginx-http2-patch.sh nginx - name: Get Nginx tests uses: actions/checkout@v4 with: diff --git a/.github/workflows/test_new.yml b/.github/workflows/test_new.yml index 70959d03..69528502 100644 --- a/.github/workflows/test_new.yml +++ b/.github/workflows/test_new.yml @@ -46,7 +46,7 @@ jobs: - name: Setup Dependencies run: | sudo apt-get update -y -qq - sudo apt-get install -y make autoconf automake pcre2-utils libpcre2-dev libyajl-dev libxml2-dev libmaxminddb-dev libcurl4-gnutls-dev $COMPDEPS + sudo apt-get install -y patch make autoconf automake pcre2-utils libpcre2-dev libyajl-dev libxml2-dev libmaxminddb-dev libcurl4-gnutls-dev $COMPDEPS - name: Get libModSecurity source uses: actions/checkout@v6 with: @@ -81,6 +81,10 @@ jobs: path: nginx fetch-depth: 0 + - name: Apply NGINX HTTP/2 reset delivery fix + if: ${{ matrix.os != 'ubuntu-24.04' || matrix.compiler != 'gcc' || matrix.nginx_version != '1.31.x' }} + run: sh ModSecurity-nginx/.github/scripts/apply-nginx-http2-patch.sh nginx + - name: Show Nginx source version working-directory: nginx shell: bash @@ -116,6 +120,19 @@ jobs: make make modules sudo make install + - name: Verify queued reset on unpatched NGINX + if: ${{ matrix.os == 'ubuntu-24.04' && matrix.compiler == 'gcc' && matrix.nginx_version == '1.31.x' }} + working-directory: nginx/test + run: | + TEST_NGINX_BINARY=../objs/nginx TEST_NGINX_H2_RESET_EXPECT_QUEUED=1 prove -v modsecurity-phase4-h2-reset.t + - name: Apply HTTP/2 fix and rebuild after negative control + if: ${{ matrix.os == 'ubuntu-24.04' && matrix.compiler == 'gcc' && matrix.nginx_version == '1.31.x' }} + working-directory: nginx + run: | + sh ../ModSecurity-nginx/.github/scripts/apply-nginx-http2-patch.sh . + make + make modules + sudo make install - name: Run ModSecurity-nginx tests working-directory: nginx/test run: | @@ -310,6 +327,7 @@ jobs: with: msystem: UCRT64 path-type: inherit + install: patch - name: Get ModSecurity-nginx workflow scripts uses: actions/checkout@v7 with: @@ -330,6 +348,9 @@ jobs: path: nginx fetch-depth: 0 + - name: Apply NGINX HTTP/2 reset delivery fix + run: sh ModSecurity-nginx/.github/scripts/apply-nginx-http2-patch.sh nginx + - name: Show Nginx source version working-directory: nginx shell: bash diff --git a/.oss-scanner/Dockerfile b/.oss-scanner/Dockerfile index 4e20d3d6..8621c0a6 100644 --- a/.oss-scanner/Dockerfile +++ b/.oss-scanner/Dockerfile @@ -5,7 +5,7 @@ FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates git build-essential clang autoconf automake libtool pkg-config \ + ca-certificates git patch build-essential clang autoconf automake libtool pkg-config \ perl curl gdb valgrind \ libpcre2-dev pcre2-utils zlib1g-dev libyajl-dev libxml2-dev libmaxminddb-dev \ libcurl4-gnutls-dev \ @@ -39,7 +39,8 @@ RUN tag=$(git ls-remote --tags --refs https://github.com/coreruleset/coreruleset && git clone --depth 1 --branch "$tag" https://github.com/coreruleset/coreruleset /opt/coreruleset # nginx with this connector built in, as CI builds it. A failing step fails the build. -RUN cd /opt/nginx \ +RUN sh /src/.github/scripts/apply-nginx-http2-patch.sh /opt/nginx \ + && cd /opt/nginx \ && ./auto/configure --with-debug --with-http_v2_module --with-http_auth_request_module \ --with-cc-opt="$DEBUG_FLAGS" --add-module=/src \ && make -j"$(nproc)" diff --git a/CHANGES b/CHANGES index 96b29de9..afa9ca98 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,17 @@ Unreleased ---------- + - fix: bundle the NGINX HTTP/2 close-stream correction so a queued + RST_STREAM is flushed while sibling streams remain active. Apply it to + native CI hosts before building NGINX and document the host rebuild + requirement for static and dynamic connector installations. + [PR #377 - @Easton97-Jens] + - test: add a passive HTTP/2 multiplex regression for prompt Strict resets, + independent allowed-stream completion and same-connection reuse. One + Linux job first verifies the queued-reset behavior on an unpatched host, + then applies the core patch and runs the normal suite. + [PR #377 - @Easton97-Jens] + - feat: add off (default), safe and strict Phase 4 intervention modes. Safe and strict explicitly enable the additional connector policy; off leaves ModSecurity response inspection and native interventions active. diff --git a/README.md b/README.md index f10df0c9..337f9853 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,15 @@ Apache. As a result, this current version has less dependencies, fewer bugs, and Before compile this software make sure that you have libmodsecurity installed. You can download it from the [ModSecurity git repository](https://github.com/owasp-modsecurity/ModSecurity). For information pertaining to the compilation and installation of libmodsecurity please consult the documentation provided along with it. +For HTTP/2 builds that use Phase 4 enforcement, apply the bundled +[NGINX reset-delivery correction](patches/README.md) to the NGINX sources before +configuration. This is a host-core change; rebuild the NGINX binary when using +a dynamic connector module as well: + +```sh +sh /path/to/ModSecurity-nginx/.github/scripts/apply-nginx-http2-patch.sh /path/to/nginx +``` + With libmodsecurity installed, you can proceed with the installation of the ModSecurity-nginx connector, which follows the nginx third-party module installation procedure. From the nginx source directory: ``` @@ -212,7 +221,7 @@ modsecurity_phase4_mode Controls the connector's additional phase 4 intervention policy. `safe` logs a late rule intervention as `log_only` and continues even if optional JSON -logging fails (with an nginx diagnostic); `strict` aborts the connection once headers +logging fails (with an nginx diagnostic); `strict` terminates the affected response once headers are sent. Before headers are sent, both modes can apply the intervention's HTTP status. Neither mode guarantees a clean 403 after headers have been sent. diff --git a/docs/phase4-handling.en.md b/docs/phase4-handling.en.md index 6faea1c7..ea6f77b0 100644 --- a/docs/phase4-handling.en.md +++ b/docs/phase4-handling.en.md @@ -168,6 +168,27 @@ same separate-load sequence. Engine MIME selection and inspection limits apply in every phase 4 mode; there is no additional connector inspection budget. +### HTTP/2 host-core correction + +A Strict intervention on an HTTP/2 response terminates that stream. The +bundled [NGINX core patch](../patches/README.md) ensures that its queued reset +is written promptly when an independent stream on the connection remains +active. The physical connection and allowed sibling streams remain usable. + +Apply the patch before configuring and rebuilding NGINX. This includes the +NGINX host binary when using a dynamic module; replacing only the connector +module cannot change an already installed host's core behavior. The helper +accepts an already-applied patch and rejects source drift without fuzzy +matching. Native CI applies the host correction to its Linux/Windows builds. + +The focused regression releases an auth-gated static Strict response while +an allowed proxied sibling is held open, then waits passively for the reset. +It sends no new +client input that could hide a stalled write. After the reset, the sibling +and a new stream must complete normally. A selected Linux job runs the +controlled queued-reset expectation against an unpatched host before +applying the patch and rebuilding it. + ## 6) Logging format and security boundary `modsecurity_phase4_log` preserves the existing JSON-lines intervention schema: diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 00000000..b571e241 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,33 @@ +# NGINX host correction for HTTP/2 reset delivery + +`nginx-http2-close-stream-flush.patch` schedules the physical HTTP/2 write +event when a stream closes with queued output while another stream remains +active. This lets a queued reset reach the client without waiting for another +socket read or the sibling stream to finish. The sibling connection remains +usable. + +The patch is copied from +[ModSecurity-test-Framework at 95f228936efe17fda24f21323ad87cd698940b48](https://github.com/Easton97-Jens/ModSecurity-test-Framework/blob/95f228936efe17fda24f21323ad87cd698940b48/ci/provisioning/patches/nginx/h2-close-stream-flush.patch). +It changes NGINX core code, not libModSecurity or the connector's rule policy. + +Before configuring and building NGINX, run from the connector checkout: + +```sh +sh .github/scripts/apply-nginx-http2-patch.sh /path/to/nginx/source +``` + +Then configure NGINX with `--with-http_v2_module` and this connector as usual. +For a dynamic connector, rebuild and deploy the patched NGINX host as well. +Replacing only the module leaves an existing host's HTTP/2 core unchanged. + +The helper requires the standard `patch` utility, rejects mismatched source +without fuzzy matching, and accepts an already-applied copy. Review this +dependency correction when changing NGINX versions or when an equivalent fix +lands upstream. NGINX binaries supplied by a distribution need the equivalent +host correction from their distributor or a locally rebuilt NGINX binary. + +CI applies the patch to its NGINX sources before native Linux/Windows builds. +A focused native regression checks that a Strict phase-4 reset arrives while +a separate allowed stream is still open, then verifies that the allowed +stream and another request complete on the same HTTP/2 connection. One Linux +job also runs the controlled unpatched baseline before applying the fix. diff --git a/patches/nginx-http2-close-stream-flush.patch b/patches/nginx-http2-close-stream-flush.patch new file mode 100644 index 00000000..d8530213 --- /dev/null +++ b/patches/nginx-http2-close-stream-flush.patch @@ -0,0 +1,15 @@ +--- a/src/http/v2/ngx_http_v2.c ++++ b/src/http/v2/ngx_http_v2.c +@@ -4803,3 +4803,12 @@ + if (h2c->processing || h2c->blocked) { ++ /* ++ * A reset queued while finalizing this stream must not wait for ++ * another stream to finish or for a new socket read event. Let the ++ * existing write handler flush it after this request is destroyed. ++ */ ++ if (h2c->last_out && !h2c->blocked) { ++ ngx_post_event(h2c->connection->write, &ngx_posted_events); ++ } ++ + return; + } diff --git a/tests/README.md b/tests/README.md index 84e6af44..737e40cd 100644 --- a/tests/README.md +++ b/tests/README.md @@ -71,3 +71,16 @@ Perl suites run in the Linux and Windows build jobs. or MIME selection. The main response must still run its Phase3 header rule and Phase4 body rule. Direct subrequest EOS ownership is exercised separately by the compiled helper suite; the auth fixture does not claim to emit a body EOS. + +## HTTP/2 reset delivery regression + +`modsecurity-phase4-h2-reset.t` uses Test::Nginx::HTTP2 and controlled proxy +backends. It requires a prompt Strict reset while an allowed sibling remains +active, then checks sibling completion and connection reuse. Normal CI runs +it with the bundled NGINX host-core correction applied. + +Only the controlled unpatched negative run sets +`TEST_NGINX_H2_RESET_EXPECT_QUEUED=1`. That run requires the reset to remain +queued during the passive wait and to arrive after the held sibling is +released. The selected Linux job subsequently applies the patch, rebuilds +NGINX, and runs the normal tests without that variable. diff --git a/tests/modsecurity-phase4-h2-reset.t b/tests/modsecurity-phase4-h2-reset.t new file mode 100644 index 00000000..815ed31e --- /dev/null +++ b/tests/modsecurity-phase4-h2-reset.t @@ -0,0 +1,236 @@ +#!/usr/bin/perl + +# Exercise Strict phase-4 stream finalization while another HTTP/2 stream is +# active. The auth backend resumes static content outside the HTTP/2 read +# handler; its small, unbuffered final body reaches the filter as last_buf. + +use warnings; +use strict; + +use Test::More; +use IO::Socket::INET; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; +use Test::Nginx::HTTP2; + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $owner = $$; +my $t = Test::Nginx->new()->has(qw/http http_v2 proxy auth_request/)->plan(21); +my $s; +my $prefix = 'allowed-prefix-'; +my $tail = 'allowed-tail'; +my $forbidden = 'H2-STRICT-FORBIDDEN-TAIL'; +my $expect_queued = $ENV{TEST_NGINX_H2_RESET_EXPECT_QUEUED} || 0; + +# Release blocked backends on failed assertions or an early bailout as well. +# Daemon children inherit END blocks and must not release the parent's gates. +END { + if ($$ == $owner && defined $t) { + eval { + $t->write_file('hold-release', "\n"); + $t->write_file('strict-release', "\n"); + }; + undef $s; + } +} + +$t->write_file_expand('nginx.conf', <<'EOF'); +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + default_type text/plain; + sendfile off; + postpone_output 0; + + server { + listen 127.0.0.1:8080 http2; + server_name localhost; + root %%TESTDIR%%; + + modsecurity on; + modsecurity_phase4_mode strict; + modsecurity_phase4_log %%TESTDIR%%/strict-h2.log; + modsecurity_rules ' + SecRuleEngine On + SecResponseBodyAccess On + SecResponseBodyMimeType text/plain + SecRule RESPONSE_BODY "@contains H2-STRICT-FORBIDDEN-TAIL" "id:942377,phase:4,deny,status:403,log" + '; + + location = /hold { + proxy_pass http://127.0.0.1:8081; + proxy_buffering off; + proxy_read_timeout 30s; + } + + location = /strict { + auth_request /authorize-strict; + } + + location = /authorize-strict { + internal; + modsecurity off; + proxy_pass http://127.0.0.1:8082; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_read_timeout 30s; + } + + location = /allow { + } + } +} +EOF + +$t->write_file('strict', $forbidden); +$t->write_file('allow', 'allowed-after-reset'); +$t->run_daemon(\&backend, 'hold', port(8081), $t->testdir()); +$t->run_daemon(\&backend, 'strict', port(8082), $t->testdir()); +$t->run(); +$t->waitforsocket('127.0.0.1:' . port(8081)) or BAIL_OUT('hold backend unavailable'); +$t->waitforsocket('127.0.0.1:' . port(8082)) or BAIL_OUT('auth backend unavailable'); + +$s = Test::Nginx::HTTP2->new(); +my $held = $s->new_stream({ path => '/hold' }); +my $frames = $s->read(all => [{ sid => $held, length => length($prefix) }]); +is(stream_status($frames, $held), 200, 'held stream has response headers'); +is(stream_body($frames, $held), $prefix, 'held stream starts delivering allowed data'); +ok(!stream_finished($frames, $held), 'held stream remains active'); + +my $strict = $s->new_stream({ path => '/strict' }); +my $ready = $t->waitforfile($t->testdir() . '/strict-ready'); +ok($ready, 'strict request reached asynchronous auth gate'); +BAIL_OUT('strict auth request did not reach its gate') unless $ready; + +# Settle all earlier input, including the constructor's SETTINGS ACK, while +# auth is still blocked. Reading frames never generates acknowledgements. +$s->h2_ping('gatewait'); +my $barrier = $s->read(all => [{ type => 'PING' }], wait => 3); +push @$frames, @$barrier; +my $settled = grep { $_->{type} eq 'PING' && ($_->{flags} & 1) + && $_->{value} eq 'gatewait' } @$barrier; +ok($settled, 'PING acknowledgement settles client input before auth release'); +BAIL_OUT('HTTP/2 input did not settle before auth release') unless $settled; +$t->write_file('strict-release', "\n"); + +# No client frame is sent between releasing auth and observing RST_STREAM. +# A real phase-4 event is required: a broken backend is never baseline proof. +my $decision = wait_for_decision(); +ok($decision, 'native Strict policy reached connection_abort'); +BAIL_OUT('native Strict phase-4 decision was not recorded') unless $decision; +my $prompt = $s->read(all => [{ type => 'RST_STREAM' }], wait => 1); +push @$frames, @$prompt; +is(!!stream_reset($prompt, $strict), !$expect_queued, + $expect_queued ? 'unpatched core queues reset while another stream is active' + : 'Strict reset arrives promptly while another stream is active'); +ok(!stream_finished($frames, $held) && !stream_reset($frames, $held), + 'held stream is still active during reset observation'); + +$t->write_file('hold-release', "\n"); +my @remaining = ({ sid => $held, fin => 1 }); +push @remaining, { type => 'RST_STREAM' } unless stream_reset($frames, $strict); +my $released = $s->read(all => \@remaining, wait => 5); +push @$frames, @$released; +ok(stream_reset($frames, $strict), 'strict stream eventually receives a definite reset'); +is(stream_status($frames, $strict), 200, 'strict response started before its late denial'); +unlike(stream_body($frames, $strict), qr/\Q$forbidden\E/, + 'Strict final body is withheld'); +ok(!stream_finished($frames, $strict), 'strict stream has no successful end of response'); +is(stream_body($frames, $held), $prefix . $tail, 'unrelated stream delivers its complete body'); +ok(stream_finished($frames, $held), 'unrelated stream ends normally'); +ok(!stream_reset($frames, $held), 'unrelated stream is not reset'); +ok(!grep({ $_->{type} eq 'GOAWAY' } @$frames), 'multiplexed connection is not closed'); + +my $allowed = $s->new_stream({ path => '/allow' }); +my $after = $s->read(all => [{ sid => $allowed, fin => 1 }], wait => 5); +is(stream_status($after, $allowed), 200, 'same connection accepts another request'); +is(stream_body($after, $allowed), 'allowed-after-reset', 'later allowed response is complete'); +ok(stream_finished($after, $allowed), 'later allowed response ends normally'); +ok(!grep({ $_->{type} eq 'GOAWAY' } @$after), 'reused connection has no GOAWAY'); +ok(!stream_reset($after, $allowed), 'later allowed stream is not reset'); + +undef $s; + +sub stream_body { + my ($list, $sid) = @_; + return join '', map { $_->{data} } + grep { $_->{type} eq 'DATA' && $_->{sid} == $sid } @$list; +} + +sub stream_status { + my ($list, $sid) = @_; + my ($header) = grep { $_->{type} eq 'HEADERS' && $_->{sid} == $sid } @$list; + return $header ? $header->{headers}->{':status'} : undef; +} + +sub stream_finished { + my ($list, $sid) = @_; + return scalar grep { $_->{sid} == $sid && ($_->{flags} & 1) + && ($_->{type} eq 'HEADERS' || $_->{type} eq 'DATA') } @$list; +} + +sub stream_reset { + my ($list, $sid) = @_; + return scalar grep { $_->{type} eq 'RST_STREAM' && $_->{sid} == $sid } @$list; +} + +sub wait_for_decision { + for (1 .. 100) { + my $log = $t->read_file('strict-h2.log'); + return 1 if defined $log && $log =~ /"uri":"\/strict"[^\n]*"actual_action":"connection_abort"/; + select undef, undef, undef, 0.05; + } + return 0; +} + +sub backend { + my ($kind, $listen, $dir) = @_; + my $server = IO::Socket::INET->new(Proto => 'tcp', + LocalHost => '127.0.0.1', LocalPort => $listen, + Listen => 5, Reuse => 1) or die "listen: $!"; + local $SIG{PIPE} = 'IGNORE'; + while (my $client = $server->accept()) { + $client->autoflush(1); + binmode $client; + my $headers = ''; + while (my $line = <$client>) { + $headers .= $line; + last if $line =~ /^\r?\n$/; + } + unless (length $headers) { + close $client; + next; + } + open my $ready, '>', "$dir/$kind-ready" or die "ready: $!"; + close $ready; + if ($kind eq 'hold') { + print $client "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n" + . 'Content-Length: ' . length($prefix . $tail) + . "\r\nConnection: close\r\n\r\n$prefix"; + } + for (1 .. 1500) { + last if -e "$dir/$kind-release"; + select undef, undef, undef, 0.02; + } + die "gate timed out: $kind" unless -e "$dir/$kind-release"; + if ($kind eq 'hold') { + print $client $tail; + } else { + print $client "HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n" + . "Connection: close\r\n\r\n"; + } + close $client; + } +} diff --git a/tests/test_nginx_patch_runtime.py b/tests/test_nginx_patch_runtime.py new file mode 100644 index 00000000..15bba629 --- /dev/null +++ b/tests/test_nginx_patch_runtime.py @@ -0,0 +1,165 @@ +"""Exercise the real NGINX patch applier at its filesystem/process boundary. + +The minimal official close-stream guard is an application fixture, not a +native HTTP/2 runtime. The multiplexed .t regression proves reset delivery. +""" + +from __future__ import annotations + +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +HELPER = ROOT / ".github/scripts/apply-nginx-http2-patch.sh" +PATCH = ROOT / "patches/nginx-http2-close-stream-flush.patch" + +PREFIX = ( + "/* Minimal official ngx_http_v2_close_stream application context. */\n" + "void\nngx_http_v2_close_stream(void)\n{\n" + " h2c->processing--;\n\n" +) +OLD_GUARD = ( + " if (h2c->processing || h2c->blocked) {\n" + " return;\n" + " }\n" +) +NEW_GUARD = ( + " if (h2c->processing || h2c->blocked) {\n" + " /*\n" + " * A reset queued while finalizing this stream must not wait for\n" + " * another stream to finish or for a new socket read event. Let the\n" + " * existing write handler flush it after this request is destroyed.\n" + " */\n" + " if (h2c->last_out && !h2c->blocked) {\n" + " ngx_post_event(h2c->connection->write, &ngx_posted_events);\n" + " }\n\n" + " return;\n" + " }\n" +) +SUFFIX = "\n ev = h2c->connection->read;\n}\n" + + +def with_eol(text: str, ending: bytes) -> bytes: + return text.encode("ascii").replace(b"\n", ending) + + +class NginxPatchRuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.shell = shutil.which("sh") + patch = shutil.which("patch") + if cls.shell is None or patch is None: + raise RuntimeError("POSIX sh and GNU patch are required") + version = subprocess.run( + [patch, "--version"], capture_output=True, text=True, + timeout=10, check=False, + ) + if version.returncode or "GNU patch" not in version.stdout: + raise RuntimeError("the patch-applier regressions require GNU patch") + cls.helper_bytes = HELPER.read_bytes() + cls.patch_bytes = PATCH.read_bytes().replace(b"\r\n", b"\n") + + def fixture(self, source_eol: bytes, patch_eol: bytes, *, spaces=False): + temporary = tempfile.TemporaryDirectory(prefix="nginx patch fixture ") + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + repository = root / ("connector checkout" if spaces else "connector") + helper = repository / ".github/scripts/apply-nginx-http2-patch.sh" + patch = repository / "patches/nginx-http2-close-stream-flush.patch" + source_root = root / ("nginx source checkout" if spaces else "nginx") + source = source_root / "src/http/v2/ngx_http_v2.c" + helper.parent.mkdir(parents=True) + patch.parent.mkdir(parents=True) + source.parent.mkdir(parents=True) + helper.write_bytes(self.helper_bytes) + patch.write_bytes(self.patch_bytes.replace(b"\n", patch_eol)) + source.write_bytes(with_eol(PREFIX + OLD_GUARD + SUFFIX, source_eol)) + return root, helper, patch, source_root, source + + def invoke(self, helper: Path, *arguments: Path): + return subprocess.run( + [self.shell, str(helper), *(str(argument) for argument in arguments)], + capture_output=True, text=True, timeout=15, check=False, + ) + + def assert_no_patch_debris(self, root: Path): + leftovers = [str(path.relative_to(root)) for path in root.rglob("*") + if path.is_file() and path.suffix in (".rej", ".orig")] + self.assertEqual(leftovers, [], "failed/dry-run application leaves no rejects/backups") + + def check_application(self, source_eol: bytes, patch_eol: bytes, *, spaces=False): + root, helper, patch, source_root, source = self.fixture( + source_eol, patch_eol, spaces=spaces, + ) + original_patch = patch.read_bytes() + result = self.invoke(helper, source_root) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + expected = with_eol(PREFIX + NEW_GUARD + SUFFIX, source_eol) + self.assertEqual(source.read_bytes(), expected, + "only the requested hunk changes and the source EOL stays intact") + self.assertEqual(patch.read_bytes(), original_patch, "input patch is never rewritten") + again = self.invoke(helper, source_root) + self.assertEqual(again.returncode, 0, again.stdout + again.stderr) + self.assertEqual(source.read_bytes(), expected, "second application is idempotent") + self.assertEqual(patch.read_bytes(), original_patch) + self.assert_no_patch_debris(root) + + def test_lf_source_lf_patch(self): + self.check_application(b"\n", b"\n") + + def test_crlf_source_lf_patch(self): + self.check_application(b"\r\n", b"\n") + + def test_lf_source_crlf_patch(self): + self.check_application(b"\n", b"\r\n") + + def test_crlf_source_crlf_patch(self): + self.check_application(b"\r\n", b"\r\n") + + def test_unmatched_hunk_preserves_source_without_rejects_or_backups(self): + for ending in (b"\n", b"\r\n"): + with self.subTest(source_eol=repr(ending)): + root, helper, patch, source_root, source = self.fixture(ending, b"\n") + invalid = with_eol( + PREFIX + OLD_GUARD.replace("h2c->processing || h2c->blocked", "h2c->processing") + + SUFFIX, ending, + ) + source.write_bytes(invalid) + original_patch = patch.read_bytes() + result = self.invoke(helper, source_root) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(source.read_bytes(), invalid) + self.assertEqual(patch.read_bytes(), original_patch) + self.assert_no_patch_debris(root) + + def test_missing_http2_source_fails_without_creating_it(self): + root, helper, patch, source_root, source = self.fixture(b"\n", b"\n") + source.unlink() + result = self.invoke(helper, source_root) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(source.exists()) + self.assertTrue(patch.is_file()) + self.assert_no_patch_debris(root) + + def test_usage_rejects_missing_or_extra_arguments(self): + root, helper, patch, source_root, source = self.fixture(b"\n", b"\n") + original = source.read_bytes() + for arguments in ((), (source_root, source_root)): + with self.subTest(argument_count=len(arguments)): + result = self.invoke(helper, *arguments) + self.assertEqual(result.returncode, 2) + self.assertIn("Usage:", result.stderr) + self.assertEqual(source.read_bytes(), original) + self.assertTrue(patch.is_file()) + self.assert_no_patch_debris(root) + + def test_source_and_repository_paths_with_spaces(self): + self.check_application(b"\r\n", b"\r\n", spaces=True) + + +if __name__ == "__main__": + unittest.main() From be318c4f0453c2403537e9339cf34aae8c5888b9 Mon Sep 17 00:00:00 2001 From: Easton97-Jens Date: Sun, 15 Nov 2026 18:25:32 +0100 Subject: [PATCH 39/40] Finalize HTTP/2 regression harness before TAP completion --- patches/README.md | 2 +- tests/modsecurity-phase4-h2-reset.t | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/patches/README.md b/patches/README.md index b571e241..8a2c25c5 100644 --- a/patches/README.md +++ b/patches/README.md @@ -20,7 +20,7 @@ Then configure NGINX with `--with-http_v2_module` and this connector as usual. For a dynamic connector, rebuild and deploy the patched NGINX host as well. Replacing only the module leaves an existing host's HTTP/2 core unchanged. -The helper requires the standard `patch` utility, rejects mismatched source +The helper requires GNU `patch`, rejects mismatched source without fuzzy matching, and accepts an already-applied copy. Review this dependency correction when changing NGINX versions or when an equivalent fix lands upstream. NGINX binaries supplied by a distribution need the equivalent diff --git a/tests/modsecurity-phase4-h2-reset.t b/tests/modsecurity-phase4-h2-reset.t index 815ed31e..c8bcceef 100644 --- a/tests/modsecurity-phase4-h2-reset.t +++ b/tests/modsecurity-phase4-h2-reset.t @@ -36,6 +36,7 @@ END { $t->write_file('strict-release', "\n"); }; undef $s; + undef $t; } } @@ -162,6 +163,8 @@ ok(!grep({ $_->{type} eq 'GOAWAY' } @$after), 'reused connection has no GOAWAY') ok(!stream_reset($after, $allowed), 'later allowed stream is not reset'); undef $s; +# Run Test::Nginx's alert/sanitizer assertions before Test::More finalizes TAP. +undef $t; sub stream_body { my ($list, $sid) = @_; From 7a5a336c58a2a6a42e8e7e34787f8da855088132 Mon Sep 17 00:00:00 2001 From: Easton97-Jens Date: Sun, 15 Nov 2026 18:29:53 +0100 Subject: [PATCH 40/40] Use official zlib release asset for Windows native builds --- .github/workflows/test_new.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_new.yml b/.github/workflows/test_new.yml index 69528502..a2fe800f 100644 --- a/.github/workflows/test_new.yml +++ b/.github/workflows/test_new.yml @@ -393,7 +393,7 @@ jobs: # without the dot progress meter. for url in \ https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.47/pcre2-10.47.tar.gz \ - https://www.zlib.net/fossils/zlib-1.3.2.tar.gz \ + https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz \ https://www.openssl.org/source/openssl-3.6.2.tar.gz \ ; do archive="$(basename "$url")"