fix: harden runtime lifecycle and request failure handling

.env.example

@@ -15,6 +15,7 @@ SearchRequestDeadlineMs=30000
 SearchReadinessTimeoutMs=5000
 RequestLogSuccessSampleRate=0.1
 RequestLogHealthChecks=false
+ShutdownTimeoutMs=35000
 WebSearchEnabled=true
 SearchRateLimitEnabled=false
 SearchRateLimitWindowMs=60000

.github/workflows/quality.yml

@@ -18,10 +18,10 @@ jobs:
 
         steps:
             - name: Check out repository
-              uses: actions/checkout@v4
+              uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
 
             - name: Set up Node.js
-              uses: actions/setup-node@v4
+              uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
               with:
                   node-version: 24.14.1
                   cache: npm
@@ -31,3 +31,53 @@ jobs:
 
             - name: Run quality checks
               run: npm run quality
+
+    container:
+        name: Runtime Container
+        runs-on: ubuntu-latest
+        timeout-minutes: 15
+
+        steps:
+            - name: Check out repository
+              uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+
+            - name: Build production image
+              run: docker build --tag cve-search-api:pr .
+
+            - name: Start production image
+              run: |
+                  docker run --detach \
+                    --name cve-search-api-pr \
+                    --publish 4100:3000 \
+                    --env NODE_CONFIG_ENV=prod \
+                    --env OpenSearchDomainEndpoint=https://search.example.com \
+                    --env OpenSearchCveIndex=cves \
+                    cve-search-api:pr
+
+            - name: Validate runtime image
+              run: |
+                  for attempt in $(seq 1 30); do
+                    if curl --fail --silent --show-error http://127.0.0.1:4100/api/health-check; then
+                      break
+                    fi
+                    if [ "$attempt" -eq 30 ]; then
+                      exit 1
+                    fi
+                    sleep 1
+                  done
+
+                  curl --fail --silent --show-error http://127.0.0.1:4100/api-docs/openapi.json \
+                    | jq --exit-status \
+                      '.openapi == "3.0.2" and .info.title == "CVE Search API"' \
+                    > /dev/null
+
+                  test "$(docker image inspect cve-search-api:pr --format '{{.Config.User}}')" = "65532:65532"
+                  test "$(docker image inspect cve-search-api:pr --format '{{json .Config.Cmd}}')" = '["server.js"]'
+
+            - name: Capture container logs
+              if: always()
+              run: docker logs cve-search-api-pr || true
+
+            - name: Remove test container
+              if: always()
+              run: docker rm --force cve-search-api-pr || true

README.md

@@ -14,11 +14,14 @@ Rename `.env.example` to `.env` and replace or check the following values.
 - `SearchPostFilterColdSearchConcurrency` - optional per-process concurrent cold CPE candidate search limit, default `10`
 - `SearchPostFilterColdSearchMaxQueueDepth` - optional per-process waiting queue depth for cold CPE candidate searches, default `500`
 - `SearchPostFilterColdSearchMaxWaitMs` - optional maximum cold CPE candidate queue wait, default `10000`
+- `SearchInFlightMaxWaitersPerOperation` - optional per-process caller limit for one shared semantic search operation, default `500`
+- `SearchPostFilterEvaluationChunkSize` - optional semantic CPE cancellation/yield interval, default `250`
 - `SearchOpenSearchRequestTimeoutMs` - optional timeout for each OpenSearch request, default `20000`
 - `SearchRequestDeadlineMs` - optional overall `/search` processing deadline, default `30000`
 - `SearchReadinessTimeoutMs` - optional timeout for the OpenSearch index readiness check, default `5000`
 - `RequestLogSuccessSampleRate` - optional fraction of successful non-health-check access logs to retain, default `0.1`
 - `RequestLogHealthChecks` - optional flag to retain successful health-check access logs, default `false`
+- `ShutdownTimeoutMs` - optional graceful HTTP shutdown deadline, default `35000`
 - `WebSearchEnabled` - optional deprecated `/webSearch` feature flag, default `true`
 - `SearchRateLimitEnabled` - optional per-process `/search` rate-limit flag, default `false`
 - `SearchRateLimitWindowMs` - optional rate-limit window in milliseconds, default `60000`
@@ -89,13 +92,21 @@ The Search API will be served at `localhost:3000`.
 
 `GET /api/health-check` is a process liveness check. It returns HTTP 200 when the Node.js process is running and deliberately does not contact OpenSearch. Use it where a dependency outage should not restart otherwise healthy API tasks.
 
-`GET /api/ready-check` sends a minimal zero-result search to the configured CVE index. This verifies connectivity, index availability, and the same search permission used by the API without retrieving CVE records or counting all matches. It returns HTTP 200 when search requests can be served. Configuration errors, connection failures, timeouts, incompatible `cve-core` SearchReader contracts, and a missing index return a sanitized HTTP 503 `SEARCH_NOT_READY` response with `Retry-After: 5`. The OpenSearch request is limited to five seconds by default and can be changed with `SearchReadinessTimeoutMs`.
+`GET /api/ready-check` sends a minimal zero-result search to the configured CVE index. This verifies connectivity, index availability, and the same search permission used by the API without retrieving CVE records or counting all matches. It returns HTTP 200 only when search requests can still be served after the dependency check completes. Configuration errors, connection failures, timeouts, incompatible `cve-core` SearchReader contracts, a missing index, and an in-progress process shutdown return a sanitized HTTP 503 `SEARCH_NOT_READY` response with `Retry-After: 5`. The OpenSearch request is limited to five seconds by default and can be changed with `SearchReadinessTimeoutMs`. If shutdown is already active, readiness fails immediately without contacting OpenSearch. If shutdown begins while the dependency check is pending, the completed check is not reported as ready.
 
 An AWS load balancer can use `/api/ready-check` to stop routing search traffic to an unready task while the container orchestrator continues using `/api/health-check` for process liveness. Coordinate that probe selection with the infrastructure team before changing deployed health-check behavior.
 
+## Graceful Shutdown
+
+When the runtime receives `SIGTERM` or `SIGINT`, it marks the process as shutting down before closing the HTTP listener. Readiness changes to HTTP 503, while liveness remains HTTP 200 for as long as the process is running. Existing requests are allowed to drain. New non-probe requests attempted over existing keep-alive connections receive HTTP 503 `SERVICE_SHUTTING_DOWN`, `Retry-After: 5`, and `Connection: close`.
+
+The server closes idle connections and waits up to `ShutdownTimeoutMs`, which defaults to 35 seconds. A successful drain exits normally. If the deadline expires, the process force-closes remaining connections and exits with a failure code. Configure the container orchestrator's stop timeout to exceed `ShutdownTimeoutMs`; otherwise the orchestrator can terminate the process before its application-level drain completes.
+
+Shutdown lifecycle events are emitted as sanitized `runtime.lifecycle` JSON logs. Unexpected Express errors emit a sanitized `request.error` event and return HTTP 500 `INTERNAL_SERVER_ERROR`. Neither event includes exception messages, stacks, query strings, request bodies, credentials, or upstream response details.
+
 ## API Input Validation
 
-`POST /search` accepts an optional JSON object request body and validates supported fields, boolean parameters, pagination values, CVSS severity values, required parameter combinations, CPE Name requirements, and date-time formats. A non-empty body must use the `application/json` media type with UTF-8 encoding; unsupported media types or encodings return HTTP 415, and malformed JSON returns HTTP 400. If OpenSearch is unavailable, the endpoint returns HTTP 503 with a `Retry-After` header.
+`POST /search` accepts an optional JSON object request body and validates supported fields, boolean parameters, pagination values, CVSS severity values, required parameter combinations, CPE Name requirements, and date-time formats. The JSON request body cannot exceed 100 KB; larger bodies return HTTP 413 before routing or OpenSearch access. A non-empty body must use the `application/json` media type with UTF-8 encoding; unsupported media types or encodings return HTTP 415, and malformed JSON returns HTTP 400. If OpenSearch is unavailable, the endpoint returns HTTP 503 with a `Retry-After` header.
 
 String inputs are bounded before query construction:
 
@@ -253,6 +264,25 @@ npm run coverage
 npm run quality
 ```
 
+Pull requests run the quality gate and independently build the production `Dockerfile`. The container job starts the built image, checks process liveness and the packaged OpenAPI document, and verifies the runtime user and command. It intentionally does not call `/api/ready-check` or `/search`, because pull-request CI does not own or provision an OpenSearch index.
+
+An opt-in, read-only OpenSearch contract check is available for an approved environment that can reach an existing index:
+
+```bash
+npm run test:opensearch-contract
+```
+
+Set `OpenSearchDomainEndpoint` and `OpenSearchCveIndex` in the environment or local `.env` before running it. `OpenSearchAllowUnknownSslCerts` remains optional and should be enabled only for local development against a self-signed certificate. The contract check sends a minimal readiness query and one single-hit search, validates the response shape, closes its OpenSearch client, and emits sanitized JSON output. It never creates, updates, maps, or deletes indexes.
+
+The contract check is not part of public pull-request CI. A future automated invocation should use a protected GitHub environment or self-hosted runner with network access to an approved test index. Connection values and credentials must remain in environment secrets rather than source files or workflow text.
+
+The validation tiers have separate purposes:
+
+- Jest and `npm run quality` verify application behavior, API contracts, and regression coverage with controlled test doubles.
+- Pull-request container validation verifies production-image packaging, non-root execution, startup, liveness, and packaged API documentation.
+- The optional OpenSearch contract check verifies that the installed client can communicate with an externally managed index using the response contract expected by this application.
+- Load and performance testing remain in the separate automation performance-testing repository.
+
 ### Operational Logging
 
 Every request receives an `X-Request-ID` response header. A caller-provided `X-Request-ID` is preserved when it contains only letters, numbers, `.`, `_`, `:`, or `-` and is no longer than 128 characters; otherwise the API generates a UUID. The same value appears as `request-id` in access logs and `requestId` in structured `/search` events.

api-docs/openapi.test.js

@@ -29,6 +29,9 @@ describe('OpenAPI contract', () => {
         expect(openapi.paths['/search'].post.operationId).toBe('search');
         expect(openapi.paths['/api/health-check'].get.operationId).toBe('healthCheck');
         expect(openapi.paths['/api/ready-check'].get.operationId).toBe('readinessCheck');
+        expect(openapi.paths['/api/ready-check'].get.description).toContain(
+            'shutdown begins while the dependency check is pending'
+        );
         expect(openapi.paths['/webSearch'].post.operationId).toBe('webSearch');
     });
 
@@ -63,7 +66,9 @@ describe('OpenAPI contract', () => {
         const mediaType = operation.requestBody.content['application/json'];
 
         expect(operation.parameters || []).toStrictEqual([]);
+        expect(operation.requestBody.description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`);
         expect(mediaType.schema).toStrictEqual({ $ref: '#/components/schemas/SearchRequest' });
+        expect(operation.responses['413'].description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`);
         expect(schema.additionalProperties).toBe(false);
         expect(Object.keys(schema.properties)).toStrictEqual(constants.SEARCH_QUERY_PARAMS);
         Object.values(schema.properties).forEach((property) => {
@@ -202,11 +207,13 @@ describe('OpenAPI contract', () => {
             }
         ]);
 
-        ['404', '415', '429', '502', '503', '504'].forEach((status) => {
+        ['404', '415', '429', '500', '502', '503', '504'].forEach((status) => {
             expect(responses[status].content['application/json'].schema).toStrictEqual({
                 $ref: '#/components/schemas/ErrorResponse'
             });
         });
+
+        expect(responses['503'].description).toContain('shutting down');
     });
 
     test('documents request correlation headers', () => {
@@ -219,6 +226,7 @@ describe('OpenAPI contract', () => {
         expect(openapi.paths['/api/health-check'].get.responses['200'].headers['X-Request-ID']).toBeDefined();
         expect(openapi.paths['/api/ready-check'].get.responses['200'].headers['X-Request-ID']).toBeDefined();
         expect(openapi.paths['/api/ready-check'].get.responses['503'].headers['Retry-After']).toBeDefined();
+        expect(openapi.paths['/api/ready-check'].get.responses['503'].description).toContain('shutting down');
         expect(openapi.paths['/webSearch'].post.responses['200'].headers['X-Request-ID']).toBeDefined();
     });
 

config/custom-environment-variables.jsonc

@@ -27,6 +27,9 @@
             "successSampleRate": "RequestLogSuccessSampleRate",
             "healthChecks": "RequestLogHealthChecks"
         },
+        "runtime": {
+            "shutdownTimeoutMs": "ShutdownTimeoutMs"
+        },
         "features": {
             "webSearchEnabled": "WebSearchEnabled"
         },

config/default.jsonc

@@ -43,6 +43,10 @@
             // Set to TRUE to retain successful health-check access logs.
             "healthChecks": "FALSE"
         },
+        "runtime": {
+            // Must remain shorter than the container orchestrator's stop timeout.
+            "shutdownTimeoutMs": "35000"
+        },
         // Feature flags used for controlled endpoint migrations.
         "features": {
             // Keep enabled until existing consumers have migrated to POST /search.

config/runtime-config.test.js

@@ -69,6 +69,18 @@ describe('runtime container and compose configuration', () => {
         expect(dockerignore).toContain('**/reports');
     });
 
+    test('pull request CI builds and smoke tests the production runtime image', () => {
+        const workflow = readProjectFile('.github', 'workflows', 'quality.yml');
+
+        expect(workflow).toContain('name: Runtime Container');
+        expect(workflow).toContain('docker build --tag cve-search-api:pr .');
+        expect(workflow).toContain('http://127.0.0.1:4100/api/health-check');
+        expect(workflow).toContain('http://127.0.0.1:4100/api-docs/openapi.json');
+        expect(workflow).toContain(`--format '{{.Config.User}}'`);
+        expect(workflow).toContain(`--format '{{json .Config.Cmd}}'`);
+        expect(workflow).not.toContain('/api/ready-check');
+    });
+
     test('server-only compose file joins the existing OpenSearch 3.5 network', () => {
         const compose = readProjectFile('docker-compose.yaml');
 
@@ -111,6 +123,7 @@ describe('runtime container and compose configuration', () => {
         expect(customEnvironmentVariables).toContain('"readinessTimeoutMs": "SearchReadinessTimeoutMs"');
         expect(customEnvironmentVariables).toContain('"successSampleRate": "RequestLogSuccessSampleRate"');
         expect(customEnvironmentVariables).toContain('"healthChecks": "RequestLogHealthChecks"');
+        expect(customEnvironmentVariables).toContain('"shutdownTimeoutMs": "ShutdownTimeoutMs"');
         expect(customEnvironmentVariables).toContain('"webSearchEnabled": "WebSearchEnabled"');
         expect(customEnvironmentVariables).toContain('"enabled": "SearchRateLimitEnabled"');
         expect(customEnvironmentVariables).toContain('"windowMs": "SearchRateLimitWindowMs"');

controllers/middleware.js

@@ -117,7 +117,7 @@ function isValidDateTime(value) {
 //Ensures parameters are valid and checks for parameters that can't be called together
 function validateParams(req, res, next) {
     try {
-        const params = req.searchParams ?? req.query ?? {};
+        const params = req.searchParams ?? {};
 
         //Ensures all specified parameter names are valid
         for (const param of Object.keys(params)) {

controllers/middleware.test.js

@@ -926,8 +926,8 @@ describe('middleware validation function', () => {
     );
 
     test('ValidateParams delegates unexpected errors to next()', () => {
-        const err = new Error('query failure');
-        Object.defineProperty(req, 'query', {
+        const err = new Error('search parameter failure');
+        Object.defineProperty(req, 'searchParams', {
             get: () => {
                 throw err;
             }