diff --git a/.env.example b/.env.example index 795db04..ef0af0e 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,6 @@ +NODE_CONFIG_ENV=devel PORT=3000 +BASE_URL= OpenSearchCveIndex=cve-index-local OpenSearchAllowUnknownSslCerts=false OpenSearchDomainEndpoint= @@ -16,6 +18,9 @@ SearchReadinessTimeoutMs=5000 RequestLogSuccessSampleRate=0.1 RequestLogHealthChecks=false ShutdownTimeoutMs=35000 +HttpHeadersTimeoutMs=10000 +HttpRequestTimeoutMs=15000 +HttpKeepAliveTimeoutMs=5000 WebSearchEnabled=true SearchRateLimitEnabled=false SearchRateLimitWindowMs=60000 diff --git a/Dockerfile b/Dockerfile index 8914334..10d044f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot FROM node:${NODE_VERSION} AS deps -ENV NODE_ENV=production +ENV NODE_ENV=production \ + NODE_CONFIG_ENV=prod WORKDIR /usr/src/app @@ -18,7 +19,8 @@ RUN npm ci --omit=dev \ FROM ${DISTROLESS_NODE_IMAGE} AS runtime -ENV NODE_ENV=production +ENV NODE_ENV=production \ + NODE_CONFIG_ENV=prod WORKDIR /usr/src/app diff --git a/Dockerfile.local b/Dockerfile.local index 65345fb..a0977de 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -5,7 +5,8 @@ ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot FROM node:${NODE_VERSION} AS deps -ENV NODE_ENV=production +ENV NODE_ENV=production \ + NODE_CONFIG_ENV=prod WORKDIR /usr/src/app @@ -23,7 +24,8 @@ RUN --mount=type=secret,id=npm_ca,required=false \ FROM ${DISTROLESS_NODE_IMAGE} AS runtime -ENV NODE_ENV=production +ENV NODE_ENV=production \ + NODE_CONFIG_ENV=prod WORKDIR /usr/src/app diff --git a/README.md b/README.md index fb9c40d..b3d9e6b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ Rename `.env.example` to `.env` and replace or check the following values. +- `NODE_CONFIG_ENV` - node-config profile name; use `devel` locally and `prod` in production images - `PORT` - port used for the search API, defaulted to 3000 for development +- `BASE_URL` - optional absolute public URL used by local runtime helpers; credentials are not permitted - `OpenSearchCveIndex` - existing CVE index name in your OpenSearch deployment - `OpenSearchAllowUnknownSslCerts` - set to `true` only for local development against self-signed certificates - `OpenSearchDomainEndpoint` - absolute URL to an existing OpenSearch deployment. This API does not create or manage the OpenSearch index or mapping. @@ -22,6 +24,9 @@ Rename `.env.example` to `.env` and replace or check the following values. - `RequestLogSuccessSampleRate` - optional fraction of successful non-health-check access logs to retain, default `0.1` - `RequestLogHealthChecks` - optional flag to retain successful health-check access logs, default `false` - `ShutdownTimeoutMs` - optional graceful HTTP shutdown deadline, default `35000` +- `HttpHeadersTimeoutMs` - optional deadline for receiving HTTP request headers, default `10000` +- `HttpRequestTimeoutMs` - optional deadline for receiving a complete HTTP request, including its body, default `15000` +- `HttpKeepAliveTimeoutMs` - optional idle keep-alive connection timeout, default `5000` - `WebSearchEnabled` - optional deprecated `/webSearch` feature flag, default `true` - `SearchRateLimitEnabled` - optional per-process `/search` rate-limit flag, default `false` - `SearchRateLimitWindowMs` - optional rate-limit window in milliseconds, default `60000` @@ -32,6 +37,45 @@ Rename `.env.example` to `.env` and replace or check the following values. mv .env.example .env ``` +### Runtime Configuration Validation + +The API loads `.env` before `cve-core`, resolves runtime settings once, and uses one immutable configuration snapshot for the life of the process. Explicitly configured Boolean values must be `true` or `false`, without regard to case. Malformed or out-of-range values stop startup instead of silently falling back to defaults. + +The following numeric bounds are enforced: + +| Setting | Accepted range | +| ----------------------------------------- | -------------------------------------------------------------------: | +| `PORT` | 1 through 65,535 | +| `SearchPostFilterCacheTtlMs` | 0 through 86,400,000 | +| `SearchPostFilterCacheMaxEntries` | 0 through 10,000 | +| `SearchPostFilterColdSearchConcurrency` | 1 through 100 | +| `SearchPostFilterColdSearchMaxQueueDepth` | 0 through 10,000 | +| `SearchPostFilterColdSearchMaxWaitMs` | 1 through 300,000 | +| `SearchInFlightMaxWaitersPerOperation` | 1 through 10,000 | +| `SearchPostFilterEvaluationChunkSize` | 1 through 10,000 | +| `SearchOpenSearchRequestTimeoutMs` | 1 through 300,000 | +| `SearchRequestDeadlineMs` | 1 through 600,000 | +| `SearchReadinessTimeoutMs` | 1 through 60,000 | +| `RequestLogSuccessSampleRate` | 0 through 1 | +| `ShutdownTimeoutMs` | 1 through 900,000 | +| `HttpHeadersTimeoutMs` | 1 through 60,000 | +| `HttpRequestTimeoutMs` | 1 through 300,000 | +| `HttpKeepAliveTimeoutMs` | 1 through 60,000 | +| `SearchRateLimitWindowMs` | 1 through 86,400,000 | +| `SearchRateLimitMaxRequests` | 0 through 10,000,000; must be positive when rate limiting is enabled | + +Cache TTL, cache entry count, and cold-search queue depth accept `0` because those settings have defined disable or no-wait semantics. The timeout settings must also satisfy: + +```text +SearchReadinessTimeoutMs <= SearchOpenSearchRequestTimeoutMs < SearchRequestDeadlineMs < ShutdownTimeoutMs +SearchPostFilterColdSearchMaxWaitMs < SearchRequestDeadlineMs +HttpHeadersTimeoutMs <= HttpRequestTimeoutMs +``` + +`OpenSearchDomainEndpoint` must be an absolute HTTP or HTTPS URL, and `OpenSearchCveIndex` must be non-empty. `BASE_URL`, when supplied, must also be an absolute HTTP or HTTPS URL and cannot contain credentials. OpenSearch connectivity is not required at startup; `/api/ready-check` remains responsible for dependency readiness. + +Production images set both `NODE_ENV=production` and `NODE_CONFIG_ENV=prod`, matching `config/prod.jsonc`. Successful startup emits a sanitized `runtime.lifecycle` JSON event with effective safety settings. Startup failures identify only the invalid configuration path and never log configured values, URLs, index names, credentials, or exception messages. + ## Docker API Only Use `docker-compose.yaml` when OpenSearch 3.5 is already running on the external Docker network `opensearch-35_opensearch-net`. @@ -113,7 +157,7 @@ String inputs are bounded before query construction: | Field | Limit | | ---------------------------- | ---------------------------------------------------------------------: | | `keywordSearch` | 512 characters and 64 whitespace-delimited terms | -| `cveIds` | 4,096 characters overall, 100 IDs, and 64 characters per ID | +| `cveIds` | 4,096 characters overall, 100 IDs, and 4-19 sequence digits per CVE ID | | `cweId` | 32 characters | | `cpeName` | 2,048 characters | | `virtualMatchString` | 2,048 characters | @@ -134,7 +178,7 @@ Publication and last-modified date filters support one-sided ranges. When both v Pagination is limited to complete pages inside the first 10,000 results: `from + resultsPerPage` must not exceed 10,000. `totalPages` reports the smaller of the pages implied by the match count and the complete pages accessible inside that window. For example, a page size of 30 allows at most 333 pages. The match total itself is not reduced by this pagination calculation. Point-in-time search and deep pagination remain deferred. -Use `cveIds` to request up to 100 CVE records by comma-separated CVE ID. +Use `cveIds` to request up to 100 CVE records by comma-separated CVE ID. Each ID must contain the `CVE` prefix, a four-digit year, and a sequence number containing 4 to 19 digits. The prefix is accepted without regard to case and normalized to uppercase for the OpenSearch query. ```bash curl -X POST "http://localhost:3000/search" \ @@ -152,6 +196,8 @@ curl -X POST "http://localhost:3000/search" \ When filtering with `cpeName`, provide a CPE 2.3 name with part `a`, `h`, or `o` and non-wildcard `vendor`, `product`, and `version` components. Invalid `cpeName` values return HTTP 404. The API compares the provided CPE Name against CPE Match Criteria, including exact match strings and version range match strings when the indexed CVE data includes range fields. Use `virtualMatchString` when you intentionally need to search for partial CPE Match String values without those required components. +Both `cpeName` and `virtualMatchString` evaluate individual CPE Match Criteria entries in the direct `cpeApplicability.nodes[].cpeMatch` arrays. They do not evaluate a node's `operator` or `negate` value as a complete system applicability expression. Evaluating those Boolean expressions requires a set of installed assets, while each search request supplies only one CPE Name or Match String. + ### CPE Name Search Use `cpeName` when a user has a concrete asset CPE Name and needs CVE records whose CPE Match Criteria apply to that asset version. The part must be `a`, `h`, or `o`; the `vendor`, `product`, and `version` components must all be concrete values, not `*`. @@ -180,7 +226,7 @@ Set `isVulnerable` to `true` with `cpeName` to require the matching CPE Match Cr ### Virtual Match String Search -Use `virtualMatchString` when a user needs CPE Match Criteria compatibility matching instead of concrete asset matching. String literal components are compared without regard to lexical case. When a part is supplied, it must be `a`, `h`, `o`, `*`, or `-`. Omitted trailing components and `*` components act as broad matches. +Use `virtualMatchString` when a user needs CPE Match Criteria compatibility matching instead of concrete asset matching. String literal components are compared without regard to lexical case. When a part is supplied, it must be `a`, `h`, `o`, `*`, or `-`. Omitted trailing components and `*` components act as broad matches. Explicit empty colon-delimited components are invalid; omit trailing components instead. Partial product example: @@ -291,13 +337,15 @@ Access logs omit query strings and request bodies. By default, the API retains 1 The API always emits a sanitized JSON `search.performance` event for `/search` queue saturation, shared-operation waiter saturation, queue wait timeout, search timeout, client disconnect, OpenSearch connection errors, and upstream search errors. These events include the request ID, outcome, HTTP status, strategy, cache state, total elapsed time, cancellation reason, and bounded queue or waiter state when applicable. Search terms, CPE values, request bodies, credentials, and upstream error details are never included. -Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, shared waiter counts, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, and total handler elapsed time. Leave it disabled during normal operation unless the deployment logging volume is acceptable. +Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, shared waiter counts, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, response bytes, JSON serialization time, process heap usage and per-request heap delta, cache entry and cached CVE ID counts, process-wide event-loop delay, and total handler elapsed time. Leave it disabled during normal operation unless the deployment logging volume is acceptable. + +Heap deltas and event-loop delay are process-level diagnostics rather than isolated per-request measurements because concurrent requests share the same Node.js process. Use them to correlate latency with memory pressure or event-loop stalls across a test window, not to attribute resource use precisely to one request. Each `/search` request owns a lifecycle cancellation signal. When its overall deadline expires, the API aborts the active OpenSearch transport request before returning HTTP 504. When the caller disconnects, the API aborts work that is needed only by that caller and does not attempt to write a response. This prevents timed-out or abandoned standard searches from continuing until the transport timeout. ### Application Rate Limiting -Application rate limiting is disabled by default and applies only to `POST /search`. It does not apply to `/webSearch`, health checks, Swagger, OpenAPI, or static assets. +Application rate limiting is disabled by default and applies only to `POST /search`. It runs before JSON body parsing so a rejected request does not consume body-parsing work. It does not apply to `/webSearch`, health checks, Swagger, OpenAPI, or static assets. The fallback limiter uses one in-memory counter per Node.js process. Set `SearchRateLimitEnabled=true`, provide a positive `SearchRateLimitMaxRequests`, and optionally change `SearchRateLimitWindowMs` to enable it. Invalid enabled configuration prevents the application from starting. Rejected requests return HTTP 429 with `SEARCH_RATE_LIMITED`, `Retry-After`, `RateLimit`, and `RateLimit-Policy` headers, and emit a sanitized `search.performance` event with the `rate_limited` outcome. @@ -309,6 +357,8 @@ Searches that use `cpeName`, `virtualMatchString`, or `virtualMatchString` with The post-filter cache stores only the filtered CVE ID list and candidate count for that expensive candidate query. It does not cache full CVE records. On a cache hit, the API skips the 10,000-candidate OpenSearch query and semantic post-filtering, slices the cached CVE IDs for the requested page, and fetches that page of full CVE records by ID. +Candidate retrieval asks OpenSearch for exact-version or wildcard-version criteria and leaves range applicability decisions to the semantic post-filter. The full-record page fetch uses an exact CVE ID set without total-hit counting or OpenSearch sorting; the API restores the requested CVE ID order locally. Standard searches retain exact total counting and deterministic sorting. + The cache is intentionally small and short-lived: - TTL: 60 seconds @@ -350,6 +400,10 @@ The cache and cold-search queue are a temporary per-process proof of concept. Th The request lifecycle and in-flight waiter registry are separate from the result cache, so a later shared-cache replacement does not need to change caller cancellation behavior. The per-process TTL, entry limit, cold-search concurrency, queue depth, queue wait, shared waiter limit, evaluation chunk size, OpenSearch timeout, and overall request deadline can be tuned with `SearchPostFilterCacheTtlMs`, `SearchPostFilterCacheMaxEntries`, `SearchPostFilterColdSearchConcurrency`, `SearchPostFilterColdSearchMaxQueueDepth`, `SearchPostFilterColdSearchMaxWaitMs`, `SearchInFlightMaxWaitersPerOperation`, `SearchPostFilterEvaluationChunkSize`, `SearchOpenSearchRequestTimeoutMs`, and `SearchRequestDeadlineMs`. +### HTTP Transport Timeouts + +The Node.js HTTP server limits request-header receipt to 10 seconds, complete request receipt (including the body) to 15 seconds, and idle keep-alive connections to 5 seconds. These limits bound slow-client connection usage before application validation begins. They do not shorten the 30-second `/search` processing deadline, which starts after the request reaches the search handler. Override the defaults with `HttpHeadersTimeoutMs`, `HttpRequestTimeoutMs`, and `HttpKeepAliveTimeoutMs`; header receipt cannot be configured longer than complete request receipt. + Docker runtime smoke checks can be run after building the image. The default `Dockerfile` avoids BuildKit `RUN --mount` syntax for AWS builders that do not support it: ```bash diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 6b1f60e..b889eed 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -403,7 +403,7 @@ "type": "string", "maxLength": 4096, "example": "CVE-2024-1234,CVE-2024-5678", - "description": "Returns one or more CVE records identified by a comma-separated list of CVE IDs. Limited to 100 CVE IDs, 64 characters per ID, and 4096 characters overall." + "description": "Returns one or more CVE records identified by a comma-separated list of CVE IDs. Each ID must contain the CVE prefix, a four-digit year, and a 4-to-19-digit sequence number. The prefix is accepted without regard to case. Limited to 100 CVE IDs and 4096 characters overall." }, "noRejected": { "type": "boolean", @@ -499,7 +499,7 @@ "type": "string", "maxLength": 2048, "example": "cpe:2.3:a:freetype:freetype:2.8.1:*:*:*:*:*:*:*", - "description": "Returns CVE records whose CPE Match Criteria apply to the provided CPE 2.3 asset name. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches." + "description": "Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches." }, "isVulnerable": { "type": "boolean", @@ -577,7 +577,7 @@ "type": "string", "maxLength": 2048, "example": "cpe:2.3:o:microsoft:windows_10_1809", - "description": "Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component." + "description": "Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component." } } }, diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 4f7e846..5162c29 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -78,7 +78,9 @@ describe('OpenAPI contract', () => { expect(schema.properties.cveIds.type).toBe('string'); expect(schema.properties.cveIds.description).toContain('comma-separated list of CVE IDs'); expect(schema.properties.cveIds.description).toContain(`${constants.MAX_CVE_IDS} CVE IDs`); - expect(schema.properties.cveIds.description).toContain(`${constants.MAX_CVE_ID_LENGTH} characters per ID`); + expect(schema.properties.cveIds.description).toContain( + `${constants.MIN_CVE_ID_SEQUENCE_DIGITS}-to-${constants.MAX_CVE_ID_SEQUENCE_DIGITS}-digit sequence number` + ); expect(schema.properties.keywordSearch.description).toContain( `${constants.MAX_KEYWORD_SEARCH_TOKENS} whitespace-delimited terms` ); @@ -97,6 +99,8 @@ describe('OpenAPI contract', () => { expect(schema.properties.cpeName.description).toContain('optional * components remain broad matches'); expect(schema.properties.cpeName.description).toContain('insensitive to lexical case'); expect(schema.properties.cpeName.description).toContain('exact CPE Match Criteria preserve leading zeros'); + expect(schema.properties.cpeName.description).toContain('entries independently'); + expect(schema.properties.cpeName.description).toContain("does not evaluate a node's operator or negate value"); expect(schema.properties.versionStartType.enum).toStrictEqual(constants.CPE_VERSION_RANGE_TYPES); expect(schema.properties.versionEndType.enum).toStrictEqual(constants.CPE_VERSION_RANGE_TYPES); expect(schema.properties.resultsPerPage.maximum).toBe(constants.MAX_RESULTS_LIMIT); @@ -116,6 +120,13 @@ describe('OpenAPI contract', () => { expect(schema.properties.virtualMatchString.description).toContain('insensitive to lexical case'); expect(schema.properties.virtualMatchString.description).toContain('without a version component'); expect(schema.properties.virtualMatchString.description).toContain('part must be a, h, o, *, or -'); + expect(schema.properties.virtualMatchString.description).toContain( + 'explicit empty colon-delimited components are invalid' + ); + expect(schema.properties.virtualMatchString.description).toContain('entries independently'); + expect(schema.properties.virtualMatchString.description).toContain( + "does not evaluate a node's operator or negate value" + ); expect(schema.properties.virtualMatchString.example).toBe('cpe:2.3:o:microsoft:windows_10_1809'); expect(schema.properties.cweId.description).toContain('CNA data'); expect(schema.properties.cweId.description).toContain('ADP-provided CWE data is not searched'); diff --git a/config/custom-environment-variables.jsonc b/config/custom-environment-variables.jsonc index 205aa1f..e9a2744 100644 --- a/config/custom-environment-variables.jsonc +++ b/config/custom-environment-variables.jsonc @@ -28,7 +28,12 @@ "healthChecks": "RequestLogHealthChecks" }, "runtime": { - "shutdownTimeoutMs": "ShutdownTimeoutMs" + "shutdownTimeoutMs": "ShutdownTimeoutMs", + "http": { + "headersTimeoutMs": "HttpHeadersTimeoutMs", + "requestTimeoutMs": "HttpRequestTimeoutMs", + "keepAliveTimeoutMs": "HttpKeepAliveTimeoutMs" + } }, "features": { "webSearchEnabled": "WebSearchEnabled" diff --git a/config/default.jsonc b/config/default.jsonc index 33561f2..0c4c0b8 100644 --- a/config/default.jsonc +++ b/config/default.jsonc @@ -5,6 +5,8 @@ // - Each configuration is mapped using custom-environment-variables.jsonc to enable environment varialbe overrides. // - For more information, see cve-core/src/adapters/config/AppConfig.ts // NOTE for consistency, all values need to be strings for proper type when using .env overrides + // Explicit invalid values fail application startup. See README.md for accepted ranges and + // required timeout relationships. "appConfig": { // constants for search capability "search": { @@ -45,7 +47,15 @@ }, "runtime": { // Must remain shorter than the container orchestrator's stop timeout. - "shutdownTimeoutMs": "35000" + "shutdownTimeoutMs": "35000", + "http": { + // Limits how long a client can take to send request headers. + "headersTimeoutMs": "10000", + // Limits how long a client can take to send the complete request, including its body. + "requestTimeoutMs": "15000", + // Limits how long an idle keep-alive connection remains open. + "keepAliveTimeoutMs": "5000" + } }, // Feature flags used for controlled endpoint migrations. "features": { diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index 4c969ec..1dc7f66 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -12,6 +12,7 @@ describe('runtime container and compose configuration', () => { expect(dockerfile).toContain('ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot'); expect(dockerfile).toContain('FROM node:${NODE_VERSION} AS deps'); expect(dockerfile).toContain('FROM ${DISTROLESS_NODE_IMAGE} AS runtime'); + expect(dockerfile).toContain('NODE_CONFIG_ENV=prod'); expect(dockerfile).toMatch(/^RUN npm ci --omit=dev \\/m); expect(dockerfile).toMatch(/^COPY --chown=65532:65532 config \.\/config\//m); expect(dockerfile).toMatch(/^COPY --chown=65532:65532 controllers \.\/controllers\//m); @@ -36,6 +37,7 @@ describe('runtime container and compose configuration', () => { expect(dockerfile).toContain('ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot'); expect(dockerfile).toContain('FROM node:${NODE_VERSION} AS deps'); expect(dockerfile).toContain('FROM ${DISTROLESS_NODE_IMAGE} AS runtime'); + expect(dockerfile).toContain('NODE_CONFIG_ENV=prod'); expect(dockerfile).toMatch(/^RUN --mount=type=secret,id=npm_ca,required=false/m); expect(dockerfile).toContain('if [ -s /run/secrets/npm_ca ]; then'); expect(dockerfile).toContain('NPM_CONFIG_CAFILE=/run/secrets/npm_ca npm ci --omit=dev'); @@ -61,6 +63,12 @@ describe('runtime container and compose configuration', () => { expect(localDockerfile).not.toContain('CMD ["./docker-entrypoint.sh"]'); }); + test('local environment example selects the development node-config profile', () => { + const environmentExample = readProjectFile('.env.example'); + + expect(environmentExample).toContain('NODE_CONFIG_ENV=devel'); + }); + test('Docker build context excludes tests and local report artifacts from runtime directory copies', () => { const dockerignore = readProjectFile('.dockerignore'); @@ -124,6 +132,9 @@ describe('runtime container and compose configuration', () => { expect(customEnvironmentVariables).toContain('"successSampleRate": "RequestLogSuccessSampleRate"'); expect(customEnvironmentVariables).toContain('"healthChecks": "RequestLogHealthChecks"'); expect(customEnvironmentVariables).toContain('"shutdownTimeoutMs": "ShutdownTimeoutMs"'); + expect(customEnvironmentVariables).toContain('"headersTimeoutMs": "HttpHeadersTimeoutMs"'); + expect(customEnvironmentVariables).toContain('"requestTimeoutMs": "HttpRequestTimeoutMs"'); + expect(customEnvironmentVariables).toContain('"keepAliveTimeoutMs": "HttpKeepAliveTimeoutMs"'); expect(customEnvironmentVariables).toContain('"webSearchEnabled": "WebSearchEnabled"'); expect(customEnvironmentVariables).toContain('"enabled": "SearchRateLimitEnabled"'); expect(customEnvironmentVariables).toContain('"windowMs": "SearchRateLimitWindowMs"'); @@ -140,16 +151,19 @@ describe('runtime container and compose configuration', () => { } ); - test.each([['README.md'], ['config/default.jsonc'], ['config/devel.jsonc'], ['config/prod.jsonc']])( - '%s does not include credential-bearing OpenSearch endpoint examples', - (filePath) => { - const content = readProjectFile(filePath); - const defaultCredentialPair = ['admin', 'admin'].join(':'); - const placeholderCredentialPair = ['', ''].join(':'); - - expect(content).not.toContain(defaultCredentialPair); - expect(content).not.toContain(placeholderCredentialPair); - expect(content).not.toMatch(/https?:\/\/[^/\s]+:[^@\s]+@/); - } - ); + test.each([ + ['README.md'], + ['config/default.jsonc'], + ['config/devel.jsonc'], + ['config/prod.jsonc'], + ['config/test.jsonc'] + ])('%s does not include credential-bearing OpenSearch endpoint examples', (filePath) => { + const content = readProjectFile(filePath); + const defaultCredentialPair = ['admin', 'admin'].join(':'); + const placeholderCredentialPair = ['', ''].join(':'); + + expect(content).not.toContain(defaultCredentialPair); + expect(content).not.toContain(placeholderCredentialPair); + expect(content).not.toMatch(/https?:\/\/[^/\s]+:[^@\s]+@/); + }); }); diff --git a/config/test.jsonc b/config/test.jsonc index 3cde5d6..d522127 100644 --- a/config/test.jsonc +++ b/config/test.jsonc @@ -1,3 +1,11 @@ { - // Tests inherit default configuration. This file exists so node-config recognizes NODE_ENV=test. + // Tests inherit default configuration. These non-routable values satisfy startup validation + // without requiring a live OpenSearch dependency. + "appConfig": { + "search": { + "providerEndpoint": "https://search.example.com", + "index": "test-index", + "allowUnknownSslCerts": "FALSE" + } + } } diff --git a/controllers/middleware.js b/controllers/middleware.js index cdbba5a..f210169 100644 --- a/controllers/middleware.js +++ b/controllers/middleware.js @@ -1,6 +1,7 @@ const errors = require('../utils/errors'); const constants = require('../utils/constants'); const cpe = require('../utils/cpe'); +const cveId = require('../utils/cveId'); const cvss = require('../utils/cvss'); const error = new errors.errors(); @@ -58,7 +59,6 @@ const nonStringSearchParams = [ const nonEmptyStringParams = constants.SEARCH_QUERY_PARAMS.filter((param) => !nonStringSearchParams.includes(param)); const dateTimeFormat = /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])T([01]\d|2[0-3]):[0-5]\d:[0-5]\d\.\d{3}Z$/; -const cveIdFormat = /^CVE-\d{4}-\d{4,}$/i; function isTrueParam(value) { return value === true || String(value).toLowerCase() === 'true'; @@ -83,8 +83,8 @@ function countWhitespaceDelimitedTokens(value) { function parseCveIds(value) { if (typeof value !== 'string') return null; - const cveIds = value.split(',').map((id) => id.trim()); - if (!cveIds.length || cveIds.some((id) => !id || !cveIdFormat.test(id))) return null; + const cveIds = value.split(',').map(cveId.normalizeCveId); + if (!cveIds.length || cveIds.some((id) => !cveId.isValidCveId(id))) return null; return cveIds; } @@ -152,15 +152,12 @@ function validateParams(req, res, next) { if (!cveIds) { return res.status(400).json(error.cveIdsFormatError()); } - if (cveIds.some((cveId) => cveId.length > constants.MAX_CVE_ID_LENGTH)) { - return res.status(400).json(error.cveIdLengthError()); - } if (cveIds.length > constants.MAX_CVE_IDS) { return res.status(400).json(error.cveIdsLimitError()); } } - if (hasParam(params, 'noRejected') && ![true, false, ''].includes(params.noRejected)) { + if (hasParam(params, 'noRejected') && ![true, false].includes(params.noRejected)) { return res.status(400).json(error.noRejectedValueError()); } diff --git a/controllers/middleware.test.js b/controllers/middleware.test.js index c793a8f..6f82844 100644 --- a/controllers/middleware.test.js +++ b/controllers/middleware.test.js @@ -158,6 +158,17 @@ describe('middleware validation function', () => { ); }); + test('ValidateParams rejects an explicit empty virtualMatchString component', () => { + validate({ + virtualMatchString: 'cpe:2.3:o::windows_10_1809' + }); + + expectValidationError( + 'VIRTUAL_MATCH_STRING_FORMAT', + 'The virtualMatchString parameter must be a CPE 2.3 match string.' + ); + }); + test('ValidateParams accepts broad virtualMatchString values', () => { validate({ virtualMatchString: 'cpe:2.3:o:*:windows_10_1809' @@ -344,6 +355,19 @@ describe('middleware validation function', () => { ); }); + test('ValidateParams rejects an explicit empty virtualMatchString component for version ranges', () => { + validate({ + virtualMatchString: 'cpe:2.3:o::linux_kernel', + versionStart: '1.0', + versionStartType: 'including' + }); + + expectValidationError( + 'VIRTUAL_MATCH_STRING_RANGE_FORMAT', + 'The virtualMatchString parameter must be a partial CPE 2.3 match string when used with version range parameters.' + ); + }); + test('ValidateParams returns an error when version range virtualMatchString includes a version component', () => { validate({ virtualMatchString: 'cpe:2.3:o:linux:linux_kernel:*', @@ -435,15 +459,20 @@ describe('middleware validation function', () => { }); test.each([ - ['empty entry', 'CVE-2024-0001,,CVE-2024-0002'], - ['invalid CVE ID', 'CVE-2024-0001,not-a-cve'], - ['repeated query parameter', ['CVE-2024-0001', 'CVE-2024-0002']] - ])('ValidateParams returns an error when cveIds has an %s.', (_caseName, cveIds) => { + ['an empty entry', 'CVE-2024-0001,,CVE-2024-0002'], + ['an invalid CVE ID', 'CVE-2024-0001,not-a-cve'], + ['a short sequence number', `CVE-2024-${'1'.repeat(constants.MIN_CVE_ID_SEQUENCE_DIGITS - 1)}`], + ['a long sequence number', `CVE-2024-${'1'.repeat(constants.MAX_CVE_ID_SEQUENCE_DIGITS + 1)}`], + ['a repeated query parameter', ['CVE-2024-0001', 'CVE-2024-0002']] + ])('ValidateParams returns an error when cveIds has %s.', (_caseName, cveIds) => { validate({ cveIds }); - expectValidationError('CVE_IDS_FORMAT', 'The cveIds parameter must be a comma-separated list of CVE IDs.'); + expectValidationError( + 'CVE_IDS_FORMAT', + `The cveIds parameter must be a comma-separated list of CVE IDs with a four-digit year and a ${constants.MIN_CVE_ID_SEQUENCE_DIGITS}-to-${constants.MAX_CVE_ID_SEQUENCE_DIGITS}-digit sequence number.` + ); }); test('ValidateParams returns an error when cveIds contains more than 100 IDs.', () => { @@ -481,30 +510,15 @@ describe('middleware validation function', () => { expectValidationSuccess(); }); - test('ValidateParams accepts cveIds and each CVE ID at their maximum lengths.', () => { - const idPrefix = 'CVE-2024-'; - const longId = `${idPrefix}${'1'.repeat(constants.MAX_CVE_ID_LENGTH - idPrefix.length)}`; - const shortId = `${idPrefix}${'2'.repeat(32 - idPrefix.length)}`; - const mediumId = `${idPrefix}${'3'.repeat(33 - idPrefix.length)}`; - const cveIds = [...Array(62).fill(longId), shortId, mediumId].join(','); - - expect(cveIds).toHaveLength(constants.SEARCH_STRING_LENGTH_LIMITS.cveIds); - validate({ cveIds }); - - expectValidationSuccess(); - }); - - test('ValidateParams returns an error when an individual CVE ID exceeds its maximum length.', () => { - const idPrefix = 'CVE-2024-'; - + test('ValidateParams accepts case-insensitive cveIds at both sequence-length boundaries.', () => { validate({ - cveIds: `${idPrefix}${'1'.repeat(constants.MAX_CVE_ID_LENGTH - idPrefix.length + 1)}` + cveIds: [ + `cve-2024-${'1'.repeat(constants.MIN_CVE_ID_SEQUENCE_DIGITS)}`, + `CVE-2024-${'2'.repeat(constants.MAX_CVE_ID_SEQUENCE_DIGITS)}` + ].join(',') }); - expectValidationError( - 'CVE_ID_LENGTH', - `Each CVE ID in the cveIds field must be ${constants.MAX_CVE_ID_LENGTH} characters or fewer.` - ); + expectValidationSuccess(); }); test.each([ @@ -560,6 +574,7 @@ describe('middleware validation function', () => { test.each([ ['true string', 'true'], ['false string', 'false'], + ['empty string', ''], ['repeated query parameter', ['', '']] ])('ValidateParams returns an error when noRejected has a non-boolean %s value.', (_caseName, noRejected) => { validate({ diff --git a/controllers/searchController.js b/controllers/searchController.js index 249ca2b..64152cc 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -1,8 +1,8 @@ const PackageInfo = require('cve-core').packageInfo; const errors = require('../utils/errors'); -const cpe = require('../utils/cpe'); const searchPerformance = require('../utils/searchPerformance'); const searchDeadline = require('../utils/searchDeadline'); +const searchCandidateProcessor = require('../utils/searchCandidateProcessor'); const searchPostFilterCache = require('../utils/searchPostFilterCache'); const searchInFlightRegistry = require('../utils/searchInFlightRegistry'); const searchPostFilterWorkQueue = require('../utils/searchPostFilterWorkQueue'); @@ -18,81 +18,6 @@ function hasSearchPostFilters(postFilters) { return Boolean(postFilters?.cpeName || postFilters?.virtualMatchString || postFilters?.virtualMatchStringRange); } -function getHitsContainer(resp) { - return resp?.data?.hits; -} - -function hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext) { - if ( - postFilters?.cpeName && - !cpe.hitAppliesToCpeName(hit, postFilters.cpeName, cpeEvaluationContext, { - requireVulnerable: postFilters.isVulnerable === true - }) - ) { - return false; - } - - if ( - postFilters?.virtualMatchString && - !cpe.hitMatchesVirtualMatchString(hit, postFilters.virtualMatchString, cpeEvaluationContext) - ) { - return false; - } - - if ( - postFilters?.virtualMatchStringRange && - !cpe.hitOverlapsVirtualMatchStringRange(hit, postFilters.virtualMatchStringRange, cpeEvaluationContext) - ) { - return false; - } - - return true; -} - -function buildEmptyFilteredResult(hitsContainer) { - return { - hitsContainer, - candidateCount: 0, - filteredHits: [] - }; -} - -function yieldToEventLoop() { - return new Promise((resolve) => setImmediate(resolve)); -} - -async function filterSearchHitsInChunks(resp, postFilters, signal, chunkSize) { - const hitsContainer = getHitsContainer(resp); - if (!Array.isArray(hitsContainer?.hits)) return buildEmptyFilteredResult(hitsContainer); - - const effectiveChunkSize = Math.max(1, chunkSize ?? searchProvider.getPostFilterEvaluationChunkSize()); - const cpeEvaluationContext = cpe.createEvaluationContext(); - const filteredHits = []; - - for (let start = 0; start < hitsContainer.hits.length; start += effectiveChunkSize) { - searchDeadline.throwIfAborted(signal); - const end = Math.min(start + effectiveChunkSize, hitsContainer.hits.length); - - for (let index = start; index < end; index += 1) { - const hit = hitsContainer.hits[index]; - if (hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext)) { - filteredHits.push(hit); - } - } - - if (end < hitsContainer.hits.length) { - await yieldToEventLoop(); - } - } - - searchDeadline.throwIfAborted(signal); - return { - hitsContainer, - candidateCount: hitsContainer.hits.length, - filteredHits - }; -} - function paginateHits(hits, responseOptions) { const from = responseOptions?.from ?? 0; const resultsPerPage = responseOptions?.resultsPerPage ?? hits.length; @@ -101,7 +26,7 @@ function paginateHits(hits, responseOptions) { } function buildPagedSearchResponse(resp, totalValue, pagedHits) { - const hitsContainer = getHitsContainer(resp); + const hitsContainer = resp?.data?.hits; return { ...resp, @@ -262,8 +187,7 @@ async function fetchFullHitsByCveIds(cveIds, metrics, signal) { const inFlight = searchInFlightRegistry.acquire( inFlightKey, async (operationSignal) => { - const { searchOptions, queryStringObjects } = searchQueryBuilder.buildCveIdsSearchRequest(cveIds); - const body = searchQueryBuilder.buildOpenSearchBody(searchOptions, queryStringObjects); + const body = searchQueryBuilder.buildCveIdsSearchBody(cveIds); const pageFetchStart = searchPerformance.markTime(); const pageData = await executeOpenSearch(body, 'page-fetch', operationSignal, metrics); metrics.pageFetchElapsedMs = searchPerformance.elapsedMs(pageFetchStart); @@ -297,7 +221,12 @@ async function buildPostFilterCandidateResult(candidateBody, postFilters, metric data: candidateData }; const postFilterStart = searchPerformance.markTime(); - const { candidateCount, filteredHits } = await filterSearchHitsInChunks(candidateResp, postFilters, signal); + const { candidateCount, filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( + candidateResp, + postFilters, + signal, + searchProvider.getPostFilterEvaluationChunkSize() + ); metrics.postFilterElapsedMs = searchPerformance.elapsedMs(postFilterStart); return { @@ -366,6 +295,10 @@ async function executeOptimizedPostFilterSearch( const candidateResult = await getPostFilterCandidateResult(cacheKey, candidateBody, postFilters, metrics, signal); searchDeadline.throwIfAborted(signal); const pageCveIds = paginateHits(candidateResult.filteredCveIds, responseOptions); + if (metrics.collectDetailedMetrics) { + metrics.cachedResultCveIdCount = candidateResult.filteredCveIds.length; + metrics.cacheState = searchPostFilterCache.getState(); + } metrics.candidateCount = candidateResult.candidateCount; metrics.filteredCount = candidateResult.filteredCount ?? candidateResult.filteredCveIds.length; metrics.returnedCount = pageCveIds.length; @@ -407,6 +340,7 @@ async function searchHandler(req, res, next) { const metrics = { requestId: req.requestId }; + metrics.collectDetailedMetrics = searchPerformance.beginSearchRequest(metrics, res); const handlerStart = searchPerformance.markTime(); const lifecycle = searchDeadline.createSearchRequestLifecycle( req, @@ -536,6 +470,7 @@ async function searchHandler(req, res, next) { } finally { lifecycle.complete(); metrics.totalElapsedMs = searchPerformance.elapsedMs(handlerStart); + searchPerformance.completeSearchRequest(metrics); searchPerformance.logSearchPerformance(metrics, searchOptions, responseOptions, postFilters); } } @@ -604,7 +539,6 @@ async function webSearchHandler(req, res, next) { module.exports = { buildWebSearchSort, - filterSearchHitsInChunks, SEARCH: searchHandler, WEB_SEARCH: webSearchHandler }; diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 5fdaf3c..cd3be86 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -57,7 +57,9 @@ jest.mock('../utils/searchProvider', () => ({ })); const controller = require('./searchController.js'); +const searchCandidateProcessor = require('../utils/searchCandidateProcessor'); const searchInFlightRegistry = require('../utils/searchInFlightRegistry'); +const searchPerformance = require('../utils/searchPerformance'); const searchPostFilterCache = require('../utils/searchPostFilterCache'); function createResponse() { @@ -214,6 +216,7 @@ describe('Search controller SEARCH handler', () => { afterEach(() => { searchInFlightRegistry.clear(); + searchPerformance.resetForTests(); jest.restoreAllMocks(); }); @@ -582,14 +585,7 @@ describe('Search controller SEARCH handler', () => { ] } }, - sort: [ - { - 'cveMetadata.cveId.keyword': { - order: 'asc' - } - } - ], - track_total_hits: true, + track_total_hits: false, size: 1, from: 0 }, @@ -615,6 +611,24 @@ describe('Search controller SEARCH handler', () => { }); }); + test('chunked filtering rejects malformed candidate hit collections', async () => { + await expect( + searchCandidateProcessor.filterSearchHitsInChunks( + { + data: { + hits: {} + } + }, + {}, + undefined, + 100 + ) + ).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_response' + }); + }); + test('chunked filtering does not combine platform and version matches from different CPE entries', async () => { const candidate = cpeHit('CVE-2026-0250', { criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:macos:*:*:*:*:*', @@ -643,7 +657,7 @@ describe('Search controller SEARCH handler', () => { } ); - const { filteredHits } = await controller.filterSearchHitsInChunks( + const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(1, [candidate]), { cpeName: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:6.1.1:*:macos:*:*:*:*:*', @@ -664,7 +678,7 @@ describe('Search controller SEARCH handler', () => { versionEndExcluding: '6.1' }); - const { filteredHits } = await controller.filterSearchHitsInChunks( + const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(1, [candidate]), { cpeName: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:6.0.1:*:*:*:*:macos:*:*', @@ -688,7 +702,7 @@ describe('Search controller SEARCH handler', () => { versionEndExcluding: '6.02' }); - const exactResult = await controller.filterSearchHitsInChunks( + const exactResult = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(1, [exactCandidate]), { cpeName: 'cpe:2.3:o:linksys:re6250_firmware:1.0.4.1:*:*:*:*:*:*:*', @@ -697,7 +711,7 @@ describe('Search controller SEARCH handler', () => { undefined, 100 ); - const includedRangeResult = await controller.filterSearchHitsInChunks( + const includedRangeResult = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(1, [rangeCandidate]), { cpeName: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:6.1:*:*:*:*:*:*:*', @@ -706,7 +720,7 @@ describe('Search controller SEARCH handler', () => { undefined, 100 ); - const excludedRangeResult = await controller.filterSearchHitsInChunks( + const excludedRangeResult = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(1, [rangeCandidate]), { cpeName: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:6.2:*:*:*:*:*:*:*', @@ -743,7 +757,7 @@ describe('Search controller SEARCH handler', () => { }) ]; - const { filteredHits } = await controller.filterSearchHitsInChunks( + const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(candidates.length, candidates), { cpeName: 'cpe:2.3:a:microsoft:edge:-:*:*:*:*:android:*:*', @@ -810,10 +824,11 @@ describe('Search controller SEARCH handler', () => { } ]); expect(mockApiSearch.mock.calls[0][0]).toMatchObject({ - track_total_hits: true, + track_total_hits: false, size: 1, from: 0 }); + expect(mockApiSearch.mock.calls[0][0].sort).toBeUndefined(); expect(mockApiSearch.mock.calls[0][1]).toEqual( expect.objectContaining({ operation: 'page-fetch', @@ -882,10 +897,11 @@ describe('Search controller SEARCH handler', () => { } ]); expect(mockApiSearch.mock.calls[1][0]).toMatchObject({ - track_total_hits: true, + track_total_hits: false, size: 1, from: 0 }); + expect(mockApiSearch.mock.calls[1][0].sort).toBeUndefined(); expect(secondRes.status).toHaveBeenCalledWith(200); expect(secondRes.json).toHaveBeenCalledWith({ page: 1, @@ -1080,7 +1096,7 @@ describe('Search controller SEARCH handler', () => { }) ]; const abortController = new AbortController(); - const filtering = controller.filterSearchHitsInChunks( + const filtering = searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(2, hits), { virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809' @@ -1323,6 +1339,39 @@ describe('Search controller SEARCH handler', () => { } }); + test('SEARCH returns a sanitized 502 and does not cache candidates rejected during post-filter processing', async () => { + mockSearchRaw.mockResolvedValue( + rawResponse(1, [ + cpeHit('CVE-2024-0001', { + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + versionStartIncluding: '2.0', + versionEndExcluding: '1.0' + }) + ]) + ); + const req = { + searchParams: { + cpeName: 'cpe:2.3:a:vendor:product:1.5:*:*:*:*:*:*:*', + resultsPerPage: '1', + pageNumber: '1' + } + }; + + await controller.SEARCH(req, res, next); + const secondRes = createResponse(); + await controller.SEARCH(req, secondRes, next); + + expect(mockSearchRaw).toHaveBeenCalledTimes(2); + for (const response of [res, secondRes]) { + expect(response.status).toHaveBeenCalledWith(502); + expect(response.json).toHaveBeenCalledWith({ + error: 'UPSTREAM_SEARCH_ERROR', + message: 'Search backend returned an error. Please try again later.' + }); + } + expect(next).not.toHaveBeenCalled(); + }); + test('SEARCH logs redacted performance timing when enabled', async () => { mockShouldLogSearchPerformance.mockReturnValue(true); mockApiSearch.mockResolvedValue(advancedResponse(1, [{ _id: 'CVE-2024-0001' }])); @@ -1356,7 +1405,9 @@ describe('Search controller SEARCH handler', () => { postFilters: [], requestedFrom: 0, requestedSize: 1, - openSearchSize: 1 + openSearchSize: 1, + heapUsedBytes: expect.any(Number), + heapUsedDeltaBytes: expect.any(Number) }); expect(console.log.mock.calls[0][0]).not.toContain('remote code'); }); diff --git a/routes/index.js b/routes/index.js index 2d30dbc..966044f 100644 --- a/routes/index.js +++ b/routes/index.js @@ -7,11 +7,9 @@ const constants = require('../utils/constants'); const featureFlags = require('../utils/featureFlags'); const searchDeadline = require('../utils/searchDeadline'); const searchProvider = require('../utils/searchProvider'); -const searchRateLimiter = require('../utils/searchRateLimiter'); const runtimeLifecycle = require('../utils/runtimeLifecycle'); const error = new errors.errors(); -const limitSearchRequests = searchRateLimiter.createSearchRateLimiter(); const SEARCH_BODY_FIELDS = new Set(constants.SEARCH_QUERY_PARAMS); const SEARCH_STRING_BODY_FIELDS = constants.SEARCH_QUERY_PARAMS.filter( (field) => @@ -391,9 +389,8 @@ content: { } } } -*/ + */ requireSearchJsonContentType, - limitSearchRequests, rejectSearchQueryParams, body().custom(isSearchBodyObject).withMessage(error.searchBodyError()), body().custom(hasOnlySearchBodyFields), diff --git a/routes/index.test.js b/routes/index.test.js index 5aa1d56..b0693a3 100644 --- a/routes/index.test.js +++ b/routes/index.test.js @@ -23,13 +23,8 @@ jest.mock('../utils/featureFlags', () => ({ isWebSearchEnabled: jest.fn(() => true) })); -const mockLimitSearchRequests = jest.fn((req, res, next) => next()); const mockCheckOpenSearchReadiness = jest.fn(); -jest.mock('../utils/searchRateLimiter', () => ({ - createSearchRateLimiter: jest.fn(() => mockLimitSearchRequests) -})); - jest.mock('../utils/searchProvider', () => ({ checkOpenSearchReadiness: mockCheckOpenSearchReadiness, getOpenSearchReadinessTimeoutMs: jest.fn(() => 5000) @@ -58,7 +53,6 @@ describe('routes/index router', () => { jest.clearAllMocks(); featureFlags.isWebSearchEnabled.mockReturnValue(true); runtimeLifecycle.resetForTests(); - mockLimitSearchRequests.mockImplementation((req, res, next) => next()); mockCheckOpenSearchReadiness.mockResolvedValue(true); }); @@ -153,7 +147,6 @@ describe('routes/index router', () => { }); expect(response.status).toBe(200); - expect(mockLimitSearchRequests).toHaveBeenCalledTimes(1); expect(mw.validateParams).toHaveBeenCalledTimes(1); expect(controller.SEARCH).toHaveBeenCalledTimes(1); expect(response.body).toStrictEqual({ @@ -172,7 +165,6 @@ describe('routes/index router', () => { }); expect(response.status).toBe(200); - expect(mockLimitSearchRequests).toHaveBeenCalledTimes(1); expect(mw.validateParams).toHaveBeenCalledTimes(1); expect(controller.SEARCH).toHaveBeenCalledTimes(1); expect(response.body).toStrictEqual({ @@ -181,7 +173,7 @@ describe('routes/index router', () => { }); }); - test('POST /search rejects a non-empty body with an unsupported media type before rate limiting', async () => { + test('POST /search rejects a non-empty body with an unsupported media type', async () => { const response = await request(app, { method: 'POST', path: '/search', @@ -196,29 +188,6 @@ describe('routes/index router', () => { error: 'SEARCH_CONTENT_TYPE', message: 'The /search request body must use the application/json media type with UTF-8 encoding.' }); - expect(mockLimitSearchRequests).not.toHaveBeenCalled(); - expect(mw.validateParams).not.toHaveBeenCalled(); - expect(controller.SEARCH).not.toHaveBeenCalled(); - }); - - test('POST /search returns a rate-limit response before validation and controller work', async () => { - mockLimitSearchRequests.mockImplementation((req, res) => - res.status(429).json({ - error: 'SEARCH_RATE_LIMITED', - message: 'Too many search requests. Please retry later.' - }) - ); - - const response = await request(app, { - method: 'POST', - path: '/search', - body: { - keywordSearch: 'test' - } - }); - - expect(response.status).toBe(429); - expect(response.body.error).toBe('SEARCH_RATE_LIMITED'); expect(mw.validateParams).not.toHaveBeenCalled(); expect(controller.SEARCH).not.toHaveBeenCalled(); }); @@ -385,7 +354,6 @@ describe('routes/index router', () => { }); expect(response.status).toBe(200); - expect(mockLimitSearchRequests).not.toHaveBeenCalled(); expect(controller.WEB_SEARCH).toHaveBeenCalledTimes(1); expect(response.body).toStrictEqual({ route: 'webSearch', diff --git a/routes/swagger.js b/routes/swagger.js index ba704cc..674ae6b 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -31,7 +31,7 @@ const searchRequestProperties = { } ), cveIds: searchRequestProperty( - `Returns one or more CVE records identified by a comma-separated list of CVE IDs. Limited to ${constants.MAX_CVE_IDS} CVE IDs, ${constants.MAX_CVE_ID_LENGTH} characters per ID, and ${constants.SEARCH_STRING_LENGTH_LIMITS.cveIds} characters overall.`, + `Returns one or more CVE records identified by a comma-separated list of CVE IDs. Each ID must contain the CVE prefix, a four-digit year, and a ${constants.MIN_CVE_ID_SEQUENCE_DIGITS}-to-${constants.MAX_CVE_ID_SEQUENCE_DIGITS}-digit sequence number. The prefix is accepted without regard to case. Limited to ${constants.MAX_CVE_IDS} CVE IDs and ${constants.SEARCH_STRING_LENGTH_LIMITS.cveIds} characters overall.`, { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cveIds, @@ -126,7 +126,7 @@ const searchRequestProperties = { } ), cpeName: searchRequestProperty( - 'Returns CVE records whose CPE Match Criteria apply to the provided CPE 2.3 asset name. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches.', + "Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches.", { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cpeName, @@ -222,7 +222,7 @@ const searchRequestProperties = { } ), virtualMatchString: searchRequestProperty( - 'Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component.', + "Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component.", { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.virtualMatchString, diff --git a/server.js b/server.js index 1340374..820510a 100644 --- a/server.js +++ b/server.js @@ -1,21 +1,24 @@ const dotenv = require('dotenv'); + +dotenv.config(); + const cors = require('cors'); const express = require('express'); const swaggerUi = require('swagger-ui-express'); const openApiSpecification = require('./api-docs/openapi.json'); const http = require('http'); const logger = require('morgan'); -const router = require('./routes/index'); const constants = require('./utils/constants'); const errors = require('./utils/errors'); const requestLogging = require('./utils/requestLogging'); +const runtimeConfig = require('./utils/runtimeConfig'); const runtimeLifecycle = require('./utils/runtimeLifecycle'); const searchProvider = require('./utils/searchProvider'); - -dotenv.load(); +const searchRateLimiter = require('./utils/searchRateLimiter'); const error = new errors.errors(); const SHUTDOWN_PROBE_PATHS = new Set(['/api/health-check', '/api/ready-check']); +let router; const swaggerSetupOptions = { swaggerOptions: { @@ -30,7 +33,7 @@ const swaggerSetupOptions = { }; function createRequestLogger() { - if (process.env.NODE_ENV === 'test') { + if (runtimeConfig.getRuntimeConfig().server.nodeEnvironment === 'test') { return (_req, _res, next) => next(); } @@ -94,7 +97,16 @@ function handleUnexpectedError(err, req, res, next) { return res.status(500).json(error.internalServerError()); } +function getRouter() { + if (!router) { + router = require('./routes/index'); + } + + return router; +} + function createApp() { + runtimeConfig.validateRuntimeConfig(); const app = express(); app.use(requestLogging.assignRequestId); @@ -102,12 +114,13 @@ function createApp() { app.use(cors()); app.use(runtimeLifecycle.trackRequest); app.use(rejectRequestsDuringShutdown); + app.post('/search', searchRateLimiter.createSearchRateLimiter()); app.use(express.json({ limit: constants.JSON_REQUEST_BODY_LIMIT_BYTES })); app.get('/api-docs/openapi.json', (req, res) => res.json(openApiSpecification)); app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(openApiSpecification, swaggerSetupOptions)); - app.use('/', router); + app.use('/', getRouter()); app.use(handleRequestError); app.use(handleUnexpectedError); @@ -115,20 +128,69 @@ function createApp() { } function getRuntimeConfig(port) { + const config = runtimeConfig.validateRuntimeConfig(); + const effectivePort = resolveListenPort(port, config.server.port); + return { - baseURL: process.env.BASE_URL || `http://127.0.0.1:${port}` + baseURL: config.server.baseURL || `http://127.0.0.1:${effectivePort}` }; } -function startServer(port = process.env.PORT || 3000) { - const app = createApp(); - const config = getRuntimeConfig(port); +function resolveListenPort(port, defaultPort) { + if (port === undefined) return defaultPort; + + const parsed = Number(port); + if (!Number.isInteger(parsed) || parsed < 0 || parsed > runtimeConfig.RUNTIME_CONFIG_LIMITS.port.max) { + throw new runtimeConfig.RuntimeConfigurationError( + 'PORT', + `must be between 0 and ${runtimeConfig.RUNTIME_CONFIG_LIMITS.port.max}` + ); + } + + return parsed; +} - return http.createServer(app).listen(port, () => { - console.log(`Listening on ${config.baseURL}`); +function startServer(port) { + const config = runtimeConfig.validateRuntimeConfig(); + const effectivePort = resolveListenPort(port, config.server.port); + const app = createApp(); + const server = configureHttpServer(http.createServer(app), config.runtime.http); + + return server.listen(effectivePort, () => { + const sanitizedConfig = runtimeConfig.sanitizeRuntimeConfig(config); + logRuntimeEvent(console.log, { + event: 'runtime.lifecycle', + outcome: 'started', + ...sanitizedConfig, + server: { + ...sanitizedConfig.server, + port: effectivePort + } + }); }); } +function configureHttpServer(server, httpConfig) { + server.headersTimeout = httpConfig.headersTimeoutMs; + server.requestTimeout = httpConfig.requestTimeoutMs; + server.keepAliveTimeout = httpConfig.keepAliveTimeoutMs; + + return server; +} + +function handleStartupError(err, log = console.error) { + const configurationError = runtimeConfig.isRuntimeConfigurationError(err); + const event = { + event: 'runtime.lifecycle', + outcome: 'startup_failed', + errorCode: configurationError ? err.code : 'STARTUP_ERROR' + }; + if (configurationError) event.configPath = err.configPath; + + logRuntimeEvent(log, event); + return event; +} + function logRuntimeEvent(log, event) { log(JSON.stringify(event)); } @@ -257,17 +319,24 @@ function installShutdownHandlers(server, { processRef = process, shutdownOptions } if (require.main === module) { - const server = startServer(); - installShutdownHandlers(server); + try { + const server = startServer(); + installShutdownHandlers(server); + } catch (err) { + handleStartupError(err); + process.exitCode = 1; + } } module.exports = { + configureHttpServer, createApp, createRequestLogger, handleUnexpectedError, installShutdownHandlers, getRedactedRequestPath: requestLogging.getRedactedRequestPath, getRuntimeConfig, + handleStartupError, rejectRequestsDuringShutdown, shutdownServer, startServer, diff --git a/server.test.js b/server.test.js index 61248a8..ba27bfe 100644 --- a/server.test.js +++ b/server.test.js @@ -18,9 +18,14 @@ const mockWebSearch = jest.fn((req, res) => ); const mockMorganMiddleware = jest.fn((req, res, next) => next()); const mockMorgan = jest.fn(() => mockMorganMiddleware); +const mockLimitSearchRequests = jest.fn((req, res, next) => next()); +const mockCreateSearchRateLimiter = jest.fn(() => mockLimitSearchRequests); mockMorgan.token = jest.fn(); jest.mock('morgan', () => mockMorgan); +jest.mock('./utils/searchRateLimiter', () => ({ + createSearchRateLimiter: mockCreateSearchRateLimiter +})); jest.mock('./controllers/middleware', () => ({ validateParams: mockValidateParams @@ -32,16 +37,19 @@ jest.mock('./controllers/searchController', () => ({ })); const { + configureHttpServer, createApp, createRequestLogger, getRedactedRequestPath, getRuntimeConfig, + handleStartupError, handleUnexpectedError, installShutdownHandlers, shutdownServer, startServer, swaggerSetupOptions } = require('./server'); +const runtimeConfig = require('./utils/runtimeConfig'); const runtimeLifecycle = require('./utils/runtimeLifecycle'); function restoreEnv(name, value) { @@ -58,11 +66,16 @@ describe('server application bootstrap', () => { const originalNodeEnv = process.env.NODE_ENV; const originalPort = process.env.PORT; + beforeEach(() => { + mockLimitSearchRequests.mockImplementation((req, res, next) => next()); + }); + afterEach(() => { restoreEnv('BASE_URL', originalBaseUrl); restoreEnv('NODE_ENV', originalNodeEnv); restoreEnv('PORT', originalPort); runtimeLifecycle.resetForTests(); + runtimeConfig.resetForTests(); jest.useRealTimers(); jest.restoreAllMocks(); jest.clearAllMocks(); @@ -124,6 +137,7 @@ describe('server application bootstrap', () => { expect(response.status).toBe(200); expect(mockWebSearch).toHaveBeenCalledTimes(1); + expect(mockLimitSearchRequests).not.toHaveBeenCalled(); expect(response.body).toStrictEqual({ route: 'webSearch', body @@ -148,6 +162,29 @@ describe('server application bootstrap', () => { route: 'search', params: body }); + expect(mockLimitSearchRequests).toHaveBeenCalledTimes(1); + }); + + test('createApp applies the /search limiter before parsing its JSON body', async () => { + mockLimitSearchRequests.mockImplementation((req, res) => + res.status(429).json({ + error: 'SEARCH_RATE_LIMITED', + message: 'Too many search requests. Please retry later.' + }) + ); + + const response = await request(createApp(), { + method: 'POST', + path: '/search', + rawBody: '{"invalid":', + headers: { + 'Content-Type': 'application/json' + } + }); + + expect(response.status).toBe(429); + expect(response.body.error).toBe('SEARCH_RATE_LIMITED'); + expect(mockSearch).not.toHaveBeenCalled(); }); test('createApp rejects non-probe requests after shutdown starts', async () => { @@ -316,6 +353,28 @@ describe('server application bootstrap', () => { expect(startServer()).toBe(fakeServer); expect(createServer).toHaveBeenCalledWith(expect.any(Function)); expect(fakeServer.listen).toHaveBeenCalledWith(3000, expect.any(Function)); + expect(fakeServer).toMatchObject({ + headersTimeout: constants.HTTP_HEADERS_TIMEOUT_MS, + requestTimeout: constants.HTTP_REQUEST_TIMEOUT_MS, + keepAliveTimeout: constants.HTTP_KEEP_ALIVE_TIMEOUT_MS + }); + }); + + test('configureHttpServer applies explicit transport deadlines', () => { + const fakeServer = {}; + + expect( + configureHttpServer(fakeServer, { + headersTimeoutMs: 9000, + requestTimeoutMs: 12000, + keepAliveTimeoutMs: 4000 + }) + ).toBe(fakeServer); + expect(fakeServer).toStrictEqual({ + headersTimeout: 9000, + requestTimeout: 12000, + keepAliveTimeout: 4000 + }); }); test('startServer passes through an explicit port', () => { @@ -324,7 +383,40 @@ describe('server application bootstrap', () => { jest.spyOn(http, 'createServer').mockReturnValue(fakeServer); expect(startServer('4567')).toBe(fakeServer); - expect(fakeServer.listen).toHaveBeenCalledWith('4567', expect.any(Function)); + expect(fakeServer.listen).toHaveBeenCalledWith(4567, expect.any(Function)); + }); + + test('startServer rejects invalid configuration before creating an HTTP server', () => { + process.env.PORT = 'invalid'; + runtimeConfig.resetForTests(); + const createServer = jest.spyOn(http, 'createServer'); + + expect(() => startServer()).toThrow('PORT must be an integer'); + expect(createServer).not.toHaveBeenCalled(); + }); + + test('handleStartupError logs only a sanitized configuration failure', () => { + const log = jest.fn(); + const thrownError = new runtimeConfig.RuntimeConfigurationError( + 'search.providerEndpoint', + 'must be configured' + ); + + expect(handleStartupError(thrownError, log)).toStrictEqual({ + event: 'runtime.lifecycle', + outcome: 'startup_failed', + errorCode: 'RUNTIME_CONFIGURATION', + configPath: 'search.providerEndpoint' + }); + expect(log).toHaveBeenCalledWith( + JSON.stringify({ + event: 'runtime.lifecycle', + outcome: 'startup_failed', + errorCode: 'RUNTIME_CONFIGURATION', + configPath: 'search.providerEndpoint' + }) + ); + expect(JSON.stringify(log.mock.calls)).not.toContain('must be configured'); }); test('shutdownServer marks readiness unavailable and drains the HTTP server', async () => { diff --git a/utils/constants.js b/utils/constants.js index 0093369..b422d8a 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -5,7 +5,8 @@ const MAX_CVSS_METRICS_LENGTH = 256; const MAX_CVSS_METRIC_TOKENS = 32; const MAX_CVSS_METRIC_TOKEN_LENGTH = 32; const MAX_CVE_IDS = 100; -const MAX_CVE_ID_LENGTH = 64; +const MIN_CVE_ID_SEQUENCE_DIGITS = 4; +const MAX_CVE_ID_SEQUENCE_DIGITS = 19; const MAX_KEYWORD_SEARCH_TOKENS = 64; const JSON_REQUEST_BODY_LIMIT_KB = 100; const JSON_REQUEST_BODY_LIMIT_BYTES = JSON_REQUEST_BODY_LIMIT_KB * 1024; @@ -29,6 +30,9 @@ const SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS = 20 * 1000; const SEARCH_REQUEST_DEADLINE_MS = 30 * 1000; const SEARCH_READINESS_TIMEOUT_MS = 5 * 1000; const RUNTIME_SHUTDOWN_TIMEOUT_MS = 35 * 1000; +const HTTP_HEADERS_TIMEOUT_MS = 10 * 1000; +const HTTP_REQUEST_TIMEOUT_MS = 15 * 1000; +const HTTP_KEEP_ALIVE_TIMEOUT_MS = 5 * 1000; const rejectedCveStates = ['REJECT', 'REJECTED', 'Rejected']; const rejectedCveStateFields = ['cveMetadata.state', 'containers.cna.x_legacyV4Record.CVE_data_meta.STATE']; @@ -54,7 +58,7 @@ const cpeMatchFields = { /** * keywordSearch: list of terms * keywordExactMatch: list of terms, must be an exact match to terms in a description - * cveIds: comma-separated list of CVE-year-XXXX identifiers + * cveIds: comma-separated list of CVE-year-sequence identifiers * noRejected: no-value flag that excludes rejected CVE records * cweId: CWE-XXX * cvssV2Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL @@ -132,7 +136,8 @@ module.exports = { MAX_CVSS_METRIC_TOKENS, MAX_CVSS_METRIC_TOKEN_LENGTH, MAX_CVE_IDS, - MAX_CVE_ID_LENGTH, + MIN_CVE_ID_SEQUENCE_DIGITS, + MAX_CVE_ID_SEQUENCE_DIGITS, MAX_KEYWORD_SEARCH_TOKENS, JSON_REQUEST_BODY_LIMIT_KB, JSON_REQUEST_BODY_LIMIT_BYTES, @@ -148,6 +153,9 @@ module.exports = { SEARCH_REQUEST_DEADLINE_MS, SEARCH_READINESS_TIMEOUT_MS, RUNTIME_SHUTDOWN_TIMEOUT_MS, + HTTP_HEADERS_TIMEOUT_MS, + HTTP_REQUEST_TIMEOUT_MS, + HTTP_KEEP_ALIVE_TIMEOUT_MS, RESULTS_PER_PAGE, REJECTED_CVE_STATES: rejectedCveStates, CPE_VERSION_RANGE_TYPES: cpeVersionRangeTypes, diff --git a/utils/cpe.js b/utils/cpe.js index 97f0620..58e6741 100644 --- a/utils/cpe.js +++ b/utils/cpe.js @@ -3,6 +3,12 @@ const CPE_COMPONENT_COUNT = 11; const CPE_NOT_APPLICABLE = '-'; const REQUIRED_CPE_COMPONENT_INDEXES = [0, 1, 2, 3]; const VALID_CPE_PARTS = new Set(['a', 'h', 'o']); +const CPE_RANGE_FIELDS = [ + 'versionStartIncluding', + 'versionStartExcluding', + 'versionEndIncluding', + 'versionEndExcluding' +]; const VERSION_QUALIFIERS = ['alpha', 'beta', 'milestone', 'rc', 'snapshot', '', 'sp']; const VERSION_QUALIFIER_ALIASES = { a: 'alpha', @@ -125,6 +131,9 @@ function parseVirtualMatchString(input) { if (rawComponents.length > CPE_COMPONENT_COUNT) { return { valid: false, reason: 'component_count' }; } + if (rawComponents.some((component) => component.trim() === '')) { + return { valid: false, reason: 'empty_component' }; + } if (rawComponents.length && !isValidCpePart(rawComponents[0], { allowLogicalValues: true })) { return { valid: false, reason: 'part' }; } @@ -391,6 +400,35 @@ function lowerBoundDoesNotExceedUpperBound(lower, upper) { return lower.inclusive && upper.inclusive; } +function isValidCpeMatch(cpeMatch, context) { + if (!cpeMatch || typeof cpeMatch !== 'object' || Array.isArray(cpeMatch)) return false; + if (typeof cpeMatch.criteria !== 'string' || cpeMatch.criteria.trim() === '') return false; + if (!getParsedCpeMatchCriteria(cpeMatch.criteria, context).valid) return false; + if (typeof cpeMatch.vulnerable !== 'boolean') return false; + if ( + !CPE_RANGE_FIELDS.every( + (field) => + cpeMatch[field] === undefined || (typeof cpeMatch[field] === 'string' && cpeMatch[field].trim() !== '') + ) + ) { + return false; + } + + if (hasBoundary(cpeMatch.versionStartIncluding) && hasBoundary(cpeMatch.versionStartExcluding)) return false; + if (hasBoundary(cpeMatch.versionEndIncluding) && hasBoundary(cpeMatch.versionEndExcluding)) return false; + + const range = buildVersionRange({ + start: hasBoundary(cpeMatch.versionStartIncluding) + ? cpeMatch.versionStartIncluding + : cpeMatch.versionStartExcluding, + startType: hasBoundary(cpeMatch.versionStartIncluding) ? 'including' : 'excluding', + end: hasBoundary(cpeMatch.versionEndIncluding) ? cpeMatch.versionEndIncluding : cpeMatch.versionEndExcluding, + endType: hasBoundary(cpeMatch.versionEndIncluding) ? 'including' : 'excluding' + }); + + return lowerBoundDoesNotExceedUpperBound(range.start, range.end); +} + function versionRangesOverlap(a, b) { return lowerBoundDoesNotExceedUpperBound(a.start, b.end) && lowerBoundDoesNotExceedUpperBound(b.start, a.end); } @@ -497,16 +535,6 @@ function cpeMatchMatchesVirtualMatchString(cpeMatch, virtualMatchString, context return true; } -function collectCpeMatchesFromNode(node, matches) { - if (!node || typeof node !== 'object') return; - - const cpeMatches = Array.isArray(node.cpeMatch) ? node.cpeMatch : []; - matches.push(...cpeMatches); - - const childNodes = Array.isArray(node.nodes) ? node.nodes : []; - childNodes.forEach((child) => collectCpeMatchesFromNode(child, matches)); -} - function getCpeMatchesFromHit(hit) { const cpeApplicability = hit?._source?.containers?.cna?.cpeApplicability; const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; @@ -514,7 +542,11 @@ function getCpeMatchesFromHit(hit) { applicabilityItems.forEach((applicability) => { const rootNodes = Array.isArray(applicability?.nodes) ? applicability.nodes : []; - rootNodes.forEach((node) => collectCpeMatchesFromNode(node, matches)); + rootNodes.forEach((node) => { + if (Array.isArray(node?.cpeMatch)) { + matches.push(...node.cpeMatch); + } + }); }); return matches; @@ -552,6 +584,7 @@ module.exports = { hitOverlapsVirtualMatchStringRange, hasConcreteComponentsAfter, hasConcreteLeadingComponents, + isValidCpeMatch, isValidCpeName, parseCpeMatchCriteria, parseCpeName, diff --git a/utils/cpe.test.js b/utils/cpe.test.js index caa429e..6068cac 100644 --- a/utils/cpe.test.js +++ b/utils/cpe.test.js @@ -84,6 +84,16 @@ describe('CPE helpers', () => { expect(parsed.components).toStrictEqual(['*', '*', '*', '*', '*', '*', '*', '*', '*', '*', '*']); }); + test.each(['cpe:2.3:', 'cpe:2.3:o:', 'cpe:2.3:o::product', 'cpe:2.3:o:vendor::1.0', 'cpe:2.3:o:vendor:product: '])( + 'parseVirtualMatchString rejects explicit empty component in %s', + (value) => { + expect(cpe.parseVirtualMatchString(value)).toStrictEqual({ + valid: false, + reason: 'empty_component' + }); + } + ); + test('parseVirtualMatchString rejects an unsupported concrete part value', () => { expect(cpe.parseVirtualMatchString('cpe:2.3:x:vendor:product')).toStrictEqual({ valid: false, @@ -624,7 +634,7 @@ describe('CPE helpers', () => { ).toBe(true); }); - test('hitAppliesToCpeName searches nested cpeApplicability nodes', () => { + test('hitAppliesToCpeName evaluates direct entries independently of node operator and negate values', () => { const hit = { _source: { containers: { @@ -632,15 +642,18 @@ describe('CPE helpers', () => { cpeApplicability: { nodes: [ { - nodes: [ + operator: 'AND', + negate: true, + cpeMatch: [ { - cpeMatch: [ - { - criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', - versionStartIncluding: '2.9', - versionEndIncluding: '2.10' - } - ] + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true, + versionStartIncluding: '2.9', + versionEndIncluding: '2.10' + }, + { + criteria: 'cpe:2.3:a:other:component:*:*:*:*:*:*:*:*', + vulnerable: true } ] } @@ -651,11 +664,15 @@ describe('CPE helpers', () => { } }; - expect(cpe.hitAppliesToCpeName(hit, 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*')).toBe(true); + expect( + cpe.hitAppliesToCpeName(hit, 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*', undefined, { + requireVulnerable: true + }) + ).toBe(true); expect(cpe.hitAppliesToCpeName(hit, 'cpe:2.3:a:vendor:product:2.11:*:*:*:*:*:*:*')).toBe(false); }); - test('hitOverlapsVirtualMatchStringRange searches nested cpeApplicability nodes', () => { + test('hitOverlapsVirtualMatchStringRange evaluates direct entries independently of node metadata', () => { const hit = { _source: { containers: { @@ -663,15 +680,18 @@ describe('CPE helpers', () => { cpeApplicability: { nodes: [ { - nodes: [ + operator: 'OR', + negate: true, + cpeMatch: [ { - cpeMatch: [ - { - criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', - versionStartIncluding: '2.9', - versionEndIncluding: '2.10' - } - ] + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true, + versionStartIncluding: '2.9', + versionEndIncluding: '2.10' + }, + { + criteria: 'cpe:2.3:a:other:component:*:*:*:*:*:*:*:*', + vulnerable: true } ] } @@ -701,4 +721,37 @@ describe('CPE helpers', () => { }) ).toBe(false); }); + + test('getCpeMatchesFromHit ignores nested-only applicability entries', () => { + const hit = { + _source: { + containers: { + cna: { + cpeApplicability: { + nodes: [ + { + operator: 'OR', + negate: false, + nodes: [ + { + cpeMatch: [ + { + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true + } + ] + } + ] + } + ] + } + } + } + } + }; + + expect(cpe.getCpeMatchesFromHit(hit)).toStrictEqual([]); + expect(cpe.hitAppliesToCpeName(hit, 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*')).toBe(false); + expect(cpe.hitMatchesVirtualMatchString(hit, 'cpe:2.3:a:vendor:product')).toBe(false); + }); }); diff --git a/utils/cveId.js b/utils/cveId.js new file mode 100644 index 0000000..759d0ef --- /dev/null +++ b/utils/cveId.js @@ -0,0 +1,18 @@ +const constants = require('./constants'); + +const CVE_ID_PATTERN = new RegExp( + `^CVE-\\d{4}-\\d{${constants.MIN_CVE_ID_SEQUENCE_DIGITS},${constants.MAX_CVE_ID_SEQUENCE_DIGITS}}$` +); + +function normalizeCveId(value) { + return typeof value === 'string' ? value.trim().toUpperCase() : ''; +} + +function isValidCveId(value) { + return typeof value === 'string' && CVE_ID_PATTERN.test(value); +} + +module.exports = { + isValidCveId, + normalizeCveId +}; diff --git a/utils/cveId.test.js b/utils/cveId.test.js new file mode 100644 index 0000000..2710819 --- /dev/null +++ b/utils/cveId.test.js @@ -0,0 +1,28 @@ +const constants = require('./constants'); +const cveId = require('./cveId'); + +describe('CVE ID helpers', () => { + test.each([ + 'CVE-2024-0001', + `CVE-2024-${'1'.repeat(constants.MIN_CVE_ID_SEQUENCE_DIGITS)}`, + `CVE-2024-${'1'.repeat(constants.MAX_CVE_ID_SEQUENCE_DIGITS)}` + ])('isValidCveId accepts a valid identifier %s', (value) => { + expect(cveId.isValidCveId(value)).toBe(true); + }); + + test.each([ + 'cve-2024-0001', + 'CVE-24-0001', + `CVE-2024-${'1'.repeat(constants.MIN_CVE_ID_SEQUENCE_DIGITS - 1)}`, + `CVE-2024-${'1'.repeat(constants.MAX_CVE_ID_SEQUENCE_DIGITS + 1)}`, + 'CVE-2024-12A4', + ' CVE-2024-0001 ' + ])('isValidCveId rejects a malformed identifier %s', (value) => { + expect(cveId.isValidCveId(value)).toBe(false); + }); + + test('normalizeCveId trims and uppercases request input', () => { + expect(cveId.normalizeCveId(' cve-2024-0001 ')).toBe('CVE-2024-0001'); + expect(cveId.normalizeCveId(undefined)).toBe(''); + }); +}); diff --git a/utils/errors.js b/utils/errors.js index d001e55..9b442d8 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -110,7 +110,10 @@ class errors { } cveIdsFormatError() { - return makeError('CVE_IDS_FORMAT', 'The cveIds parameter must be a comma-separated list of CVE IDs.'); + return makeError( + 'CVE_IDS_FORMAT', + `The cveIds parameter must be a comma-separated list of CVE IDs with a four-digit year and a ${constants.MIN_CVE_ID_SEQUENCE_DIGITS}-to-${constants.MAX_CVE_ID_SEQUENCE_DIGITS}-digit sequence number.` + ); } cveIdsLimitError() { @@ -165,13 +168,6 @@ class errors { return makeError('SEARCH_FIELD_TOKEN_LIMIT', `The ${field} field must contain ${maxTokens} or fewer terms.`); } - cveIdLengthError() { - return makeError( - 'CVE_ID_LENGTH', - `Each CVE ID in the cveIds field must be ${constants.MAX_CVE_ID_LENGTH} characters or fewer.` - ); - } - cvssV2SeverityError(severity) { return makeError( 'INVALID_CVSSV2_SEVERITY', diff --git a/utils/errors.test.js b/utils/errors.test.js index b40efc8..582098e 100644 --- a/utils/errors.test.js +++ b/utils/errors.test.js @@ -156,7 +156,7 @@ describe('error response helpers', () => { [], { error: 'CVE_IDS_FORMAT', - message: 'The cveIds parameter must be a comma-separated list of CVE IDs.' + message: `The cveIds parameter must be a comma-separated list of CVE IDs with a four-digit year and a ${constants.MIN_CVE_ID_SEQUENCE_DIGITS}-to-${constants.MAX_CVE_ID_SEQUENCE_DIGITS}-digit sequence number.` } ], [ diff --git a/utils/featureFlags.js b/utils/featureFlags.js index 61ff67f..29626de 100644 --- a/utils/featureFlags.js +++ b/utils/featureFlags.js @@ -1,25 +1,14 @@ -const { AppConfig } = require('cve-core'); - -const DEFAULT_WEB_SEARCH_ENABLED = true; - -function normalizeBoolean(value, defaultValue) { - if (value === undefined || value === null || value === '') return defaultValue; - if (typeof value === 'boolean') return value; - if (typeof value === 'number') return value !== 0; - - const normalized = String(value).trim().toLowerCase(); - if (['true', '1', 'yes', 'y', 'on'].includes(normalized)) return true; - if (['false', '0', 'no', 'n', 'off'].includes(normalized)) return false; - - return defaultValue; -} +const runtimeConfig = require('./runtimeConfig'); function isWebSearchEnabled() { - if (!AppConfig.has('features.webSearchEnabled')) return DEFAULT_WEB_SEARCH_ENABLED; + return runtimeConfig.getRuntimeConfig().features.webSearchEnabled; +} - return normalizeBoolean(AppConfig.get('features.webSearchEnabled'), false); +function resetForTests() { + runtimeConfig.resetForTests(); } module.exports = { - isWebSearchEnabled + isWebSearchEnabled, + resetForTests }; diff --git a/utils/featureFlags.test.js b/utils/featureFlags.test.js index 3d47d75..8114b87 100644 --- a/utils/featureFlags.test.js +++ b/utils/featureFlags.test.js @@ -12,6 +12,7 @@ const featureFlags = require('./featureFlags'); describe('featureFlags', () => { beforeEach(() => { + featureFlags.resetForTests(); mockAppConfigHas.mockReset(); mockAppConfigGet.mockReset(); mockAppConfigHas.mockReturnValue(false); @@ -24,21 +25,19 @@ describe('featureFlags', () => { test.each([ ['TRUE', true], ['false', false], - ['1', true], - ['0', false], [true, true], [false, false] ])('normalizes a configured webSearch value of %p', (configuredValue, expected) => { - mockAppConfigHas.mockReturnValue(true); + mockAppConfigHas.mockImplementation((path) => path === 'features.webSearchEnabled'); mockAppConfigGet.mockReturnValue(configuredValue); expect(featureFlags.isWebSearchEnabled()).toBe(expected); }); - test('fails closed when an explicitly configured webSearch value is invalid', () => { - mockAppConfigHas.mockReturnValue(true); - mockAppConfigGet.mockReturnValue('invalid'); + test.each(['invalid', '1', '0'])('rejects an explicitly configured webSearch value of %p', (value) => { + mockAppConfigHas.mockImplementation((path) => path === 'features.webSearchEnabled'); + mockAppConfigGet.mockReturnValue(value); - expect(featureFlags.isWebSearchEnabled()).toBe(false); + expect(() => featureFlags.isWebSearchEnabled()).toThrow('features.webSearchEnabled'); }); }); diff --git a/utils/requestLogging.js b/utils/requestLogging.js index 57e7248..e47f60c 100644 --- a/utils/requestLogging.js +++ b/utils/requestLogging.js @@ -1,49 +1,11 @@ const { randomUUID } = require('crypto'); -const { AppConfig } = require('cve-core'); +const runtimeConfig = require('./runtimeConfig'); -const DEFAULT_SUCCESS_SAMPLE_RATE = 0.1; -const DEFAULT_LOG_HEALTH_CHECKS = false; const REQUEST_ID_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/; const HEALTH_CHECK_PATHS = new Set(['/api/health-check', '/api/ready-check']); -let requestLoggingConfig; - -function normalizeBoolean(value, defaultValue = false) { - if (value === undefined || value === null || value === '') return defaultValue; - if (typeof value === 'boolean') return value; - if (typeof value === 'number') return value !== 0; - - const normalized = String(value).trim().toLowerCase(); - if (['true', '1', 'yes', 'y', 'on'].includes(normalized)) return true; - if (['false', '0', 'no', 'n', 'off'].includes(normalized)) return false; - - return defaultValue; -} - -function readSuccessSampleRate() { - if (!AppConfig.has('requestLogging.successSampleRate')) return DEFAULT_SUCCESS_SAMPLE_RATE; - - const value = Number(AppConfig.get('requestLogging.successSampleRate')); - if (Number.isFinite(value) && value >= 0 && value <= 1) return value; - - return DEFAULT_SUCCESS_SAMPLE_RATE; -} - -function readLogHealthChecks() { - if (!AppConfig.has('requestLogging.healthChecks')) return DEFAULT_LOG_HEALTH_CHECKS; - - return normalizeBoolean(AppConfig.get('requestLogging.healthChecks'), DEFAULT_LOG_HEALTH_CHECKS); -} - function getRequestLoggingConfig() { - if (!requestLoggingConfig) { - requestLoggingConfig = Object.freeze({ - successSampleRate: readSuccessSampleRate(), - healthChecks: readLogHealthChecks() - }); - } - - return requestLoggingConfig; + return runtimeConfig.getRuntimeConfig().requestLogging; } function getSuccessSampleRate() { @@ -83,7 +45,7 @@ function shouldSkipAccessLog(req, res, random = Math.random) { } function resetForTests() { - requestLoggingConfig = undefined; + runtimeConfig.resetForTests(); } module.exports = { @@ -91,7 +53,6 @@ module.exports = { getRedactedRequestPath, getRequestLoggingConfig, getSuccessSampleRate, - normalizeBoolean, resetForTests, resolveRequestId, shouldLogHealthChecks, diff --git a/utils/requestLogging.test.js b/utils/requestLogging.test.js index 1cae11e..77badb8 100644 --- a/utils/requestLogging.test.js +++ b/utils/requestLogging.test.js @@ -120,12 +120,14 @@ describe('requestLogging', () => { expect(requestLogging.shouldSkipAccessLog(req, { statusCode: 500 }, () => 1)).toBe(false); }); - test('request logging settings fall back when config values are invalid', () => { - mockAppConfigHas.mockReturnValue(true); + test.each([ + ['requestLogging.successSampleRate', () => requestLogging.getSuccessSampleRate()], + ['requestLogging.healthChecks', () => requestLogging.shouldLogHealthChecks()] + ])('rejects an invalid explicit %s value', (path, readValue) => { + mockAppConfigHas.mockImplementation((candidatePath) => candidatePath === path); mockAppConfigGet.mockReturnValue('invalid'); - expect(requestLogging.getSuccessSampleRate()).toBe(0.1); - expect(requestLogging.shouldLogHealthChecks()).toBe(false); + expect(readValue).toThrow(path); }); test('successful health checks can be enabled through config', () => { @@ -151,6 +153,7 @@ describe('requestLogging', () => { expect(first).toBe(second); expect(Object.isFrozen(first)).toBe(true); - expect(mockAppConfigHas).toHaveBeenCalledTimes(2); + expect(mockAppConfigHas).toHaveBeenCalledWith('requestLogging.successSampleRate'); + expect(mockAppConfigHas).toHaveBeenCalledWith('requestLogging.healthChecks'); }); }); diff --git a/utils/runtimeConfig.js b/utils/runtimeConfig.js new file mode 100644 index 0000000..9fc709f --- /dev/null +++ b/utils/runtimeConfig.js @@ -0,0 +1,430 @@ +const { AppConfig, SearchProviderSpec } = require('cve-core'); +const constants = require('./constants'); + +const DEFAULT_PORT = 3000; +const DEFAULT_REQUEST_LOG_SUCCESS_SAMPLE_RATE = 0.1; +const DEFAULT_REQUEST_LOG_HEALTH_CHECKS = false; +const DEFAULT_WEB_SEARCH_ENABLED = true; +const DEFAULT_SEARCH_RATE_LIMIT_ENABLED = false; +const DEFAULT_SEARCH_RATE_LIMIT_WINDOW_MS = 60 * 1000; +const DEFAULT_SEARCH_RATE_LIMIT_MAX_REQUESTS = 0; + +const RUNTIME_CONFIG_LIMITS = Object.freeze({ + port: Object.freeze({ min: 1, max: 65535 }), + postFilterCacheTtlMs: Object.freeze({ min: 0, max: 24 * 60 * 60 * 1000 }), + postFilterCacheMaxEntries: Object.freeze({ min: 0, max: 10000 }), + postFilterColdSearchConcurrency: Object.freeze({ min: 1, max: 100 }), + postFilterColdSearchMaxQueueDepth: Object.freeze({ min: 0, max: 10000 }), + postFilterColdSearchMaxWaitMs: Object.freeze({ min: 1, max: 5 * 60 * 1000 }), + inFlightMaxWaitersPerOperation: Object.freeze({ min: 1, max: 10000 }), + postFilterEvaluationChunkSize: Object.freeze({ min: 1, max: 10000 }), + openSearchRequestTimeoutMs: Object.freeze({ min: 1, max: 5 * 60 * 1000 }), + requestDeadlineMs: Object.freeze({ min: 1, max: 10 * 60 * 1000 }), + readinessTimeoutMs: Object.freeze({ min: 1, max: 60 * 1000 }), + shutdownTimeoutMs: Object.freeze({ min: 1, max: 15 * 60 * 1000 }), + httpHeadersTimeoutMs: Object.freeze({ min: 1, max: 60 * 1000 }), + httpRequestTimeoutMs: Object.freeze({ min: 1, max: 5 * 60 * 1000 }), + httpKeepAliveTimeoutMs: Object.freeze({ min: 1, max: 60 * 1000 }), + searchRateLimitWindowMs: Object.freeze({ min: 1, max: 24 * 60 * 60 * 1000 }), + searchRateLimitMaxRequests: Object.freeze({ min: 0, max: 10000000 }) +}); + +class RuntimeConfigurationError extends Error { + constructor(configPath, requirement) { + super(`${configPath} ${requirement}.`); + this.name = 'RuntimeConfigurationError'; + this.code = 'RUNTIME_CONFIGURATION'; + this.configPath = configPath; + } +} + +let resolvedRuntimeConfig; + +function deepFreeze(value) { + if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value; + + Object.values(value).forEach(deepFreeze); + return Object.freeze(value); +} + +function hasConfigValue(appConfig, path) { + return typeof appConfig?.has === 'function' && appConfig.has(path); +} + +function readConfigValue(appConfig, path, defaultValue) { + return hasConfigValue(appConfig, path) ? appConfig.get(path) : defaultValue; +} + +function parseBoolean(path, value) { + if (typeof value === 'boolean') return value; + + const normalized = typeof value === 'string' ? value.trim().toLowerCase() : ''; + if (normalized === 'true') return true; + if (normalized === 'false') return false; + + throw new RuntimeConfigurationError(path, 'must be true or false'); +} + +function parseInteger(path, value, limits) { + const parsed = Number(value); + if (!Number.isInteger(parsed)) { + throw new RuntimeConfigurationError(path, 'must be an integer'); + } + if (parsed < limits.min || parsed > limits.max) { + throw new RuntimeConfigurationError(path, `must be between ${limits.min} and ${limits.max}`); + } + + return parsed; +} + +function parseNumber(path, value, { min, max }) { + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + throw new RuntimeConfigurationError(path, 'must be a finite number'); + } + if (parsed < min || parsed > max) { + throw new RuntimeConfigurationError(path, `must be between ${min} and ${max}`); + } + + return parsed; +} + +function parseOptionalString(path, value, { maxLength } = {}) { + if (value === undefined || value === null || value === '') return ''; + if (typeof value !== 'string' || value.trim() === '') { + throw new RuntimeConfigurationError(path, 'must be a non-empty string'); + } + + const normalized = value.trim(); + if (maxLength && normalized.length > maxLength) { + throw new RuntimeConfigurationError(path, `must not exceed ${maxLength} characters`); + } + + return normalized; +} + +function parseHttpUrl(path, value, { required = false, allowCredentials = true } = {}) { + if (!value) { + if (required) throw new RuntimeConfigurationError(path, 'must be configured'); + return ''; + } + + let parsed; + try { + parsed = new URL(value); + } catch (_err) { + throw new RuntimeConfigurationError(path, 'must be an absolute HTTP or HTTPS URL'); + } + + if (!['http:', 'https:'].includes(parsed.protocol)) { + throw new RuntimeConfigurationError(path, 'must be an absolute HTTP or HTTPS URL'); + } + if (!allowCredentials && (parsed.username || parsed.password)) { + throw new RuntimeConfigurationError(path, 'must not contain credentials'); + } + + return value; +} + +function getDefaultSearchProviderSpec(searchProviderSpec) { + if (typeof searchProviderSpec?.getDefaultSearchProviderSpec !== 'function') return {}; + + return searchProviderSpec.getDefaultSearchProviderSpec() ?? {}; +} + +function validateRelationships(config) { + const search = config.search; + + if (search.readinessTimeoutMs > search.openSearchRequestTimeoutMs) { + throw new RuntimeConfigurationError( + 'search.readinessTimeoutMs', + 'must be less than or equal to search.openSearchRequestTimeoutMs' + ); + } + if (search.openSearchRequestTimeoutMs >= search.requestDeadlineMs) { + throw new RuntimeConfigurationError( + 'search.openSearchRequestTimeoutMs', + 'must be less than search.requestDeadlineMs' + ); + } + if (search.postFilterColdSearchMaxWaitMs >= search.requestDeadlineMs) { + throw new RuntimeConfigurationError( + 'search.postFilterColdSearchMaxWaitMs', + 'must be less than search.requestDeadlineMs' + ); + } + if (config.runtime.shutdownTimeoutMs <= search.requestDeadlineMs) { + throw new RuntimeConfigurationError( + 'runtime.shutdownTimeoutMs', + 'must be greater than search.requestDeadlineMs' + ); + } + if (config.runtime.http.headersTimeoutMs > config.runtime.http.requestTimeoutMs) { + throw new RuntimeConfigurationError( + 'runtime.http.headersTimeoutMs', + 'must be less than or equal to runtime.http.requestTimeoutMs' + ); + } + if (config.rateLimiting.search.enabled && config.rateLimiting.search.maxRequests < 1) { + throw new RuntimeConfigurationError( + 'rateLimiting.search.maxRequests', + 'must be a positive integer when rate limiting is enabled' + ); + } +} + +function resolveRuntimeConfig({ + appConfig = AppConfig, + env = process.env, + searchProviderSpec = SearchProviderSpec +} = {}) { + const defaultSearchProvider = getDefaultSearchProviderSpec(searchProviderSpec); + const providerEndpoint = parseOptionalString( + 'search.providerEndpoint', + readConfigValue(appConfig, 'search.providerEndpoint', defaultSearchProvider.providerEndpoint), + { maxLength: 4096 } + ); + const index = parseOptionalString( + 'search.index', + readConfigValue(appConfig, 'search.index', defaultSearchProvider.index), + { maxLength: 255 } + ); + const baseURL = parseOptionalString('BASE_URL', env.BASE_URL, { maxLength: 4096 }); + + const config = { + server: { + port: parseInteger('PORT', env.PORT ?? DEFAULT_PORT, RUNTIME_CONFIG_LIMITS.port), + baseURL: parseHttpUrl('BASE_URL', baseURL, { allowCredentials: false }), + nodeEnvironment: String(env.NODE_ENV || 'development'), + configEnvironment: String(env.NODE_CONFIG_ENV || env.NODE_ENV || 'default') + }, + search: { + provider: { + providerEndpoint, + index, + allowUnknownSslCerts: parseBoolean( + 'search.allowUnknownSslCerts', + readConfigValue( + appConfig, + 'search.allowUnknownSslCerts', + defaultSearchProvider.allowUnknownSslCerts ?? false + ) + ) + }, + logPerformanceTimings: parseBoolean( + 'search.logPerformanceTimings', + readConfigValue(appConfig, 'search.logPerformanceTimings', false) + ), + postFilterCacheTtlMs: parseInteger( + 'search.postFilterCacheTtlMs', + readConfigValue(appConfig, 'search.postFilterCacheTtlMs', constants.SEARCH_POST_FILTER_CACHE_TTL_MS), + RUNTIME_CONFIG_LIMITS.postFilterCacheTtlMs + ), + postFilterCacheMaxEntries: parseInteger( + 'search.postFilterCacheMaxEntries', + readConfigValue( + appConfig, + 'search.postFilterCacheMaxEntries', + constants.SEARCH_POST_FILTER_CACHE_MAX_ENTRIES + ), + RUNTIME_CONFIG_LIMITS.postFilterCacheMaxEntries + ), + postFilterColdSearchConcurrency: parseInteger( + 'search.postFilterColdSearchConcurrency', + readConfigValue( + appConfig, + 'search.postFilterColdSearchConcurrency', + constants.SEARCH_POST_FILTER_COLD_SEARCH_CONCURRENCY + ), + RUNTIME_CONFIG_LIMITS.postFilterColdSearchConcurrency + ), + postFilterColdSearchMaxQueueDepth: parseInteger( + 'search.postFilterColdSearchMaxQueueDepth', + readConfigValue( + appConfig, + 'search.postFilterColdSearchMaxQueueDepth', + constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_QUEUE_DEPTH + ), + RUNTIME_CONFIG_LIMITS.postFilterColdSearchMaxQueueDepth + ), + postFilterColdSearchMaxWaitMs: parseInteger( + 'search.postFilterColdSearchMaxWaitMs', + readConfigValue( + appConfig, + 'search.postFilterColdSearchMaxWaitMs', + constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_WAIT_MS + ), + RUNTIME_CONFIG_LIMITS.postFilterColdSearchMaxWaitMs + ), + inFlightMaxWaitersPerOperation: parseInteger( + 'search.inFlightMaxWaitersPerOperation', + readConfigValue( + appConfig, + 'search.inFlightMaxWaitersPerOperation', + constants.SEARCH_IN_FLIGHT_MAX_WAITERS_PER_OPERATION + ), + RUNTIME_CONFIG_LIMITS.inFlightMaxWaitersPerOperation + ), + postFilterEvaluationChunkSize: parseInteger( + 'search.postFilterEvaluationChunkSize', + readConfigValue( + appConfig, + 'search.postFilterEvaluationChunkSize', + constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE + ), + RUNTIME_CONFIG_LIMITS.postFilterEvaluationChunkSize + ), + openSearchRequestTimeoutMs: parseInteger( + 'search.openSearchRequestTimeoutMs', + readConfigValue( + appConfig, + 'search.openSearchRequestTimeoutMs', + constants.SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS + ), + RUNTIME_CONFIG_LIMITS.openSearchRequestTimeoutMs + ), + requestDeadlineMs: parseInteger( + 'search.requestDeadlineMs', + readConfigValue(appConfig, 'search.requestDeadlineMs', constants.SEARCH_REQUEST_DEADLINE_MS), + RUNTIME_CONFIG_LIMITS.requestDeadlineMs + ), + readinessTimeoutMs: parseInteger( + 'search.readinessTimeoutMs', + readConfigValue(appConfig, 'search.readinessTimeoutMs', constants.SEARCH_READINESS_TIMEOUT_MS), + RUNTIME_CONFIG_LIMITS.readinessTimeoutMs + ) + }, + requestLogging: { + successSampleRate: parseNumber( + 'requestLogging.successSampleRate', + readConfigValue(appConfig, 'requestLogging.successSampleRate', DEFAULT_REQUEST_LOG_SUCCESS_SAMPLE_RATE), + { min: 0, max: 1 } + ), + healthChecks: parseBoolean( + 'requestLogging.healthChecks', + readConfigValue(appConfig, 'requestLogging.healthChecks', DEFAULT_REQUEST_LOG_HEALTH_CHECKS) + ) + }, + runtime: { + shutdownTimeoutMs: parseInteger( + 'runtime.shutdownTimeoutMs', + readConfigValue(appConfig, 'runtime.shutdownTimeoutMs', constants.RUNTIME_SHUTDOWN_TIMEOUT_MS), + RUNTIME_CONFIG_LIMITS.shutdownTimeoutMs + ), + http: { + headersTimeoutMs: parseInteger( + 'runtime.http.headersTimeoutMs', + readConfigValue(appConfig, 'runtime.http.headersTimeoutMs', constants.HTTP_HEADERS_TIMEOUT_MS), + RUNTIME_CONFIG_LIMITS.httpHeadersTimeoutMs + ), + requestTimeoutMs: parseInteger( + 'runtime.http.requestTimeoutMs', + readConfigValue(appConfig, 'runtime.http.requestTimeoutMs', constants.HTTP_REQUEST_TIMEOUT_MS), + RUNTIME_CONFIG_LIMITS.httpRequestTimeoutMs + ), + keepAliveTimeoutMs: parseInteger( + 'runtime.http.keepAliveTimeoutMs', + readConfigValue(appConfig, 'runtime.http.keepAliveTimeoutMs', constants.HTTP_KEEP_ALIVE_TIMEOUT_MS), + RUNTIME_CONFIG_LIMITS.httpKeepAliveTimeoutMs + ) + } + }, + features: { + webSearchEnabled: parseBoolean( + 'features.webSearchEnabled', + readConfigValue(appConfig, 'features.webSearchEnabled', DEFAULT_WEB_SEARCH_ENABLED) + ) + }, + rateLimiting: { + search: { + enabled: parseBoolean( + 'rateLimiting.search.enabled', + readConfigValue(appConfig, 'rateLimiting.search.enabled', DEFAULT_SEARCH_RATE_LIMIT_ENABLED) + ), + windowMs: parseInteger( + 'rateLimiting.search.windowMs', + readConfigValue(appConfig, 'rateLimiting.search.windowMs', DEFAULT_SEARCH_RATE_LIMIT_WINDOW_MS), + RUNTIME_CONFIG_LIMITS.searchRateLimitWindowMs + ), + maxRequests: parseInteger( + 'rateLimiting.search.maxRequests', + readConfigValue( + appConfig, + 'rateLimiting.search.maxRequests', + DEFAULT_SEARCH_RATE_LIMIT_MAX_REQUESTS + ), + RUNTIME_CONFIG_LIMITS.searchRateLimitMaxRequests + ) + } + } + }; + + validateRelationships(config); + return deepFreeze(config); +} + +function getRuntimeConfig() { + if (!resolvedRuntimeConfig) { + resolvedRuntimeConfig = resolveRuntimeConfig(); + } + + return resolvedRuntimeConfig; +} + +function validateRuntimeConfig(config = getRuntimeConfig()) { + const provider = config.search.provider; + parseHttpUrl('search.providerEndpoint', provider.providerEndpoint, { required: true }); + if (!provider.index) { + throw new RuntimeConfigurationError('search.index', 'must be configured'); + } + + validateRelationships(config); + return config; +} + +function sanitizeRuntimeConfig(config) { + return deepFreeze({ + server: { + port: config.server.port, + nodeEnvironment: config.server.nodeEnvironment, + configEnvironment: config.server.configEnvironment + }, + search: { + allowUnknownSslCerts: config.search.provider.allowUnknownSslCerts, + logPerformanceTimings: config.search.logPerformanceTimings, + postFilterCacheTtlMs: config.search.postFilterCacheTtlMs, + postFilterCacheMaxEntries: config.search.postFilterCacheMaxEntries, + postFilterColdSearchConcurrency: config.search.postFilterColdSearchConcurrency, + postFilterColdSearchMaxQueueDepth: config.search.postFilterColdSearchMaxQueueDepth, + postFilterColdSearchMaxWaitMs: config.search.postFilterColdSearchMaxWaitMs, + inFlightMaxWaitersPerOperation: config.search.inFlightMaxWaitersPerOperation, + postFilterEvaluationChunkSize: config.search.postFilterEvaluationChunkSize, + openSearchRequestTimeoutMs: config.search.openSearchRequestTimeoutMs, + requestDeadlineMs: config.search.requestDeadlineMs, + readinessTimeoutMs: config.search.readinessTimeoutMs + }, + requestLogging: config.requestLogging, + runtime: config.runtime, + features: config.features, + rateLimiting: config.rateLimiting + }); +} + +function isRuntimeConfigurationError(err) { + return err instanceof RuntimeConfigurationError || err?.code === 'RUNTIME_CONFIGURATION'; +} + +function resetForTests() { + resolvedRuntimeConfig = undefined; +} + +module.exports = { + getRuntimeConfig, + isRuntimeConfigurationError, + resetForTests, + resolveRuntimeConfig, + sanitizeRuntimeConfig, + validateRuntimeConfig, + RUNTIME_CONFIG_LIMITS, + RuntimeConfigurationError +}; diff --git a/utils/runtimeConfig.test.js b/utils/runtimeConfig.test.js new file mode 100644 index 0000000..850ead3 --- /dev/null +++ b/utils/runtimeConfig.test.js @@ -0,0 +1,272 @@ +const constants = require('./constants'); +const runtimeConfig = require('./runtimeConfig'); + +function createAppConfig(values = {}) { + return { + get: jest.fn((path) => values[path]), + has: jest.fn((path) => Object.hasOwn(values, path)) + }; +} + +function createSearchProviderSpec(overrides = {}) { + return { + getDefaultSearchProviderSpec: jest.fn(() => ({ + providerEndpoint: 'https://search.example', + index: 'test-index', + allowUnknownSslCerts: false, + ...overrides + })) + }; +} + +function resolve(values = {}, env = {}, providerOverrides = {}) { + return runtimeConfig.resolveRuntimeConfig({ + appConfig: createAppConfig(values), + env: { + NODE_ENV: 'test', + ...env + }, + searchProviderSpec: createSearchProviderSpec(providerOverrides) + }); +} + +describe('runtimeConfig', () => { + test('resolves and deeply freezes documented defaults', () => { + const config = resolve(); + + expect(config).toMatchObject({ + server: { + port: 3000, + baseURL: '', + nodeEnvironment: 'test', + configEnvironment: 'test' + }, + search: { + provider: { + providerEndpoint: 'https://search.example', + index: 'test-index', + allowUnknownSslCerts: false + }, + logPerformanceTimings: false, + postFilterCacheTtlMs: constants.SEARCH_POST_FILTER_CACHE_TTL_MS, + postFilterCacheMaxEntries: constants.SEARCH_POST_FILTER_CACHE_MAX_ENTRIES, + postFilterColdSearchConcurrency: constants.SEARCH_POST_FILTER_COLD_SEARCH_CONCURRENCY, + postFilterColdSearchMaxQueueDepth: constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_QUEUE_DEPTH, + postFilterColdSearchMaxWaitMs: constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_WAIT_MS, + inFlightMaxWaitersPerOperation: constants.SEARCH_IN_FLIGHT_MAX_WAITERS_PER_OPERATION, + postFilterEvaluationChunkSize: constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE, + openSearchRequestTimeoutMs: constants.SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS, + requestDeadlineMs: constants.SEARCH_REQUEST_DEADLINE_MS, + readinessTimeoutMs: constants.SEARCH_READINESS_TIMEOUT_MS + }, + requestLogging: { + successSampleRate: 0.1, + healthChecks: false + }, + runtime: { + shutdownTimeoutMs: constants.RUNTIME_SHUTDOWN_TIMEOUT_MS, + http: { + headersTimeoutMs: constants.HTTP_HEADERS_TIMEOUT_MS, + requestTimeoutMs: constants.HTTP_REQUEST_TIMEOUT_MS, + keepAliveTimeoutMs: constants.HTTP_KEEP_ALIVE_TIMEOUT_MS + } + }, + features: { + webSearchEnabled: true + }, + rateLimiting: { + search: { + enabled: false, + windowMs: 60000, + maxRequests: 0 + } + } + }); + expect(Object.isFrozen(config)).toBe(true); + expect(Object.isFrozen(config.search)).toBe(true); + expect(Object.isFrozen(config.search.provider)).toBe(true); + }); + + test('resolves typed environment and AppConfig overrides', () => { + const config = resolve( + { + 'search.allowUnknownSslCerts': 'TRUE', + 'search.logPerformanceTimings': 'true', + 'search.postFilterCacheTtlMs': '120000', + 'search.postFilterCacheMaxEntries': '250', + 'search.postFilterColdSearchConcurrency': '6', + 'search.postFilterColdSearchMaxQueueDepth': '50', + 'search.postFilterColdSearchMaxWaitMs': '15000', + 'search.inFlightMaxWaitersPerOperation': '125', + 'search.postFilterEvaluationChunkSize': '100', + 'search.openSearchRequestTimeoutMs': '25000', + 'search.requestDeadlineMs': '45000', + 'search.readinessTimeoutMs': '7000', + 'requestLogging.successSampleRate': '0.25', + 'requestLogging.healthChecks': 'TRUE', + 'runtime.shutdownTimeoutMs': '60000', + 'runtime.http.headersTimeoutMs': '12000', + 'runtime.http.requestTimeoutMs': '20000', + 'runtime.http.keepAliveTimeoutMs': '6000', + 'features.webSearchEnabled': 'FALSE', + 'rateLimiting.search.enabled': 'TRUE', + 'rateLimiting.search.windowMs': '30000', + 'rateLimiting.search.maxRequests': '25' + }, + { + PORT: '4000', + BASE_URL: 'https://api.example', + NODE_CONFIG_ENV: 'prod' + } + ); + + expect(config).toMatchObject({ + server: { + port: 4000, + baseURL: 'https://api.example', + nodeEnvironment: 'test', + configEnvironment: 'prod' + }, + search: { + provider: { + allowUnknownSslCerts: true + }, + logPerformanceTimings: true, + postFilterCacheTtlMs: 120000, + postFilterCacheMaxEntries: 250, + postFilterColdSearchConcurrency: 6, + postFilterColdSearchMaxQueueDepth: 50, + postFilterColdSearchMaxWaitMs: 15000, + inFlightMaxWaitersPerOperation: 125, + postFilterEvaluationChunkSize: 100, + openSearchRequestTimeoutMs: 25000, + requestDeadlineMs: 45000, + readinessTimeoutMs: 7000 + }, + requestLogging: { + successSampleRate: 0.25, + healthChecks: true + }, + runtime: { + shutdownTimeoutMs: 60000, + http: { + headersTimeoutMs: 12000, + requestTimeoutMs: 20000, + keepAliveTimeoutMs: 6000 + } + }, + features: { + webSearchEnabled: false + }, + rateLimiting: { + search: { + enabled: true, + windowMs: 30000, + maxRequests: 25 + } + } + }); + }); + + test.each([ + ['1', 'search.allowUnknownSslCerts'], + ['yes', 'search.logPerformanceTimings'], + ['invalid', 'requestLogging.healthChecks'], + [1, 'features.webSearchEnabled'], + ['on', 'rateLimiting.search.enabled'] + ])('rejects non-boolean value %p for %s', (value, path) => { + expect(() => resolve({ [path]: value })).toThrow(`${path} must be true or false.`); + }); + + test.each([ + ['PORT', { PORT: '0' }, {}], + ['search.postFilterCacheTtlMs', {}, { 'search.postFilterCacheTtlMs': '-1' }], + ['search.postFilterCacheMaxEntries', {}, { 'search.postFilterCacheMaxEntries': '10001' }], + ['search.postFilterColdSearchConcurrency', {}, { 'search.postFilterColdSearchConcurrency': '101' }], + ['search.postFilterColdSearchMaxQueueDepth', {}, { 'search.postFilterColdSearchMaxQueueDepth': '10001' }], + ['search.postFilterEvaluationChunkSize', {}, { 'search.postFilterEvaluationChunkSize': '1.5' }], + ['runtime.http.headersTimeoutMs', {}, { 'runtime.http.headersTimeoutMs': '0' }], + ['runtime.http.requestTimeoutMs', {}, { 'runtime.http.requestTimeoutMs': 'invalid' }], + ['runtime.http.keepAliveTimeoutMs', {}, { 'runtime.http.keepAliveTimeoutMs': '60001' }], + ['requestLogging.successSampleRate', {}, { 'requestLogging.successSampleRate': '1.1' }] + ])('rejects an out-of-range or malformed %s value', (path, env, values) => { + expect(() => resolve(values, env)).toThrow(path); + }); + + test.each([ + [ + 'search.readinessTimeoutMs', + { + 'search.readinessTimeoutMs': '21000' + } + ], + [ + 'search.openSearchRequestTimeoutMs', + { + 'search.openSearchRequestTimeoutMs': '30000' + } + ], + [ + 'search.postFilterColdSearchMaxWaitMs', + { + 'search.postFilterColdSearchMaxWaitMs': '30000' + } + ], + [ + 'runtime.shutdownTimeoutMs', + { + 'runtime.shutdownTimeoutMs': '30000' + } + ], + [ + 'runtime.http.headersTimeoutMs', + { + 'runtime.http.headersTimeoutMs': '16000' + } + ], + [ + 'rateLimiting.search.maxRequests', + { + 'rateLimiting.search.enabled': 'true' + } + ] + ])('rejects invalid cross-setting relationship for %s', (path, values) => { + expect(() => resolve(values)).toThrow(path); + }); + + test.each([ + [{ providerEndpoint: '' }, 'search.providerEndpoint'], + [{ index: '' }, 'search.index'], + [{ providerEndpoint: 'not-a-url' }, 'search.providerEndpoint'] + ])('startup validation rejects an invalid provider configuration', (providerOverrides, path) => { + const config = resolve({}, {}, providerOverrides); + + expect(() => runtimeConfig.validateRuntimeConfig(config)).toThrow(path); + }); + + test('rejects credential-bearing BASE_URL values', () => { + expect(() => + resolve( + {}, + { + BASE_URL: 'https://admin:secret@api.example' + } + ) + ).toThrow('BASE_URL must not contain credentials.'); + }); + + test('sanitized startup metadata does not expose provider or base URLs', () => { + const config = resolve( + {}, + { + BASE_URL: 'https://api.example' + } + ); + + const serialized = JSON.stringify(runtimeConfig.sanitizeRuntimeConfig(config)); + + expect(serialized).not.toContain('search.example'); + expect(serialized).not.toContain('test-index'); + expect(serialized).not.toContain('api.example'); + }); +}); diff --git a/utils/runtimeLifecycle.js b/utils/runtimeLifecycle.js index cc2c86f..09bc3a8 100644 --- a/utils/runtimeLifecycle.js +++ b/utils/runtimeLifecycle.js @@ -1,18 +1,10 @@ -const { AppConfig } = require('cve-core'); -const constants = require('./constants'); +const runtimeConfig = require('./runtimeConfig'); let shuttingDown = false; let activeRequestCount = 0; function getShutdownTimeoutMs() { - if (!AppConfig.has('runtime.shutdownTimeoutMs')) { - return constants.RUNTIME_SHUTDOWN_TIMEOUT_MS; - } - - const value = Number(AppConfig.get('runtime.shutdownTimeoutMs')); - if (Number.isInteger(value) && value >= 1) return value; - - return constants.RUNTIME_SHUTDOWN_TIMEOUT_MS; + return runtimeConfig.getRuntimeConfig().runtime.shutdownTimeoutMs; } function getState() { @@ -52,6 +44,7 @@ function trackRequest(_req, res, next) { function resetForTests() { shuttingDown = false; activeRequestCount = 0; + runtimeConfig.resetForTests(); } module.exports = { diff --git a/utils/runtimeLifecycle.test.js b/utils/runtimeLifecycle.test.js index 47b2571..a01edf4 100644 --- a/utils/runtimeLifecycle.test.js +++ b/utils/runtimeLifecycle.test.js @@ -65,13 +65,10 @@ describe('runtimeLifecycle', () => { expect(mockAppConfigGet).not.toHaveBeenCalled(); }); - test.each(['not-a-number', '0', '-1', '1.5'])( - 'falls back to the shutdown timeout default for invalid value %s', - (value) => { - mockAppConfigHas.mockReturnValue(true); - mockAppConfigGet.mockReturnValue(value); - - expect(runtimeLifecycle.getShutdownTimeoutMs()).toBe(35000); - } - ); + test.each(['not-a-number', '0', '-1', '1.5'])('rejects an invalid shutdown timeout value %s', (value) => { + mockAppConfigHas.mockImplementation((path) => path === 'runtime.shutdownTimeoutMs'); + mockAppConfigGet.mockReturnValue(value); + + expect(() => runtimeLifecycle.getShutdownTimeoutMs()).toThrow('runtime.shutdownTimeoutMs'); + }); }); diff --git a/utils/searchCandidateProcessor.js b/utils/searchCandidateProcessor.js new file mode 100644 index 0000000..5e408c1 --- /dev/null +++ b/utils/searchCandidateProcessor.js @@ -0,0 +1,105 @@ +const cpe = require('./cpe'); +const constants = require('./constants'); +const cveId = require('./cveId'); +const searchDeadline = require('./searchDeadline'); +const { OPEN_SEARCH_RESPONSE_FAILURE_TYPES, OpenSearchResponseError } = require('./searchResponseValidator'); + +function isObject(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function isValidCpeCandidateHit(hit, cpeEvaluationContext) { + if (!isObject(hit) || !isObject(hit._source)) return false; + + const candidateCveId = hit._source.cveMetadata?.cveId; + if (!cveId.isValidCveId(candidateCveId)) return false; + + const cpeApplicability = hit._source.containers?.cna?.cpeApplicability; + const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; + if (!applicabilityItems.length || applicabilityItems.some((item) => !isObject(item))) return false; + + let matchCount = 0; + for (const applicability of applicabilityItems) { + if (!Array.isArray(applicability.nodes)) return false; + + for (const node of applicability.nodes) { + if (!isObject(node) || !Array.isArray(node.cpeMatch)) return false; + if (!node.cpeMatch.every((match) => cpe.isValidCpeMatch(match, cpeEvaluationContext))) return false; + matchCount += node.cpeMatch.length; + } + } + + return matchCount > 0; +} + +function hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext) { + if ( + postFilters?.cpeName && + !cpe.hitAppliesToCpeName(hit, postFilters.cpeName, cpeEvaluationContext, { + requireVulnerable: postFilters.isVulnerable === true + }) + ) { + return false; + } + + if ( + postFilters?.virtualMatchString && + !cpe.hitMatchesVirtualMatchString(hit, postFilters.virtualMatchString, cpeEvaluationContext) + ) { + return false; + } + + if ( + postFilters?.virtualMatchStringRange && + !cpe.hitOverlapsVirtualMatchStringRange(hit, postFilters.virtualMatchStringRange, cpeEvaluationContext) + ) { + return false; + } + + return true; +} + +function yieldToEventLoop() { + return new Promise((resolve) => setImmediate(resolve)); +} + +async function filterSearchHitsInChunks(resp, postFilters, signal, chunkSize) { + const hitsContainer = resp?.data?.hits; + if (!Array.isArray(hitsContainer?.hits)) { + throw new OpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + const effectiveChunkSize = Math.max(1, chunkSize ?? constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE); + const cpeEvaluationContext = cpe.createEvaluationContext(); + const filteredHits = []; + + for (let start = 0; start < hitsContainer.hits.length; start += effectiveChunkSize) { + searchDeadline.throwIfAborted(signal); + const end = Math.min(start + effectiveChunkSize, hitsContainer.hits.length); + + for (let index = start; index < end; index += 1) { + const hit = hitsContainer.hits[index]; + if (!isValidCpeCandidateHit(hit, cpeEvaluationContext)) { + throw new OpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_CANDIDATE); + } + if (hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext)) { + filteredHits.push(hit); + } + } + + if (end < hitsContainer.hits.length) { + await yieldToEventLoop(); + } + } + + searchDeadline.throwIfAborted(signal); + return { + hitsContainer, + candidateCount: hitsContainer.hits.length, + filteredHits + }; +} + +module.exports = { + filterSearchHitsInChunks +}; diff --git a/utils/searchCandidateProcessor.test.js b/utils/searchCandidateProcessor.test.js new file mode 100644 index 0000000..6048b35 --- /dev/null +++ b/utils/searchCandidateProcessor.test.js @@ -0,0 +1,212 @@ +const constants = require('./constants'); +const searchCandidateProcessor = require('./searchCandidateProcessor'); +const searchDeadline = require('./searchDeadline'); + +function cpeCandidateHit(cpeMatch = {}) { + return { + _id: 'CVE-2024-0001', + _source: { + cveMetadata: { + cveId: 'CVE-2024-0001' + }, + containers: { + cna: { + cpeApplicability: [ + { + nodes: [ + { + cpeMatch: [ + { + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true, + ...cpeMatch + } + ] + } + ] + } + ] + } + } + } + }; +} + +function candidateResponse(hits) { + return { + data: { + hits: { + hits + } + } + }; +} + +function nestedOnlyCpeCandidateHit() { + const candidate = cpeCandidateHit(); + candidate._source.containers.cna.cpeApplicability[0].nodes = [ + { + operator: 'OR', + negate: false, + nodes: [ + { + cpeMatch: [ + { + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true + } + ] + } + ] + } + ]; + return candidate; +} + +describe('searchCandidateProcessor', () => { + test('validates and filters candidates in the same processing pass', async () => { + const matching = cpeCandidateHit({ + versionStartIncluding: '2.9', + versionEndExcluding: '2.11' + }); + const notMatching = cpeCandidateHit({ + versionStartIncluding: '3.0' + }); + notMatching._id = 'CVE-2024-0002'; + notMatching._source.cveMetadata.cveId = 'CVE-2024-0002'; + + await expect( + searchCandidateProcessor.filterSearchHitsInChunks( + candidateResponse([matching, notMatching]), + { + cpeName: 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*', + isVulnerable: true + }, + undefined, + 1 + ) + ).resolves.toMatchObject({ + candidateCount: 2, + filteredHits: [matching] + }); + }); + + test.each([ + [ + 'missing projected CVE ID', + { + ...cpeCandidateHit(), + _source: { + ...cpeCandidateHit()._source, + cveMetadata: {} + } + } + ], + [ + 'lowercase projected CVE ID', + { + ...cpeCandidateHit(), + _source: { + ...cpeCandidateHit()._source, + cveMetadata: { + cveId: 'cve-2024-0001' + } + } + } + ], + [ + 'oversized projected CVE ID sequence', + { + ...cpeCandidateHit(), + _source: { + ...cpeCandidateHit()._source, + cveMetadata: { + cveId: `CVE-2024-${'1'.repeat(constants.MAX_CVE_ID_SEQUENCE_DIGITS + 1)}` + } + } + } + ], + [ + 'missing projected CPE applicability', + { + ...cpeCandidateHit(), + _source: { + ...cpeCandidateHit()._source, + containers: { + cna: {} + } + } + } + ], + ['nested-only CPE applicability entry', nestedOnlyCpeCandidateHit()], + ['missing CPE criteria', cpeCandidateHit({ criteria: undefined })], + ['invalid CPE criteria', cpeCandidateHit({ criteria: 'not-a-cpe-name' })], + ['missing CPE vulnerability flag', cpeCandidateHit({ vulnerable: undefined })], + ['invalid CPE range boundary', cpeCandidateHit({ versionEndExcluding: 2 })], + [ + 'conflicting inclusive and exclusive lower bounds', + cpeCandidateHit({ + versionStartIncluding: '1.0', + versionStartExcluding: '1.1' + }) + ], + [ + 'conflicting inclusive and exclusive upper bounds', + cpeCandidateHit({ + versionEndIncluding: '2.0', + versionEndExcluding: '2.1' + }) + ], + [ + 'reversed version range', + cpeCandidateHit({ + versionStartIncluding: '2.0', + versionEndIncluding: '1.0' + }) + ], + [ + 'empty exclusive version range', + cpeCandidateHit({ + versionStartIncluding: '1.0', + versionEndExcluding: '1.0' + }) + ] + ])('rejects CPE candidate hits with %s', async (_caseName, hit) => { + await expect( + searchCandidateProcessor.filterSearchHitsInChunks(candidateResponse([hit]), {}, undefined, 1) + ).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_candidate' + }); + }); + + test('accepts a one-version range when both equal boundaries are inclusive', async () => { + const candidate = cpeCandidateHit({ + versionStartIncluding: '1.0', + versionEndIncluding: '1.0' + }); + + await expect( + searchCandidateProcessor.filterSearchHitsInChunks(candidateResponse([candidate]), {}, undefined, 1) + ).resolves.toMatchObject({ + candidateCount: 1, + filteredHits: [candidate] + }); + }); + + test('honors cancellation before candidate validation begins', async () => { + const abortController = new AbortController(); + abortController.abort(new searchDeadline.SearchClientDisconnectedError()); + + await expect( + searchCandidateProcessor.filterSearchHitsInChunks( + candidateResponse([cpeCandidateHit({ criteria: undefined })]), + {}, + abortController.signal, + 1 + ) + ).rejects.toMatchObject({ + code: 'SEARCH_CLIENT_DISCONNECTED' + }); + }); +}); diff --git a/utils/searchPerformance.js b/utils/searchPerformance.js index 7dfa764..95caf0b 100644 --- a/utils/searchPerformance.js +++ b/utils/searchPerformance.js @@ -1,5 +1,10 @@ +const { monitorEventLoopDelay } = require('perf_hooks'); const searchProvider = require('./searchProvider'); +const EVENT_LOOP_DELAY_RESOLUTION_MS = 20; +const RESPONSE_JSON_INSTRUMENTED = Symbol('responseJsonInstrumented'); +let eventLoopDelayMonitor; + function markTime() { return process.hrtime.bigint(); } @@ -20,6 +25,61 @@ function getCacheStatus(metrics, postFilters) { return 'miss'; } +function nanosecondsToMs(value) { + return Number.isFinite(value) ? value / 1e6 : undefined; +} + +function getEventLoopDelayMonitor() { + if (!eventLoopDelayMonitor) { + eventLoopDelayMonitor = monitorEventLoopDelay({ + resolution: EVENT_LOOP_DELAY_RESOLUTION_MS + }); + eventLoopDelayMonitor.enable(); + } + + return eventLoopDelayMonitor; +} + +function instrumentJsonResponse(res, metrics) { + if (typeof res?.json !== 'function' || res[RESPONSE_JSON_INSTRUMENTED]) return; + + const originalJson = res.json; + res[RESPONSE_JSON_INSTRUMENTED] = true; + res.json = function instrumentedJson(body) { + const serializationStart = markTime(); + const result = originalJson.call(this, body); + metrics.responseSerializationElapsedMs = elapsedMs(serializationStart); + + const contentLength = Number(this.getHeader?.('content-length')); + if (Number.isFinite(contentLength) && contentLength >= 0) { + metrics.responseBytes = contentLength; + } + + return result; + }; +} + +function beginSearchRequest(metrics, res) { + if (!searchProvider.shouldLogSearchPerformance()) return false; + + metrics.heapUsedStartBytes = process.memoryUsage().heapUsed; + getEventLoopDelayMonitor(); + instrumentJsonResponse(res, metrics); + return true; +} + +function completeSearchRequest(metrics) { + if (metrics.heapUsedStartBytes === undefined) return; + + const heapUsedBytes = process.memoryUsage().heapUsed; + const monitor = getEventLoopDelayMonitor(); + metrics.heapUsedBytes = heapUsedBytes; + metrics.heapUsedDeltaBytes = heapUsedBytes - metrics.heapUsedStartBytes; + metrics.eventLoopDelayMeanMs = nanosecondsToMs(monitor.mean); + metrics.eventLoopDelayP99Ms = nanosecondsToMs(monitor.percentile(99)); + metrics.eventLoopDelayMaxMs = nanosecondsToMs(monitor.max); +} + function logSearchPerformance(metrics, searchOptions, responseOptions, postFilters) { const outcome = metrics.outcome ?? 'success'; const includeDetailedTimings = searchProvider.shouldLogSearchPerformance(); @@ -57,6 +117,15 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte candidateCount: metrics.candidateCount, filteredCount: metrics.filteredCount, returnedCount: metrics.returnedCount, + cachedResultCveIdCount: metrics.cachedResultCveIdCount, + cacheState: metrics.cacheState, + responseBytes: metrics.responseBytes, + responseSerializationElapsedMs: roundMs(metrics.responseSerializationElapsedMs), + heapUsedBytes: metrics.heapUsedBytes, + heapUsedDeltaBytes: metrics.heapUsedDeltaBytes, + eventLoopDelayMeanMs: roundMs(metrics.eventLoopDelayMeanMs), + eventLoopDelayP99Ms: roundMs(metrics.eventLoopDelayP99Ms), + eventLoopDelayMaxMs: roundMs(metrics.eventLoopDelayMaxMs), openSearchTookMs: metrics.openSearchTookMs, pageFetchTookMs: metrics.pageFetchTookMs, openSearchElapsedMs: roundMs(metrics.openSearchElapsedMs), @@ -72,10 +141,18 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte console.log(JSON.stringify(event)); } +function resetForTests() { + eventLoopDelayMonitor?.disable(); + eventLoopDelayMonitor = undefined; +} + module.exports = { + beginSearchRequest, + completeSearchRequest, elapsedMs, getCacheStatus, logSearchPerformance, markTime, + resetForTests, roundMs }; diff --git a/utils/searchPerformance.test.js b/utils/searchPerformance.test.js index 4d93959..52a7036 100644 --- a/utils/searchPerformance.test.js +++ b/utils/searchPerformance.test.js @@ -13,6 +13,7 @@ describe('searchPerformance', () => { }); afterEach(() => { + searchPerformance.resetForTests(); jest.restoreAllMocks(); }); @@ -33,6 +34,18 @@ describe('searchPerformance', () => { candidateCount: 10000, filteredCount: 4, returnedCount: 1, + cachedResultCveIdCount: 4, + cacheState: { + entryCount: 2, + cachedCveIdCount: 8 + }, + responseBytes: 2048, + responseSerializationElapsedMs: 1.234, + heapUsedBytes: 2000000, + heapUsedDeltaBytes: 50000, + eventLoopDelayMeanMs: 2.345, + eventLoopDelayP99Ms: 8.765, + eventLoopDelayMaxMs: 12.345, openSearchTookMs: 97, pageFetchTookMs: 8, openSearchElapsedMs: 123.456, @@ -84,6 +97,18 @@ describe('searchPerformance', () => { candidateCount: 10000, filteredCount: 4, returnedCount: 1, + cachedResultCveIdCount: 4, + cacheState: { + entryCount: 2, + cachedCveIdCount: 8 + }, + responseBytes: 2048, + responseSerializationElapsedMs: 1.23, + heapUsedBytes: 2000000, + heapUsedDeltaBytes: 50000, + eventLoopDelayMeanMs: 2.35, + eventLoopDelayP99Ms: 8.77, + eventLoopDelayMaxMs: 12.35, openSearchTookMs: 97, pageFetchTookMs: 8, openSearchElapsedMs: 123.46, @@ -118,6 +143,40 @@ describe('searchPerformance', () => { expect(console.log).not.toHaveBeenCalled(); }); + test('does not instrument response or process resources when detailed logging is disabled', () => { + mockShouldLogSearchPerformance.mockReturnValue(false); + const metrics = {}; + const json = jest.fn(); + const res = { json }; + + expect(searchPerformance.beginSearchRequest(metrics, res)).toBe(false); + searchPerformance.completeSearchRequest(metrics); + + expect(res.json).toBe(json); + expect(metrics).toStrictEqual({}); + }); + + test('captures response size and process resource metrics only when detailed logging is enabled', () => { + mockShouldLogSearchPerformance.mockReturnValue(true); + const metrics = {}; + const res = { + getHeader: jest.fn(() => '42'), + json: jest.fn(() => res) + }; + + searchPerformance.beginSearchRequest(metrics, res); + res.json({ status: 'ok' }); + searchPerformance.completeSearchRequest(metrics); + + expect(metrics).toMatchObject({ + responseBytes: 42, + responseSerializationElapsedMs: expect.any(Number), + heapUsedBytes: expect.any(Number), + heapUsedDeltaBytes: expect.any(Number) + }); + expect(metrics.responseSerializationElapsedMs).toBeGreaterThanOrEqual(0); + }); + test('logSearchPerformance always emits sanitized failure outcomes', () => { mockShouldLogSearchPerformance.mockReturnValue(false); diff --git a/utils/searchPostFilterCache.js b/utils/searchPostFilterCache.js index adfd906..ee90e58 100644 --- a/utils/searchPostFilterCache.js +++ b/utils/searchPostFilterCache.js @@ -77,6 +77,20 @@ function set(cacheKey, value, now = Date.now()) { } } +function getState(now = Date.now()) { + for (const [key, entry] of entries) { + if (entry.expiresAt <= now) entries.delete(key); + } + + return { + entryCount: entries.size, + cachedCveIdCount: [...entries.values()].reduce( + (total, entry) => total + (entry.value?.filteredCveIds?.length ?? 0), + 0 + ) + }; +} + function clear() { entries.clear(); } @@ -85,6 +99,7 @@ module.exports = { buildCacheKey, clear, get, + getState, normalizeForKey, set }; diff --git a/utils/searchPostFilterCache.test.js b/utils/searchPostFilterCache.test.js index 4fd81c6..7d2f41d 100644 --- a/utils/searchPostFilterCache.test.js +++ b/utils/searchPostFilterCache.test.js @@ -143,4 +143,30 @@ describe('searchPostFilterCache', () => { expect(searchPostFilterCache.get(key, 1001)).toBe(value); expect(searchPostFilterCache.get(key, 61001)).toBeNull(); }); + + test('reports entry and cached CVE ID counts while pruning expired entries', () => { + searchPostFilterCache.set( + 'first', + { + filteredCveIds: ['CVE-2024-0001', 'CVE-2024-0002'] + }, + 1000 + ); + searchPostFilterCache.set( + 'second', + { + filteredCveIds: ['CVE-2024-0003'] + }, + 2000 + ); + + expect(searchPostFilterCache.getState(3000)).toStrictEqual({ + entryCount: 2, + cachedCveIdCount: 3 + }); + expect(searchPostFilterCache.getState(62000)).toStrictEqual({ + entryCount: 0, + cachedCveIdCount: 0 + }); + }); }); diff --git a/utils/searchProvider.js b/utils/searchProvider.js index 4d8b9eb..eb4393b 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -1,7 +1,7 @@ -const { AppConfig, BasicSearchManager, SearchProviderSpec, SearchReader } = require('cve-core'); -const constants = require('./constants'); -const cpe = require('./cpe'); +const { BasicSearchManager, SearchReader } = require('cve-core'); +const runtimeConfig = require('./runtimeConfig'); const searchDeadline = require('./searchDeadline'); +const searchResponseValidator = require('./searchResponseValidator'); class SearchProviderContractError extends Error { constructor() { @@ -19,31 +19,6 @@ class SearchProviderConfigurationError extends Error { } } -const OPEN_SEARCH_RESPONSE_FAILURE_TYPES = Object.freeze({ - MALFORMED_CANDIDATE: 'malformed_candidate', - MALFORMED_RESPONSE: 'malformed_response', - PARTIAL_SHARDS: 'partial_shards', - TIMEOUT: 'timeout' -}); -const CPE_CANDIDATE_OPERATION = 'cpe-candidate'; -const CVE_ID_PATTERN = /^CVE-\d{4}-\d{4,19}$/; -const CPE_RANGE_FIELDS = [ - 'versionStartIncluding', - 'versionStartExcluding', - 'versionEndIncluding', - 'versionEndExcluding' -]; - -class OpenSearchResponseError extends Error { - constructor(failureType) { - super(`OpenSearch returned an invalid ${failureType} response.`); - this.name = 'OpenSearchResponseError'; - this.code = 'OPEN_SEARCH_RESPONSE_INVALID'; - this.failureType = failureType; - } -} - -let searchProviderConfig; let basicSearchManager; let searchReader; let activeOpenSearchRequestCount = 0; @@ -63,59 +38,14 @@ function configureManagerRequestTimeout(manager) { return manager; } -function normalizeBoolean(value, defaultValue = false) { - if (value === undefined || value === null || value === '') return defaultValue; - if (typeof value === 'boolean') return value; - if (typeof value === 'number') return value !== 0; - - const normalized = String(value).trim().toLowerCase(); - if (['true', '1', 'yes', 'y', 'on'].includes(normalized)) return true; - if (['false', '0', 'no', 'n', 'off'].includes(normalized)) return false; - - return defaultValue; -} - -function getIntegerConfig(path, defaultValue, { min = 0 } = {}) { - if (!AppConfig.has(path)) return defaultValue; - - const value = Number(AppConfig.get(path)); - if (Number.isInteger(value) && value >= min) return value; - - return defaultValue; -} - -function getAllowUnknownSslCerts(spec) { - if (AppConfig.has('search.allowUnknownSslCerts')) { - return normalizeBoolean(AppConfig.get('search.allowUnknownSslCerts')); - } - - return normalizeBoolean(spec?.allowUnknownSslCerts); -} - -function normalizeSearchProviderConfig(spec) { - const providerEndpoint = spec?.providerEndpoint; - const index = spec?.index; - - if (!providerEndpoint || !index) { - throw new SearchProviderConfigurationError(); - } - - return Object.freeze({ - providerEndpoint, - index, - allowUnknownSslCerts: getAllowUnknownSslCerts(spec) - }); -} - function getSearchProviderConfig() { - if (!searchProviderConfig) { - const defaultSpec = SearchProviderSpec.getDefaultSearchProviderSpec(); - searchProviderConfig = normalizeSearchProviderConfig(defaultSpec); - defaultSpec.allowUnknownSslCerts = searchProviderConfig.allowUnknownSslCerts; - SearchProviderSpec._sDefaultSearchProviderSpec = defaultSpec; + const config = runtimeConfig.getRuntimeConfig().search.provider; + if (!config.providerEndpoint || !config.index) { + throw new SearchProviderConfigurationError(); } - return searchProviderConfig; + runtimeConfig.validateRuntimeConfig(); + return config; } function getBasicSearchManager() { @@ -137,90 +67,6 @@ function validateSearchReader(reader) { return reader; } -function throwOpenSearchResponseError(failureType) { - throw new OpenSearchResponseError(failureType); -} - -function isObject(value) { - return Boolean(value) && typeof value === 'object' && !Array.isArray(value); -} - -function isValidCpeMatch(match) { - if (!isObject(match) || typeof match.criteria !== 'string' || match.criteria.trim() === '') return false; - if (!cpe.parseCpeMatchCriteria(match.criteria).valid) return false; - if (typeof match.vulnerable !== 'boolean') return false; - - return CPE_RANGE_FIELDS.every( - (field) => match[field] === undefined || (typeof match[field] === 'string' && match[field].trim() !== '') - ); -} - -function isValidCpeCandidateHit(hit) { - if (!isObject(hit) || !isObject(hit._source)) return false; - - const cveId = hit._source.cveMetadata?.cveId; - if (typeof cveId !== 'string' || !CVE_ID_PATTERN.test(cveId)) return false; - - const cpeApplicability = hit._source.containers?.cna?.cpeApplicability; - const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; - if (!applicabilityItems.length || applicabilityItems.some((item) => !isObject(item))) return false; - - let matchCount = 0; - for (const applicability of applicabilityItems) { - if (!Array.isArray(applicability.nodes)) return false; - - for (const node of applicability.nodes) { - if (!isObject(node) || !Array.isArray(node.cpeMatch)) return false; - if (!node.cpeMatch.every(isValidCpeMatch)) return false; - matchCount += node.cpeMatch.length; - } - } - - return matchCount > 0; -} - -function validateCpeCandidateHits(hits) { - if (hits.every(isValidCpeCandidateHit)) return; - - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_CANDIDATE); -} - -function validateOpenSearchResponse(body, { operation } = {}) { - if (!body || typeof body !== 'object' || Array.isArray(body)) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); - } - - if (body.timed_out === true) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.TIMEOUT); - } - - if (body._shards !== undefined) { - if (!body._shards || typeof body._shards !== 'object' || Array.isArray(body._shards)) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); - } - - if (body._shards.failed !== undefined) { - const failedShards = Number(body._shards.failed); - if (!Number.isInteger(failedShards) || failedShards < 0) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); - } - if (failedShards > 0) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.PARTIAL_SHARDS); - } - } - } - - if (!Array.isArray(body.hits?.hits)) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); - } - - if (operation === CPE_CANDIDATE_OPERATION) { - validateCpeCandidateHits(body.hits.hits); - } - - return body; -} - function getSearchReader() { if (!searchReader) { const config = getSearchProviderConfig(); @@ -356,7 +202,7 @@ async function searchOpenSearch(body, { signal, operation = 'search', onStart, r } ); - return validateOpenSearchResponse(response.body, { operation }); + return searchResponseValidator.validateOpenSearchResponse(response.body, { operation }); } async function checkOpenSearchReadiness({ signal } = {}) { @@ -387,7 +233,7 @@ async function checkOpenSearchReadiness({ signal } = {}) { operation: 'readiness' } ); - validateOpenSearchResponse(response.body); + searchResponseValidator.validateOpenSearchResponse(response.body); return true; } @@ -401,93 +247,65 @@ function isSearchProviderConfigurationError(err) { } function isOpenSearchResponseError(err) { - return err instanceof OpenSearchResponseError || err?.code === 'OPEN_SEARCH_RESPONSE_INVALID'; + return searchResponseValidator.isOpenSearchResponseError(err); } function getOpenSearchReadinessTimeoutMs() { - return getIntegerConfig('search.readinessTimeoutMs', constants.SEARCH_READINESS_TIMEOUT_MS, { - min: 1 - }); + return runtimeConfig.getRuntimeConfig().search.readinessTimeoutMs; } function shouldLogSearchPerformance() { - if (!AppConfig.has('search.logPerformanceTimings')) return false; - - return normalizeBoolean(AppConfig.get('search.logPerformanceTimings')); + return runtimeConfig.getRuntimeConfig().search.logPerformanceTimings; } function getPostFilterCacheTtlMs() { - return getIntegerConfig('search.postFilterCacheTtlMs', constants.SEARCH_POST_FILTER_CACHE_TTL_MS); + return runtimeConfig.getRuntimeConfig().search.postFilterCacheTtlMs; } function getPostFilterCacheMaxEntries() { - return getIntegerConfig('search.postFilterCacheMaxEntries', constants.SEARCH_POST_FILTER_CACHE_MAX_ENTRIES); + return runtimeConfig.getRuntimeConfig().search.postFilterCacheMaxEntries; } function getPostFilterColdSearchConcurrency() { - return getIntegerConfig( - 'search.postFilterColdSearchConcurrency', - constants.SEARCH_POST_FILTER_COLD_SEARCH_CONCURRENCY, - { min: 1 } - ); + return runtimeConfig.getRuntimeConfig().search.postFilterColdSearchConcurrency; } function getPostFilterColdSearchMaxQueueDepth() { - return getIntegerConfig( - 'search.postFilterColdSearchMaxQueueDepth', - constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_QUEUE_DEPTH, - { min: 0 } - ); + return runtimeConfig.getRuntimeConfig().search.postFilterColdSearchMaxQueueDepth; } function getPostFilterColdSearchMaxWaitMs() { - return getIntegerConfig( - 'search.postFilterColdSearchMaxWaitMs', - constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_WAIT_MS, - { min: 1 } - ); + return runtimeConfig.getRuntimeConfig().search.postFilterColdSearchMaxWaitMs; } function getInFlightMaxWaitersPerOperation() { - return getIntegerConfig( - 'search.inFlightMaxWaitersPerOperation', - constants.SEARCH_IN_FLIGHT_MAX_WAITERS_PER_OPERATION, - { min: 1 } - ); + return runtimeConfig.getRuntimeConfig().search.inFlightMaxWaitersPerOperation; } function getPostFilterEvaluationChunkSize() { - return getIntegerConfig( - 'search.postFilterEvaluationChunkSize', - constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE, - { min: 1 } - ); + return runtimeConfig.getRuntimeConfig().search.postFilterEvaluationChunkSize; } function getOpenSearchRequestTimeoutMs() { - return getIntegerConfig('search.openSearchRequestTimeoutMs', constants.SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS, { - min: 1 - }); + return runtimeConfig.getRuntimeConfig().search.openSearchRequestTimeoutMs; } function getSearchRequestDeadlineMs() { - return getIntegerConfig('search.requestDeadlineMs', constants.SEARCH_REQUEST_DEADLINE_MS, { min: 1 }); + return runtimeConfig.getRuntimeConfig().search.requestDeadlineMs; } function resetSearchProviderForTests() { - searchProviderConfig = undefined; basicSearchManager = undefined; searchReader = undefined; activeOpenSearchRequestCount = 0; activeOpenSearchRequestsByOperation.clear(); + runtimeConfig.resetForTests(); } module.exports = { checkOpenSearchReadiness, closeSearchProviderClients, getBasicSearchManager, - getAllowUnknownSslCerts, - getIntegerConfig, getOpenSearchRequestState, getOpenSearchReadinessTimeoutMs, getInFlightMaxWaitersPerOperation, @@ -504,14 +322,13 @@ module.exports = { isOpenSearchResponseError, isSearchProviderConfigurationError, isSearchProviderContractError, - normalizeBoolean, resetSearchProviderForTests, searchOpenSearch, shouldLogSearchPerformance, - validateOpenSearchResponse, + validateOpenSearchResponse: searchResponseValidator.validateOpenSearchResponse, validateSearchReader, - OPEN_SEARCH_RESPONSE_FAILURE_TYPES, - OpenSearchResponseError, + OPEN_SEARCH_RESPONSE_FAILURE_TYPES: searchResponseValidator.OPEN_SEARCH_RESPONSE_FAILURE_TYPES, + OpenSearchResponseError: searchResponseValidator.OpenSearchResponseError, SearchProviderConfigurationError, SearchProviderContractError }; diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index 8abc85d..ed13c55 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -115,11 +115,6 @@ describe('searchProvider', () => { allowUnknownSslCerts: false }); expect(Object.isFrozen(first)).toBe(true); - expect(mockSearchProviderSpec._sDefaultSearchProviderSpec).toStrictEqual({ - providerEndpoint: 'https://search.example', - index: 'test-index', - allowUnknownSslCerts: false - }); expect(mockGetDefaultSearchProviderSpec).toHaveBeenCalledTimes(1); }); @@ -327,6 +322,40 @@ describe('searchProvider', () => { }); }); + test.each([ + ['missing', undefined], + ['non-object', 1], + ['missing value', {}], + ['negative', { value: -1 }], + ['non-integer', { value: 1.5 }], + ['string', { value: '1' }] + ])('rejects a standard OpenSearch response with a %s total', async (_caseName, total) => { + mockClientSearch.mockResolvedValue({ + body: { + hits: { + total, + hits: [] + } + } + }); + + await expect( + searchProvider.searchOpenSearch( + { + query: { + match_all: {} + } + }, + { + operation: 'standard' + } + ) + ).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_response' + }); + }); + test('accepts schema-shaped CPE candidate hits for candidate operations', async () => { const body = { hits: { @@ -371,38 +400,12 @@ describe('searchProvider', () => { ).resolves.toBe(body); }); - test.each([ - [ - 'missing projected CVE ID', - { - ...cpeCandidateHit(), - _source: { - ...cpeCandidateHit()._source, - cveMetadata: {} - } - } - ], - [ - 'missing projected CPE applicability', - { - ...cpeCandidateHit(), - _source: { - ...cpeCandidateHit()._source, - containers: { - cna: {} - } - } - } - ], - ['missing CPE criteria', cpeCandidateHit({ criteria: undefined })], - ['invalid CPE criteria', cpeCandidateHit({ criteria: 'not-a-cpe-name' })], - ['missing CPE vulnerability flag', cpeCandidateHit({ vulnerable: undefined })], - ['invalid CPE range boundary', cpeCandidateHit({ versionEndExcluding: 2 })] - ])('rejects CPE candidate hits with %s', async (_caseName, hit) => { + test('does not synchronously traverse CPE candidate elements during response validation', async () => { + const malformedHit = cpeCandidateHit({ criteria: undefined }); mockClientSearch.mockResolvedValue({ body: { hits: { - hits: [hit] + hits: [malformedHit] } } }); @@ -418,9 +421,10 @@ describe('searchProvider', () => { operation: 'cpe-candidate' } ) - ).rejects.toMatchObject({ - code: 'OPEN_SEARCH_RESPONSE_INVALID', - failureType: 'malformed_candidate' + ).resolves.toStrictEqual({ + hits: { + hits: [malformedHit] + } }); }); @@ -649,7 +653,7 @@ describe('searchProvider', () => { index: 'test-index', allowUnknownSslCerts: true }); - mockAppConfigHas.mockReturnValue(true); + mockAppConfigHas.mockImplementation((path) => path === 'search.allowUnknownSslCerts'); mockAppConfigGet.mockReturnValue('FALSE'); expect(searchProvider.getSearchProviderConfig()).toStrictEqual({ @@ -657,11 +661,10 @@ describe('searchProvider', () => { index: 'test-index', allowUnknownSslCerts: false }); - expect(mockSearchProviderSpec._sDefaultSearchProviderSpec.allowUnknownSslCerts).toBe(false); }); test('normalizes string TRUE from AppConfig for local certificate bypasses', () => { - mockAppConfigHas.mockReturnValue(true); + mockAppConfigHas.mockImplementation((path) => path === 'search.allowUnknownSslCerts'); mockAppConfigGet.mockReturnValue('TRUE'); expect(searchProvider.getSearchProviderConfig().allowUnknownSslCerts).toBe(true); @@ -686,7 +689,8 @@ describe('searchProvider', () => { 'search.postFilterEvaluationChunkSize', 'search.openSearchRequestTimeoutMs', 'search.requestDeadlineMs', - 'search.readinessTimeoutMs' + 'search.readinessTimeoutMs', + 'runtime.shutdownTimeoutMs' ].includes(key) ); mockAppConfigGet.mockImplementation((key) => { @@ -700,7 +704,8 @@ describe('searchProvider', () => { 'search.postFilterEvaluationChunkSize': '100', 'search.openSearchRequestTimeoutMs': '25000', 'search.requestDeadlineMs': '45000', - 'search.readinessTimeoutMs': '7000' + 'search.readinessTimeoutMs': '7000', + 'runtime.shutdownTimeoutMs': '60000' }; return values[key]; }); @@ -717,19 +722,10 @@ describe('searchProvider', () => { expect(searchProvider.getOpenSearchReadinessTimeoutMs()).toBe(7000); }); - test('falls back to CPE post-filter runtime defaults when config values are invalid', () => { - mockAppConfigHas.mockReturnValue(true); + test('rejects an invalid explicit CPE post-filter runtime value', () => { + mockAppConfigHas.mockImplementation((key) => key === 'search.postFilterCacheTtlMs'); mockAppConfigGet.mockReturnValue('not-a-number'); - expect(searchProvider.getPostFilterCacheTtlMs()).toBe(60000); - expect(searchProvider.getPostFilterCacheMaxEntries()).toBe(100); - expect(searchProvider.getPostFilterColdSearchConcurrency()).toBe(10); - expect(searchProvider.getPostFilterColdSearchMaxQueueDepth()).toBe(500); - expect(searchProvider.getPostFilterColdSearchMaxWaitMs()).toBe(10000); - expect(searchProvider.getInFlightMaxWaitersPerOperation()).toBe(500); - expect(searchProvider.getPostFilterEvaluationChunkSize()).toBe(250); - expect(searchProvider.getOpenSearchRequestTimeoutMs()).toBe(20000); - expect(searchProvider.getSearchRequestDeadlineMs()).toBe(30000); - expect(searchProvider.getOpenSearchReadinessTimeoutMs()).toBe(5000); + expect(() => searchProvider.getPostFilterCacheTtlMs()).toThrow('search.postFilterCacheTtlMs'); }); }); diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index f5c59ec..4f86dd9 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -1,5 +1,6 @@ const constants = require('./constants'); const cpe = require('./cpe'); +const cveId = require('./cveId'); const cvss = require('./cvss'); const { MAX_RESULTS_LIMIT, RESULTS_PER_PAGE } = constants; @@ -70,7 +71,11 @@ function buildSearchRequest(params) { return request; } -function buildOpenSearchBody(searchOptions, queryStringObjects = [], { sourceIncludes, trackTotalHits = true } = {}) { +function buildOpenSearchBody( + searchOptions, + queryStringObjects = [], + { includeSort = true, sourceIncludes, trackTotalHits = true } = {} +) { const normalizedQueryStringObjects = Array.isArray(queryStringObjects) ? queryStringObjects : []; const queryObj = { must: [...normalizedQueryStringObjects], @@ -80,12 +85,14 @@ function buildOpenSearchBody(searchOptions, queryStringObjects = [], { sourceInc query: { bool: queryObj }, - sort: buildDeterministicSort(normalizedQueryStringObjects), track_total_hits: trackTotalHits, size: searchOptions.resultsPerPage, from: searchOptions.from }; + if (includeSort) { + body.sort = buildDeterministicSort(normalizedQueryStringObjects); + } if (sourceIncludes?.length) { body._source = sourceIncludes; } @@ -100,18 +107,22 @@ function buildCpeCandidateSearchBody(searchOptions, queryStringObjects = []) { }); } -function buildCveIdsSearchRequest(cveIds) { +function buildCveIdsSearchBody(cveIds) { const filter = buildCveIdsFilter(cveIds.join(','), constants.FIELDS.cveIds); - return { - searchOptions: { + return buildOpenSearchBody( + { filters: filter ? [filter] : [], rangeObjects: [], resultsPerPage: Math.max(cveIds.length, 1), from: 0 }, - queryStringObjects: [] - }; + [], + { + includeSort: false, + trackTotalHits: false + } + ); } function buildPostFilters(params) { @@ -256,35 +267,17 @@ function escapeQueryStringTerm(term, { escapeWildcards = false } = {}) { } function buildQueryStrings(params) { - const queryStringObjects = []; - - getOrderedSearchParamNames(params).forEach((param) => { - if (param === 'virtualMatchString') return; - - const fieldSpec = constants.FIELDS[param]; - const fieldList = Array.isArray(fieldSpec) ? fieldSpec : [fieldSpec]; - - if (param === 'keywordSearch') { - queryStringObjects.push({ - simple_query_string: { - query: params[param], - default_operator: 'AND', - fields: fieldList - } - }); - return; - } + if (!Object.prototype.hasOwnProperty.call(params, 'keywordSearch')) return []; - queryStringObjects.push({ - query_string: { - query: params[param], + return [ + { + simple_query_string: { + query: params.keywordSearch, default_operator: 'AND', - fields: fieldList + fields: [constants.FIELDS.keywordSearch] } - }); - }); - - return queryStringObjects; + } + ]; } function getFilterFields(params) { @@ -308,7 +301,7 @@ function getFilterFields(params) { if (!fieldSpec) return; if (param === 'noRejected') { - if (params[param] === true || params[param] === '') { + if (params[param] === true) { fieldsToReturn.push(buildNoRejectedFilter(fieldSpec)); } return; @@ -379,7 +372,7 @@ function buildCvssTokenRegexp(token) { function parseCveIds(value) { return String(value ?? '') .split(',') - .map((id) => id.trim().toUpperCase()) + .map(cveId.normalizeCveId) .filter(Boolean); } @@ -418,29 +411,10 @@ function buildCpeNameFilter(cpeName) { const wildcardVersionCriteria = buildCpeCriteriaFilter(criteriaKeywordField, parsedCpe, 4, { versionMode: 'wildcard' }); - const rangeFields = [ - fields.versionStartIncluding, - fields.versionStartExcluding, - fields.versionEndIncluding, - fields.versionEndExcluding - ]; return { bool: { - should: [ - exactVersionCriteria, - { - bool: { - filter: [wildcardVersionCriteria], - must_not: rangeFields.map(buildExistsFilter) - } - }, - { - bool: { - filter: [wildcardVersionCriteria, buildHasAnyRangeFieldFilter(rangeFields)] - } - } - ], + should: [exactVersionCriteria, wildcardVersionCriteria], minimum_should_match: 1 } }; @@ -506,23 +480,6 @@ function buildCpeCriteriaRegexpFilter(field, value) { }; } -function buildExistsFilter(field) { - return { - exists: { - field - } - }; -} - -function buildHasAnyRangeFieldFilter(rangeFields) { - return { - bool: { - should: rangeFields.map(buildExistsFilter), - minimum_should_match: 1 - } - }; -} - function buildCvssRegexpFilter(inputVectorString, param, fieldSpec) { const parsed = cvss.parseCvssMetrics(inputVectorString, param); if (!parsed.valid) { @@ -610,7 +567,7 @@ function cvssV2SeverityToRange(sev) { module.exports = { buildCpeCandidateSearchBody, - buildCveIdsSearchRequest, + buildCveIdsSearchBody, buildOpenSearchBody, buildQueryStrings, buildSearchRequest, diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index e8f4b88..ff7cfa1 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -353,21 +353,50 @@ describe('searchQueryBuilder buildSearchRequest()', () => { }); }); - test('buildCveIdsSearchRequest builds exact page fetch request for selected CVE IDs', () => { - expect(queryBuilder.buildCveIdsSearchRequest(['CVE-2024-0002', 'CVE-2024-0001'])).toStrictEqual({ - searchOptions: { - filters: [ - { - terms: { - 'cveMetadata.cveId.keyword': ['CVE-2024-0001', 'CVE-2024-0002'] + test('buildCveIdsSearchBody omits total counting and sorting for locally ordered page fetches', () => { + expect(queryBuilder.buildCveIdsSearchBody(['CVE-2024-0002', 'CVE-2024-0001'])).toStrictEqual({ + query: { + bool: { + must: [], + filter: [ + { + terms: { + 'cveMetadata.cveId.keyword': ['CVE-2024-0001', 'CVE-2024-0002'] + } } - } - ], - rangeObjects: [], - resultsPerPage: 2, - from: 0 + ] + } }, - queryStringObjects: [] + track_total_hits: false, + size: 2, + from: 0 + }); + }); + + test('buildOpenSearchBody can omit deterministic sorting for bounded internal fetches', () => { + expect( + queryBuilder.buildOpenSearchBody( + { + filters: [], + rangeObjects: [], + resultsPerPage: 1, + from: 0 + }, + [], + { + includeSort: false + } + ) + ).toStrictEqual({ + query: { + bool: { + must: [], + filter: [] + } + }, + track_total_hits: true, + size: 1, + from: 0 }); }); }); @@ -393,8 +422,9 @@ describe('searchQueryBuilder buildQueryStrings() function', () => { expect(output).toStrictEqual(testQueryString); }); - test('buildQueryStrings() ignores virtualMatchString because it is handled as a structured filter.', () => { + test('buildQueryStrings() ignores non-keyword fields instead of generating query_string clauses.', () => { const output = queryBuilder.buildQueryStrings({ + cweId: 'CWE-79', virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809' }); @@ -496,6 +526,14 @@ describe('searchQueryBuilder getFilterFields() function', () => { expect(output).toStrictEqual([]); }); + test('getFilterFields() does not retain legacy empty-string noRejected behavior.', () => { + const output = queryBuilder.getFilterFields({ + noRejected: '' + }); + + expect(output).toStrictEqual([]); + }); + test('getFilterFields() builds a cvssV2 severity range filter on baseScore.', () => { const output = queryBuilder.getFilterFields({ cvssV2Severity: 'MEDIUM' @@ -523,7 +561,7 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds CPE Name criteria filters for exact and range match strings.', () => { + test('getFilterFields() builds exact-version and wildcard-version CPE Name candidate filters.', () => { const output = queryBuilder.getFilterFields({ cpeName: 'cpe:2.3:a:artifex:ghostscript:10.05.1:*:*:*:*:*:*:*' }); @@ -542,124 +580,28 @@ describe('searchQueryBuilder getFilterFields() function', () => { 'cpe:2.3:*:*:*:10.05.1:' ]) ); - expect(output[0].bool.should[1]).toStrictEqual({ - bool: { - filter: [ - prefixShould([ - 'cpe:2.3:a:artifex:ghostscript:*:', - 'cpe:2.3:a:artifex:*:*:', - 'cpe:2.3:a:*:ghostscript:*:', - 'cpe:2.3:a:*:*:*:', - 'cpe:2.3:*:artifex:ghostscript:*:', - 'cpe:2.3:*:artifex:*:*:', - 'cpe:2.3:*:*:ghostscript:*:', - 'cpe:2.3:*:*:*:*:' - ]) - ], - must_not: [ - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartExcluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndExcluding' - } - } - ] - } - }); - expect(output[0].bool.should[2]).toStrictEqual({ - bool: { - filter: [ - prefixShould([ - 'cpe:2.3:a:artifex:ghostscript:*:', - 'cpe:2.3:a:artifex:*:*:', - 'cpe:2.3:a:*:ghostscript:*:', - 'cpe:2.3:a:*:*:*:', - 'cpe:2.3:*:artifex:ghostscript:*:', - 'cpe:2.3:*:artifex:*:*:', - 'cpe:2.3:*:*:ghostscript:*:', - 'cpe:2.3:*:*:*:*:' - ]), - { - bool: { - should: [ - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartExcluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndExcluding' - } - } - ], - minimum_should_match: 1 - } - } - ] - } - }); + expect(output[0].bool.should[1]).toStrictEqual( + prefixShould([ + 'cpe:2.3:a:artifex:ghostscript:*:', + 'cpe:2.3:a:artifex:*:*:', + 'cpe:2.3:a:*:ghostscript:*:', + 'cpe:2.3:a:*:*:*:', + 'cpe:2.3:*:artifex:ghostscript:*:', + 'cpe:2.3:*:artifex:*:*:', + 'cpe:2.3:*:*:ghostscript:*:', + 'cpe:2.3:*:*:*:*:' + ]) + ); + expect(output[0].bool.should).toHaveLength(2); }); - test('getFilterFields() avoids lexicographic keyword range clauses for CPE version boundaries.', () => { + test('getFilterFields() leaves CPE version range applicability to semantic post-filtering.', () => { const output = queryBuilder.getFilterFields({ cpeName: 'cpe:2.3:a:artifex:ghostscript:10.05.1:*:*:*:*:*:*:*' }); - const rangeBranchFilters = output[0].bool.should[2].bool.filter; - - expect(rangeBranchFilters).toHaveLength(2); - expect(rangeBranchFilters[1]).toStrictEqual({ - bool: { - should: [ - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartExcluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndExcluding' - } - } - ], - minimum_should_match: 1 - } - }); + expect(output[0].bool.should).toHaveLength(2); + expect(JSON.stringify(output)).not.toContain('"exists"'); expect(JSON.stringify(output)).not.toContain('.keyword":{"lte"'); expect(JSON.stringify(output)).not.toContain('.keyword":{"lt"'); expect(JSON.stringify(output)).not.toContain('.keyword":{"gte"'); diff --git a/utils/searchRateLimiter.js b/utils/searchRateLimiter.js index 00026e1..eee12ad 100644 --- a/utils/searchRateLimiter.js +++ b/utils/searchRateLimiter.js @@ -1,62 +1,14 @@ -const { AppConfig } = require('cve-core'); const { rateLimit } = require('express-rate-limit'); const errors = require('./errors'); +const runtimeConfig = require('./runtimeConfig'); const searchPerformance = require('./searchPerformance'); -const DEFAULT_SEARCH_RATE_LIMIT_ENABLED = false; -const DEFAULT_SEARCH_RATE_LIMIT_WINDOW_MS = 60000; const GLOBAL_SEARCH_RATE_LIMIT_KEY = 'search-api-process'; const error = new errors.errors(); -function parseBoolean(value) { - if (typeof value === 'boolean') return value; - - const normalized = String(value).trim().toLowerCase(); - if (['true', '1', 'yes', 'y', 'on'].includes(normalized)) return true; - if (['false', '0', 'no', 'n', 'off'].includes(normalized)) return false; - - return undefined; -} - -function readBooleanConfig(path, defaultValue) { - if (!AppConfig.has(path)) return defaultValue; - - const value = parseBoolean(AppConfig.get(path)); - if (value === undefined) throw new Error(`${path} must be a boolean.`); - - return value; -} - -function readPositiveIntegerConfig(path, defaultValue) { - if (!AppConfig.has(path)) return defaultValue; - - const value = Number(AppConfig.get(path)); - if (!Number.isInteger(value) || value < 1) return undefined; - - return value; -} - function getSearchRateLimitConfig() { - const enabled = readBooleanConfig('rateLimiting.search.enabled', DEFAULT_SEARCH_RATE_LIMIT_ENABLED); - if (!enabled) { - return Object.freeze({ - enabled, - windowMs: DEFAULT_SEARCH_RATE_LIMIT_WINDOW_MS - }); - } - - const windowMs = readPositiveIntegerConfig('rateLimiting.search.windowMs', DEFAULT_SEARCH_RATE_LIMIT_WINDOW_MS); - const maxRequests = readPositiveIntegerConfig('rateLimiting.search.maxRequests'); - - if (!windowMs) throw new Error('rateLimiting.search.windowMs must be a positive integer when enabled.'); - if (!maxRequests) throw new Error('rateLimiting.search.maxRequests must be a positive integer when enabled.'); - - return Object.freeze({ - enabled, - windowMs, - maxRequests - }); + return runtimeConfig.getRuntimeConfig().rateLimiting.search; } function disabledRateLimiter(_req, _res, next) { @@ -93,8 +45,13 @@ function createSearchRateLimiter(config = getSearchRateLimitConfig()) { }); } +function resetForTests() { + runtimeConfig.resetForTests(); +} + module.exports = { createSearchRateLimiter, getSearchRateLimitConfig, - handleRateLimit + handleRateLimit, + resetForTests }; diff --git a/utils/searchRateLimiter.test.js b/utils/searchRateLimiter.test.js index da00ea6..170d5ef 100644 --- a/utils/searchRateLimiter.test.js +++ b/utils/searchRateLimiter.test.js @@ -20,6 +20,7 @@ const searchRateLimiter = require('./searchRateLimiter'); describe('searchRateLimiter', () => { beforeEach(() => { + searchRateLimiter.resetForTests(); mockAppConfigHas.mockReset(); mockAppConfigGet.mockReset(); mockLogSearchPerformance.mockReset(); @@ -56,7 +57,7 @@ describe('searchRateLimiter', () => { { 'rateLimiting.search.enabled': 'invalid' }, - 'rateLimiting.search.enabled must be a boolean.' + 'rateLimiting.search.enabled' ], [ { @@ -64,7 +65,7 @@ describe('searchRateLimiter', () => { 'rateLimiting.search.windowMs': '0', 'rateLimiting.search.maxRequests': '25' }, - 'rateLimiting.search.windowMs must be a positive integer when enabled.' + 'rateLimiting.search.windowMs' ], [ { @@ -72,7 +73,7 @@ describe('searchRateLimiter', () => { 'rateLimiting.search.windowMs': '60000', 'rateLimiting.search.maxRequests': '0' }, - 'rateLimiting.search.maxRequests must be a positive integer when enabled.' + 'rateLimiting.search.maxRequests' ] ])('rejects invalid explicit configuration', (values, expectedMessage) => { mockAppConfigHas.mockImplementation((path) => Object.hasOwn(values, path)); diff --git a/utils/searchResponseValidator.js b/utils/searchResponseValidator.js new file mode 100644 index 0000000..47a4acc --- /dev/null +++ b/utils/searchResponseValidator.js @@ -0,0 +1,75 @@ +const OPEN_SEARCH_RESPONSE_FAILURE_TYPES = Object.freeze({ + MALFORMED_CANDIDATE: 'malformed_candidate', + MALFORMED_RESPONSE: 'malformed_response', + PARTIAL_SHARDS: 'partial_shards', + TIMEOUT: 'timeout' +}); +const TOTAL_REQUIRED_OPERATIONS = new Set(['standard']); + +class OpenSearchResponseError extends Error { + constructor(failureType) { + super(`OpenSearch returned an invalid ${failureType} response.`); + this.name = 'OpenSearchResponseError'; + this.code = 'OPEN_SEARCH_RESPONSE_INVALID'; + this.failureType = failureType; + } +} + +function throwOpenSearchResponseError(failureType) { + throw new OpenSearchResponseError(failureType); +} + +function isObject(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function hasValidTotal(hits) { + return isObject(hits.total) && Number.isInteger(hits.total.value) && hits.total.value >= 0; +} + +function validateOpenSearchResponse(body, { operation } = {}) { + if (!isObject(body)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + if (body.timed_out === true) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.TIMEOUT); + } + + if (body._shards !== undefined) { + if (!isObject(body._shards)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + if (body._shards.failed !== undefined) { + const failedShards = Number(body._shards.failed); + if (!Number.isInteger(failedShards) || failedShards < 0) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + if (failedShards > 0) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.PARTIAL_SHARDS); + } + } + } + + if (!isObject(body.hits) || !Array.isArray(body.hits.hits)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + if (TOTAL_REQUIRED_OPERATIONS.has(operation) && !hasValidTotal(body.hits)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + return body; +} + +function isOpenSearchResponseError(err) { + return err instanceof OpenSearchResponseError || err?.code === 'OPEN_SEARCH_RESPONSE_INVALID'; +} + +module.exports = { + isOpenSearchResponseError, + validateOpenSearchResponse, + OPEN_SEARCH_RESPONSE_FAILURE_TYPES, + OpenSearchResponseError +};