From 36cd8761f87a586e7a9f53590ece9131ac7de620 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 10:14:50 -0400 Subject: [PATCH 01/33] Fail closed on incomplete OpenSearch responses --- controllers/searchController.js | 42 +++++++++ controllers/searchController.test.js | 135 ++++++++++++++++++++++++++- utils/errors.js | 5 +- utils/errors.test.js | 3 +- utils/searchPerformance.js | 2 + utils/searchProvider.js | 66 ++++++++++++- utils/searchProvider.test.js | 59 ++++++++++++ 7 files changed, 302 insertions(+), 10 deletions(-) diff --git a/controllers/searchController.js b/controllers/searchController.js index 5deabdb..06c2a77 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -103,6 +103,14 @@ function getOpenSearchErrorStatusCode(err) { return Number(err?.statusCode ?? err?.meta?.statusCode ?? err?.meta?.body?.status); } +function recordUpstreamFailure(metrics, err, failureType) { + const upstreamStatusCode = getOpenSearchErrorStatusCode(err); + if (Number.isInteger(upstreamStatusCode) && upstreamStatusCode > 0) { + metrics.upstreamStatusCode = upstreamStatusCode; + } + metrics.upstreamFailureType = failureType; +} + function isOpenSearchUnavailableError(err) { return ( searchProvider.isSearchProviderContractError?.(err) === true || @@ -421,7 +429,35 @@ async function searchHandler(req, res, next) { return res.status(429).json(error.searchQueueTimeoutError()); } + if (searchProvider.isOpenSearchResponseError?.(err)) { + const failureType = err?.failureType ?? 'malformed_response'; + recordUpstreamFailure(metrics, err, failureType); + + if (failureType === 'timeout') { + metrics.outcome = 'search_timeout'; + metrics.statusCode = 504; + metrics.timeoutSource = 'opensearch'; + return res.status(504).json(error.searchTimeoutError()); + } + + metrics.outcome = 'upstream_invalid_response'; + metrics.statusCode = 502; + return res.status(502).json(error.upstreamSearchError()); + } + + if (getOpenSearchErrorStatusCode(err) === 429) { + recordUpstreamFailure(metrics, err, 'throttled'); + metrics.outcome = 'opensearch_throttled'; + metrics.statusCode = 503; + if (typeof res.set === 'function') res.set('Retry-After', '5'); + + return res.status(503).json(error.openSearchError()); + } + if (searchDeadline.isSearchTimeoutError(err)) { + if (err?.name === 'TimeoutError') { + recordUpstreamFailure(metrics, err, 'timeout'); + } metrics.outcome = 'search_timeout'; metrics.statusCode = 504; metrics.timeoutSource = err?.name === 'TimeoutError' ? 'opensearch' : 'request_deadline'; @@ -430,6 +466,11 @@ async function searchHandler(req, res, next) { } if (isOpenSearchUnavailableError(err)) { + recordUpstreamFailure( + metrics, + err, + searchProvider.isSearchProviderContractError?.(err) ? 'provider_contract' : 'unavailable' + ); metrics.outcome = 'opensearch_unavailable'; metrics.statusCode = 503; if (typeof res.set === 'function') res.set('Retry-After', '5'); @@ -437,6 +478,7 @@ async function searchHandler(req, res, next) { return res.status(503).json(error.openSearchError()); } + recordUpstreamFailure(metrics, err, 'upstream_error'); metrics.outcome = 'upstream_error'; metrics.statusCode = 502; return res.status(502).json(error.upstreamSearchError()); diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 2a116f5..2016506 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -22,6 +22,7 @@ const mockGetPostFilterColdSearchConcurrency = jest.fn(() => 10); const mockGetPostFilterColdSearchMaxQueueDepth = jest.fn(() => 500); const mockGetPostFilterColdSearchMaxWaitMs = jest.fn(() => 10000); const mockGetSearchRequestDeadlineMs = jest.fn(() => 30000); +const mockIsOpenSearchResponseError = jest.fn((err) => err?.code === 'OPEN_SEARCH_RESPONSE_INVALID'); const mockIsSearchProviderContractError = jest.fn((err) => err?.code === 'SEARCH_PROVIDER_CONTRACT'); const mockGetBasicSearchManager = jest.fn(() => ({ search: mockBasicSearch @@ -43,6 +44,7 @@ jest.mock('../utils/searchProvider', () => ({ getPostFilterColdSearchMaxQueueDepth: mockGetPostFilterColdSearchMaxQueueDepth, getPostFilterColdSearchMaxWaitMs: mockGetPostFilterColdSearchMaxWaitMs, getSearchRequestDeadlineMs: mockGetSearchRequestDeadlineMs, + isOpenSearchResponseError: mockIsOpenSearchResponseError, isSearchProviderContractError: mockIsSearchProviderContractError, searchOpenSearch: mockSearchOpenSearch, shouldLogSearchPerformance: mockShouldLogSearchPerformance @@ -158,6 +160,7 @@ describe('Search controller SEARCH handler', () => { mockGetPostFilterColdSearchMaxQueueDepth.mockClear(); mockGetPostFilterColdSearchMaxWaitMs.mockClear(); mockGetSearchRequestDeadlineMs.mockClear(); + mockIsOpenSearchResponseError.mockClear(); mockIsSearchProviderContractError.mockClear(); mockGetPostFilterCacheTtlMs.mockReturnValue(60000); mockGetPostFilterCacheMaxEntries.mockReturnValue(100); @@ -1167,6 +1170,33 @@ describe('Search controller SEARCH handler', () => { }); }); + test('SEARCH does not cache a partial OpenSearch candidate response', async () => { + mockSearchRaw.mockRejectedValue({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'partial_shards' + }); + const req = { + query: { + virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809', + resultsPerPage: '1', + pageNumber: '1' + } + }; + + await controller.SEARCH(req, res, next); + const secondRes = createResponse(); + await controller.SEARCH(req, secondRes, next); + + expect(mockSearchRaw).toHaveBeenCalledTimes(2); + for (const response of [res, secondRes]) { + expect(response.status).toHaveBeenCalledWith(502); + expect(response.json).toHaveBeenCalledWith({ + error: 'UPSTREAM_SEARCH_ERROR', + message: 'Search backend returned an error. Please try again later.' + }); + } + }); + test('SEARCH logs redacted performance timing when enabled', async () => { mockShouldLogSearchPerformance.mockReturnValue(true); mockApiSearch.mockResolvedValue(advancedResponse(1, [{ _id: 'CVE-2024-0001' }])); @@ -1516,8 +1546,7 @@ describe('Search controller SEARCH handler', () => { expect(res.set).toHaveBeenCalledWith('Retry-After', '5'); expect(res.json).toHaveBeenCalledWith({ error: 'OPEN_SEARCH_ERROR', - message: - 'OpenSearch refused connection, please check that OpenSearch is running and accessible then try again.' + message: 'Search backend is temporarily unavailable. Please try again later.' }); expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ event: 'search.performance', @@ -1527,6 +1556,108 @@ describe('Search controller SEARCH handler', () => { }); }); + test('SEARCH returns a retryable sanitized 503 when OpenSearch throttles the request', async () => { + mockApiSearch.mockRejectedValue({ + name: 'ResponseError', + message: 'sensitive throttling details', + meta: { + statusCode: 429, + body: { + reason: 'sensitive shard details' + } + } + }); + + await controller.SEARCH( + { + requestId: 'throttled-request', + query: {} + }, + res, + next + ); + + expect(res.status).toHaveBeenCalledWith(503); + expect(res.set).toHaveBeenCalledWith('Retry-After', '5'); + expect(res.json).toHaveBeenCalledWith({ + error: 'OPEN_SEARCH_ERROR', + message: 'Search backend is temporarily unavailable. Please try again later.' + }); + const logEntry = console.log.mock.calls[0][0]; + expect(JSON.parse(logEntry)).toMatchObject({ + event: 'search.performance', + requestId: 'throttled-request', + outcome: 'opensearch_throttled', + statusCode: 503, + upstreamStatusCode: 429, + upstreamFailureType: 'throttled' + }); + expect(logEntry).not.toContain('sensitive throttling details'); + expect(logEntry).not.toContain('sensitive shard details'); + }); + + test('SEARCH returns a sanitized 504 for a timed-out OpenSearch response', async () => { + mockApiSearch.mockRejectedValue({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'timeout' + }); + + await controller.SEARCH( + { + requestId: 'response-timeout-request', + query: {} + }, + res, + next + ); + + expect(res.status).toHaveBeenCalledWith(504); + expect(res.json).toHaveBeenCalledWith({ + error: 'SEARCH_TIMEOUT', + message: 'Search request timed out. Please try again later.' + }); + expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ + event: 'search.performance', + requestId: 'response-timeout-request', + outcome: 'search_timeout', + statusCode: 504, + timeoutSource: 'opensearch', + upstreamFailureType: 'timeout' + }); + }); + + test.each(['partial_shards', 'malformed_response'])( + 'SEARCH returns a sanitized 502 for an OpenSearch %s response', + async (failureType) => { + mockApiSearch.mockRejectedValue({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType + }); + + await controller.SEARCH( + { + requestId: `${failureType}-request`, + query: {} + }, + res, + next + ); + + expect(res.status).toHaveBeenCalledWith(502); + expect(res.json).toHaveBeenCalledWith({ + error: 'UPSTREAM_SEARCH_ERROR', + message: 'Search backend returned an error. Please try again later.' + }); + expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ + event: 'search.performance', + requestId: `${failureType}-request`, + outcome: 'upstream_invalid_response', + statusCode: 502, + upstreamFailureType: failureType + }); + } + ); + test('SEARCH returns a sanitized 504 when OpenSearch times out', async () => { mockApiSearch.mockRejectedValue({ name: 'TimeoutError', diff --git a/utils/errors.js b/utils/errors.js index 4c126e0..889d702 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -267,10 +267,7 @@ class errors { } openSearchError() { - return makeError( - 'OPEN_SEARCH_ERROR', - 'OpenSearch refused connection, please check that OpenSearch is running and accessible then try again.' - ); + return makeError('OPEN_SEARCH_ERROR', 'Search backend is temporarily unavailable. Please try again later.'); } searchNotReadyError() { diff --git a/utils/errors.test.js b/utils/errors.test.js index 682af30..ec465b9 100644 --- a/utils/errors.test.js +++ b/utils/errors.test.js @@ -400,8 +400,7 @@ describe('error response helpers', () => { [], { error: 'OPEN_SEARCH_ERROR', - message: - 'OpenSearch refused connection, please check that OpenSearch is running and accessible then try again.' + message: 'Search backend is temporarily unavailable. Please try again later.' } ], [ diff --git a/utils/searchPerformance.js b/utils/searchPerformance.js index 1240cfc..f666aec 100644 --- a/utils/searchPerformance.js +++ b/utils/searchPerformance.js @@ -33,6 +33,8 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte statusCode: metrics.statusCode, timeoutSource: metrics.timeoutSource, abortReason: metrics.abortReason, + upstreamStatusCode: metrics.upstreamStatusCode, + upstreamFailureType: metrics.upstreamFailureType, strategy: metrics.strategy, cacheStatus: getCacheStatus(metrics, postFilters), coldSearchQueueWaitElapsedMs: roundMs(metrics.coldSearchQueueWaitElapsedMs), diff --git a/utils/searchProvider.js b/utils/searchProvider.js index 929ec6b..5b45207 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -10,6 +10,21 @@ class SearchProviderContractError extends Error { } } +const OPEN_SEARCH_RESPONSE_FAILURE_TYPES = Object.freeze({ + MALFORMED_RESPONSE: 'malformed_response', + PARTIAL_SHARDS: 'partial_shards', + TIMEOUT: 'timeout' +}); + +class OpenSearchResponseError extends Error { + constructor(failureType) { + super(`OpenSearch returned an invalid ${failureType} response.`); + this.name = 'OpenSearchResponseError'; + this.code = 'OPEN_SEARCH_RESPONSE_INVALID'; + this.failureType = failureType; + } +} + let searchProviderConfig; let basicSearchManager; let searchReader; @@ -104,6 +119,42 @@ function validateSearchReader(reader) { return reader; } +function throwOpenSearchResponseError(failureType) { + throw new OpenSearchResponseError(failureType); +} + +function validateOpenSearchResponse(body) { + if (!body || typeof body !== 'object' || Array.isArray(body)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + if (body.timed_out === true) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.TIMEOUT); + } + + if (body._shards !== undefined) { + if (!body._shards || typeof body._shards !== 'object' || Array.isArray(body._shards)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + if (body._shards.failed !== undefined) { + const failedShards = Number(body._shards.failed); + if (!Number.isInteger(failedShards) || failedShards < 0) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + if (failedShards > 0) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.PARTIAL_SHARDS); + } + } + } + + if (!Array.isArray(body.hits?.hits)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + return body; +} + function getSearchReader() { if (!searchReader) { const config = getSearchProviderConfig(); @@ -198,6 +249,7 @@ async function searchOpenSearch(body, { signal, operation = 'search', onStart } reader._client.search( { index: reader._cveIndex, + allow_partial_search_results: false, body }, { @@ -212,17 +264,18 @@ async function searchOpenSearch(body, { signal, operation = 'search', onStart } } ); - return response.body; + return validateOpenSearchResponse(response.body); } async function checkOpenSearchReadiness({ signal } = {}) { searchDeadline.throwIfAborted(signal); const reader = getSearchReader(); - await executeOpenSearchRequest( + const response = await executeOpenSearchRequest( () => reader._client.search( { index: reader._cveIndex, + allow_partial_search_results: false, body: { query: { match_all: {} @@ -242,6 +295,7 @@ async function checkOpenSearchReadiness({ signal } = {}) { operation: 'readiness' } ); + validateOpenSearchResponse(response.body); return true; } @@ -250,6 +304,10 @@ function isSearchProviderContractError(err) { return err instanceof SearchProviderContractError || err?.code === 'SEARCH_PROVIDER_CONTRACT'; } +function isOpenSearchResponseError(err) { + return err instanceof OpenSearchResponseError || err?.code === 'OPEN_SEARCH_RESPONSE_INVALID'; +} + function getOpenSearchReadinessTimeoutMs() { return getIntegerConfig('search.readinessTimeoutMs', constants.SEARCH_READINESS_TIMEOUT_MS, { min: 1 @@ -328,11 +386,15 @@ module.exports = { getSearchRequestDeadlineMs, getSearchProviderConfig, getSearchReader, + isOpenSearchResponseError, isSearchProviderContractError, normalizeBoolean, resetSearchProviderForTests, searchOpenSearch, shouldLogSearchPerformance, + validateOpenSearchResponse, validateSearchReader, + OPEN_SEARCH_RESPONSE_FAILURE_TYPES, + OpenSearchResponseError, SearchProviderContractError }; diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index c817ad9..e3d7c74 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -137,6 +137,7 @@ describe('searchProvider', () => { expect(mockClientSearch).toHaveBeenCalledWith( { index: 'test-index', + allow_partial_search_results: false, body }, { @@ -166,6 +167,45 @@ describe('searchProvider', () => { }); }); + test.each([ + [ + 'timed-out', + { + timed_out: true, + hits: { + hits: [] + } + }, + 'timeout' + ], + [ + 'partial-shard', + { + _shards: { + failed: 1 + }, + hits: { + hits: [] + } + }, + 'partial_shards' + ], + [ + 'malformed', + { + hits: {} + }, + 'malformed_response' + ] + ])('rejects a %s OpenSearch response', async (_caseName, body, failureType) => { + mockClientSearch.mockResolvedValue({ body }); + + await expect(searchProvider.searchOpenSearch({ query: { match_all: {} } })).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType + }); + }); + test('checks OpenSearch readiness through the configured CVE index', async () => { mockClientSearch.mockResolvedValue({ body: { @@ -180,6 +220,7 @@ describe('searchProvider', () => { expect(mockClientSearch).toHaveBeenCalledWith( { index: 'test-index', + allow_partial_search_results: false, body: { query: { match_all: {} @@ -200,6 +241,24 @@ describe('searchProvider', () => { }); }); + test('fails readiness when OpenSearch reports a partial response', async () => { + mockClientSearch.mockResolvedValue({ + body: { + _shards: { + failed: 1 + }, + hits: { + hits: [] + } + } + }); + + await expect(searchProvider.checkOpenSearchReadiness()).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'partial_shards' + }); + }); + test('fails readiness when the configured CVE index is unavailable', async () => { const unavailable = new Error('index unavailable'); mockClientSearch.mockRejectedValue(unavailable); From e19399d35e428ffcdcce8749be92b02819b07e8d Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 10:54:19 -0400 Subject: [PATCH 02/33] Harden structured search filter validation --- README.md | 10 +- api-docs/openapi.json | 36 +++--- api-docs/openapi.test.js | 14 ++- controllers/middleware.js | 45 ++++---- controllers/middleware.test.js | 55 ++++++++- controllers/searchController.test.js | 87 ++++----------- routes/cpeName.test.js | 2 +- routes/index.js | 2 +- routes/search.integration.test.js | 54 +++++++++ routes/swagger.js | 45 ++++---- utils/constants.js | 18 ++- utils/cpe.js | 15 +++ utils/cpe.test.js | 12 ++ utils/cvss.js | 71 ++++++++++++ utils/cvss.test.js | 63 +++++++++++ utils/errors.js | 13 ++- utils/errors.test.js | 19 +++- utils/searchQueryBuilder.js | 70 ++++++++---- utils/searchQueryBuilder.test.js | 161 ++++++++++----------------- 19 files changed, 522 insertions(+), 270 deletions(-) create mode 100644 utils/cvss.js create mode 100644 utils/cvss.test.js diff --git a/README.md b/README.md index 6201913..0a837ac 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,10 @@ String inputs are bounded before query construction: Requests that exceed a limit return HTTP 400 before contacting OpenSearch. These bounds limit input size and query complexity without changing the documented CPE matching semantics. +CVSS vector filters accept full or partial vectors containing at least one unique `KEY:VALUE` metric token. An optional vector header must identify a version supported by the selected parameter. Requested metric tokens can appear in any order, but all tokens must occur in the same stored vector string. CVSS and CWE filters search CNA-provided data only; ADP-provided enrichment is not currently included. + +Publication and last-modified date filters support one-sided ranges. When both values are supplied, `pubStartDate` must be less than or equal to `pubEndDate`, and `lastModStartDate` must be less than or equal to `lastModEndDate`. + Use `cveIds` to request up to 100 CVE records by comma-separated CVE ID. ```bash @@ -127,11 +131,11 @@ curl -X POST "http://localhost:3000/search" \ -d '{"noRejected":true}' ``` -When filtering with `cpeName`, provide a CPE 2.3 name with non-wildcard `part`, `vendor`, `product`, and `version` components. Invalid `cpeName` values return HTTP 404. The API compares the provided CPE Name against CPE Match Criteria, including exact match strings and version range match strings when the indexed CVE data includes range fields. Use `virtualMatchString` when you intentionally need to search for partial CPE Match String values without those required components. +When filtering with `cpeName`, provide a CPE 2.3 name with part `a`, `h`, or `o` and non-wildcard `vendor`, `product`, and `version` components. Invalid `cpeName` values return HTTP 404. The API compares the provided CPE Name against CPE Match Criteria, including exact match strings and version range match strings when the indexed CVE data includes range fields. Use `virtualMatchString` when you intentionally need to search for partial CPE Match String values without those required components. ### CPE Name Search -Use `cpeName` when a user has a concrete asset CPE Name and needs CVE records whose CPE Match Criteria apply to that asset version. The `part`, `vendor`, `product`, and `version` components must all be concrete values, not `*`. +Use `cpeName` when a user has a concrete asset CPE Name and needs CVE records whose CPE Match Criteria apply to that asset version. The part must be `a`, `h`, or `o`; the `vendor`, `product`, and `version` components must all be concrete values, not `*`. Valid `cpeName` example: @@ -157,7 +161,7 @@ Set `isVulnerable` to `true` with `cpeName` to require the matching CPE Match Cr ### Virtual Match String Search -Use `virtualMatchString` when a user needs CPE Match Criteria compatibility matching instead of concrete asset matching. String literal components are compared without regard to lexical case. Omitted trailing components and `*` components act as broad matches. +Use `virtualMatchString` when a user needs CPE Match Criteria compatibility matching instead of concrete asset matching. String literal components are compared without regard to lexical case. When a part is supplied, it must be `a`, `h`, `o`, `*`, or `-`. Omitted trailing components and `*` components act as broad matches. Partial product example: diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 61279e4..d3c4010 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -36,7 +36,7 @@ } }, "400": { - "description": "Malformed JSON, invalid request body, parameter combination, or version range" + "description": "Malformed JSON or invalid request body, parameter, CVSS vector, date range, or version range" }, "404": { "description": "Invalid cpeName value" @@ -327,7 +327,7 @@ "type": "string", "maxLength": 32, "example": "CWE-79", - "description": "Returns CVE records that include the provided Common Weakness Enumeration ID." + "description": "Returns CVE records whose CNA data includes the provided Common Weakness Enumeration ID. ADP-provided CWE data is not searched." }, "cvssV2Severity": { "type": "string", @@ -336,7 +336,7 @@ "MEDIUM", "HIGH" ], - "description": "Returns CVE records that contain the provided CVSS v2 qualitative severity rating." + "description": "Returns CVE records whose CNA data contains the provided CVSS v2 qualitative severity rating. ADP-provided CVSS data is not searched." }, "cvssV3Severity": { "type": "string", @@ -346,7 +346,7 @@ "HIGH", "CRITICAL" ], - "description": "Returns CVE records that contain the provided CVSS v3.0 or v3.1 qualitative severity rating." + "description": "Returns CVE records whose CNA data contains the provided CVSS v3.0 or v3.1 qualitative severity rating. ADP-provided CVSS data is not searched." }, "cvssV3_0Severity": { "type": "string", @@ -356,7 +356,7 @@ "HIGH", "CRITICAL" ], - "description": "Returns CVE records that contain the provided CVSS v3.0 qualitative severity rating." + "description": "Returns CVE records whose CNA data contains the provided CVSS v3.0 qualitative severity rating. ADP-provided CVSS data is not searched." }, "cvssV3_1Severity": { "type": "string", @@ -366,7 +366,7 @@ "HIGH", "CRITICAL" ], - "description": "Returns CVE records that contain the provided CVSS v3.1 qualitative severity rating." + "description": "Returns CVE records whose CNA data contains the provided CVSS v3.1 qualitative severity rating. ADP-provided CVSS data is not searched." }, "cvssV4Severity": { "type": "string", @@ -376,43 +376,43 @@ "HIGH", "CRITICAL" ], - "description": "Returns CVE records that contain the provided CVSS v4.0 qualitative severity rating." + "description": "Returns CVE records whose CNA data contains the provided CVSS v4.0 qualitative severity rating. ADP-provided CVSS data is not searched." }, "cvssV2Metrics": { "type": "string", "maxLength": 256, "example": "AV:N/AC:L/Au:N/C:P/I:P/A:P", - "description": "Returns CVE records that contain the provided CVSS v2 vector string. Values are limited to 256 characters, 32 slash-delimited metric tokens, and 32 characters per metric token." + "description": "Returns CVE records whose CNA data contains a CVSS v2 vector with every provided metric token in the same stored vector string. ADP-provided CVSS data is not searched. Full and partial vectors are accepted, tokens are order independent, and at least one unique KEY:VALUE token is required. Values are limited to 256 characters, 32 slash-delimited metric tokens, and 32 characters per metric token." }, "cvssV3Metrics": { "type": "string", "maxLength": 256, "example": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "description": "Returns CVE records that contain the provided CVSS v3.0 or v3.1 vector string. Values are limited to 256 characters, 32 slash-delimited metric tokens, and 32 characters per metric token." + "description": "Returns CVE records whose CNA data contains a CVSS v3.0 or v3.1 vector with every provided metric token in the same stored vector string. ADP-provided CVSS data is not searched. Full and partial vectors are accepted, tokens are order independent, and at least one unique KEY:VALUE token is required. When present, the header must specify CVSS 3.0 or 3.1 and restricts the searched version. Values are limited to 256 characters, 32 slash-delimited metric tokens, and 32 characters per metric token." }, "cvssV3_0Metrics": { "type": "string", "maxLength": 256, "example": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "description": "Returns CVE records that contain the provided CVSS v3.0 vector string. Values are limited to 256 characters, 32 slash-delimited metric tokens, and 32 characters per metric token." + "description": "Returns CVE records whose CNA data contains a CVSS v3.0 vector with every provided metric token in the same stored vector string. ADP-provided CVSS data is not searched. Full and partial vectors are accepted, tokens are order independent, and at least one unique KEY:VALUE token is required. When present, the header must specify CVSS 3.0. Values are limited to 256 characters, 32 slash-delimited metric tokens, and 32 characters per metric token." }, "cvssV3_1Metrics": { "type": "string", "maxLength": 256, "example": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "description": "Returns CVE records that contain the provided CVSS v3.1 vector string. Values are limited to 256 characters, 32 slash-delimited metric tokens, and 32 characters per metric token." + "description": "Returns CVE records whose CNA data contains a CVSS v3.1 vector with every provided metric token in the same stored vector string. ADP-provided CVSS data is not searched. Full and partial vectors are accepted, tokens are order independent, and at least one unique KEY:VALUE token is required. When present, the header must specify CVSS 3.1. Values are limited to 256 characters, 32 slash-delimited metric tokens, and 32 characters per metric token." }, "cvssV4Metrics": { "type": "string", "maxLength": 256, "example": "CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N", - "description": "Returns CVE records that contain the provided CVSS v4.0 vector string. Values are limited to 256 characters, 32 slash-delimited metric tokens, and 32 characters per metric token." + "description": "Returns CVE records whose CNA data contains a CVSS v4.0 vector with every provided metric token in the same stored vector string. ADP-provided CVSS data is not searched. Full and partial vectors are accepted, tokens are order independent, and at least one unique KEY:VALUE token is required. When present, the header must specify CVSS 4.0. Values are limited to 256 characters, 32 slash-delimited metric tokens, and 32 characters per metric token." }, "cpeName": { "type": "string", "maxLength": 2048, "example": "cpe:2.3:a:freetype:freetype:2.8.1:*:*:*:*:*:*:*", - "description": "Returns CVE records whose CPE Match Criteria apply to the provided CPE 2.3 asset name. CPE string literal comparisons are insensitive to lexical case. The part, vendor, product, and version components must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches." + "description": "Returns CVE records whose CPE Match Criteria apply to the provided CPE 2.3 asset name. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches." }, "isVulnerable": { "type": "boolean", @@ -423,25 +423,25 @@ "type": "string", "format": "date-time", "example": "2024-01-01T00:00:00.000Z", - "description": "Returns CVE records with dateUpdated timestamps on or after this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ." + "description": "Returns CVE records with dateUpdated timestamps on or after this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When lastModEndDate is also provided, this value must be less than or equal to it." }, "lastModEndDate": { "type": "string", "format": "date-time", "example": "2024-12-31T23:59:59.999Z", - "description": "Returns CVE records with dateUpdated timestamps on or before this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ." + "description": "Returns CVE records with dateUpdated timestamps on or before this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When lastModStartDate is also provided, that value must be less than or equal to this one." }, "pubStartDate": { "type": "string", "format": "date-time", "example": "2024-01-01T00:00:00.000Z", - "description": "Returns CVE records with datePublished timestamps on or after this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ." + "description": "Returns CVE records with datePublished timestamps on or after this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When pubEndDate is also provided, this value must be less than or equal to it." }, "pubEndDate": { "type": "string", "format": "date-time", "example": "2024-12-31T23:59:59.999Z", - "description": "Returns CVE records with datePublished timestamps on or before this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ." + "description": "Returns CVE records with datePublished timestamps on or before this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When pubStartDate is also provided, that value must be less than or equal to this one." }, "resultsPerPage": { "type": "integer", @@ -490,7 +490,7 @@ "type": "string", "maxLength": 2048, "example": "cpe:2.3:o:microsoft:windows_10_1809", - "description": "Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. Omitted trailing components and * components act as broad matches. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component." + "description": "Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component." } } } diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 0b49ee4..5029f32 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -78,6 +78,7 @@ describe('OpenAPI contract', () => { expect(schema.properties.isVulnerable.description).toContain('same CPE Match Criteria entry'); expect(schema.properties.isVulnerable.description).toContain('False or omitted'); expect(schema.properties.cpeName.description).toContain('Maven-style ordering'); + expect(schema.properties.cpeName.description).toContain('part must be a, h, or o'); expect(schema.properties.cpeName.description).toContain( 'Every concrete non-version component must be compatible with the same CPE Match Criteria entry' ); @@ -102,7 +103,16 @@ describe('OpenAPI contract', () => { expect(schema.properties.virtualMatchString.description).toContain('compatible'); expect(schema.properties.virtualMatchString.description).toContain('insensitive to lexical case'); expect(schema.properties.virtualMatchString.description).toContain('without a version component'); + expect(schema.properties.virtualMatchString.description).toContain('part must be a, h, o, *, or -'); expect(schema.properties.virtualMatchString.example).toBe('cpe:2.3:o:microsoft:windows_10_1809'); + expect(schema.properties.cweId.description).toContain('CNA data'); + expect(schema.properties.cweId.description).toContain('ADP-provided CWE data is not searched'); + expect(schema.properties.cvssV3Metrics.description).toContain('same stored vector string'); + expect(schema.properties.cvssV3Metrics.description).toContain('CNA data'); + expect(schema.properties.cvssV3Metrics.description).toContain('ADP-provided CVSS data is not searched'); + expect(schema.properties.cvssV3Metrics.description).toContain('at least one unique KEY:VALUE token'); + expect(schema.properties.pubStartDate.description).toContain('less than or equal'); + expect(schema.properties.lastModEndDate.description).toContain('less than or equal'); expect(Object.keys(mediaType.examples)).toStrictEqual([ 'allParameters', 'keywordSearch', @@ -133,7 +143,7 @@ describe('OpenAPI contract', () => { const cpeName = openapi.components.schemas.SearchRequest.properties.cpeName; expect(cpeName.description).toContain('CPE Match Criteria'); - expect(cpeName.description).toContain('part, vendor, product, and version'); + expect(cpeName.description).toContain('part must be a, h, or o'); expect(cpeName.description).toContain('version range fields'); expect(cpeName.description).toContain('insensitive to lexical case'); expect(cpeName.description).toContain('exact CPE Match Criteria preserve leading zeros'); @@ -150,6 +160,8 @@ describe('OpenAPI contract', () => { const responses = openapi.paths['/search'].post.responses; expect(responses['400'].description).toContain('Malformed JSON'); + expect(responses['400'].description).toContain('CVSS vector'); + expect(responses['400'].description).toContain('date range'); expect(responses['415'].description).toContain('application/json'); expect(responses['503'].description).toContain('OpenSearch'); expect(responses['503'].headers['Retry-After']).toStrictEqual({ diff --git a/controllers/middleware.js b/controllers/middleware.js index 8eedf02..524d052 100644 --- a/controllers/middleware.js +++ b/controllers/middleware.js @@ -1,6 +1,7 @@ const errors = require('../utils/errors'); const constants = require('../utils/constants'); const cpe = require('../utils/cpe'); +const cvss = require('../utils/cvss'); const error = new errors.errors(); const cvssV2SeverityEnum = ['LOW', 'MEDIUM', 'HIGH']; @@ -38,6 +39,10 @@ const severityValidators = [ const cvssMetricsParams = ['cvssV2Metrics', 'cvssV3Metrics', 'cvssV3_0Metrics', 'cvssV3_1Metrics', 'cvssV4Metrics']; const dateParams = ['lastModStartDate', 'lastModEndDate', 'pubStartDate', 'pubEndDate']; +const dateRangePairs = [ + ['lastModStartDate', 'lastModEndDate'], + ['pubStartDate', 'pubEndDate'] +]; const versionRangeParams = ['versionStart', 'versionStartType', 'versionEnd', 'versionEndType']; const versionRangeValueParams = ['versionStart', 'versionEnd']; const versionRangeTypeParams = ['versionStartType', 'versionEndType']; @@ -71,25 +76,6 @@ function hasAnyParam(params, paramList) { return paramList.some((param) => hasParam(params, param)); } -function hasInvalidCvssMetricsShape(value) { - const input = String(value ?? ''); - if (input.length > constants.MAX_CVSS_METRICS_LENGTH) return true; - - const tokens = input - .split('/') - .map((t) => t.trim()) - .filter(Boolean) - .filter((t) => { - const up = t.toUpperCase(); - return !up.startsWith('CVSS:') && !up.startsWith('CVSS#'); - }); - - return ( - tokens.length > constants.MAX_CVSS_METRIC_TOKENS || - tokens.some((t) => t.length > constants.MAX_CVSS_METRIC_TOKEN_LENGTH) - ); -} - function countWhitespaceDelimitedTokens(value) { return value.trim().split(/\s+/).length; } @@ -290,8 +276,15 @@ function validateParams(req, res, next) { } for (const param of cvssMetricsParams) { - if (params?.[param] && hasInvalidCvssMetricsShape(params[param])) { - return res.status(400).json(error.cvssMetricsLimitError(param)); + if (!params?.[param]) continue; + + const parsed = cvss.parseCvssMetrics(params[param], param); + if (!parsed.valid) { + const responseBody = + parsed.reason === 'limit' + ? error.cvssMetricsLimitError(param) + : error.cvssMetricsFormatError(param); + return res.status(400).json(responseBody); } } @@ -302,6 +295,16 @@ function validateParams(req, res, next) { } } + for (const [startParam, endParam] of dateRangePairs) { + if ( + params?.[startParam] && + params?.[endParam] && + new Date(params[startParam]).getTime() > new Date(params[endParam]).getTime() + ) { + return res.status(400).json(error.dateRangeOrderError(startParam, endParam)); + } + } + next(); } catch (err) { next(err); diff --git a/controllers/middleware.test.js b/controllers/middleware.test.js index 5079be4..0261e9b 100644 --- a/controllers/middleware.test.js +++ b/controllers/middleware.test.js @@ -93,6 +93,7 @@ describe('middleware validation function', () => { test.each([ ['malformed CPE name', 'not-a-cpe'], + ['unsupported part', 'cpe:2.3:x:vendor:product:1.0:*:*:*:*:*:*:*'], ['wildcard part', 'cpe:2.3:*:vendor:product:1.0:*:*:*:*:*:*:*'], ['wildcard vendor', 'cpe:2.3:a:*:product:1.0:*:*:*:*:*:*:*'], ['wildcard product', 'cpe:2.3:a:vendor:*:1.0:*:*:*:*:*:*:*'], @@ -104,11 +105,19 @@ describe('middleware validation function', () => { expectValidationError( 'INVALID_CPE_NAME', - 'The cpeName parameter must be a CPE 2.3 name with non-wildcard part, vendor, product, and version components.', + 'The cpeName parameter must be a CPE 2.3 name with part a, h, or o and non-wildcard vendor, product, and version components.', 404 ); }); + test('ValidateParams accepts a cpeName with an uppercase valid part', () => { + validate({ + cpeName: 'cpe:2.3:A:vendor:product:1.0:*:*:*:*:*:*:*' + }); + + expectValidationSuccess(); + }); + test('ValidateParams accepts an incomplete cpeName when required components are present', () => { validate({ cpeName: 'cpe:2.3:a:vendor:product:1.0' @@ -802,6 +811,23 @@ describe('middleware validation function', () => { ); }); + test.each([ + ['header-only value', 'cvssV3Metrics', 'CVSS:3.1'], + ['slash-only value', 'cvssV3Metrics', '////'], + ['malformed token', 'cvssV4Metrics', 'AV'], + ['duplicate metric', 'cvssV2Metrics', 'AV:N/av:L'], + ['incompatible version header', 'cvssV3_0Metrics', 'CVSS:3.1/AV:N'] + ])('ValidateParams rejects a CVSS metrics %s.', (_caseName, param, value) => { + validate({ + [param]: value + }); + + expectValidationError( + 'CVSS_METRICS_FORMAT', + `The ${param} parameter must contain at least one unique KEY:VALUE metric token and use a compatible CVSS version header when a header is provided.` + ); + }); + test.each([['lastModStartDate'], ['lastModEndDate'], ['pubStartDate'], ['pubEndDate']])( 'ValidateParams returns an error when an invalid dateTime value is passed for %s.', (dateParam) => { @@ -816,6 +842,33 @@ describe('middleware validation function', () => { } ); + test.each([ + ['lastModStartDate', 'lastModEndDate'], + ['pubStartDate', 'pubEndDate'] + ])('ValidateParams rejects a reversed %s and %s range.', (startParam, endParam) => { + validate({ + [startParam]: '2024-01-02T00:00:00.000Z', + [endParam]: '2024-01-01T00:00:00.000Z' + }); + + expectValidationError( + 'DATE_RANGE_ORDER', + `The ${startParam} parameter must be less than or equal to ${endParam}.` + ); + }); + + test.each([ + ['lastModStartDate', 'lastModEndDate'], + ['pubStartDate', 'pubEndDate'] + ])('ValidateParams accepts equal %s and %s values.', (startParam, endParam) => { + validate({ + [startParam]: '2024-01-01T00:00:00.000Z', + [endParam]: '2024-01-01T00:00:00.000Z' + }); + + expectValidationSuccess(); + }); + test.each([ ['2024'], ['2024-01'], diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 2016506..8bc6aeb 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -80,6 +80,23 @@ function rawResponse(totalValue, hits = []) { return advancedResponse(totalValue, hits).data; } +function cvssRegexp(field, value) { + return { + regexp: { + [`${field}.keyword`]: { + value, + case_insensitive: true, + flags: 'INTERSECTION|ANYSTRING', + rewrite: 'constant_score' + } + } + }; +} + +function cvssValue(...tokens) { + return tokens.map((token) => `(@/)?${token}(/@)?`).join('&'); +} + function basicResponse({ ok = true, status = 'ok', total = 0, hits = [], notes = [], errors = [] } = {}) { return { status, @@ -1278,33 +1295,7 @@ describe('Search controller SEARCH handler', () => { ); expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - bool: { - filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*AC:L*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*Au:N*' - } - } - } - ] - } - } + cvssRegexp('containers.cna.metrics.cvssV2_0.vectorString', cvssValue('AV:N', 'AC:L', 'Au:N')) ]); }); @@ -1323,46 +1314,8 @@ describe('Search controller SEARCH handler', () => { ); expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - bool: { - filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV3_0.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV3_0.vectorString.keyword': { - value: '*AC:L*' - } - } - } - ] - } - }, - { - bool: { - filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV3_1.vectorString.keyword': { - value: '*PR:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV3_1.vectorString.keyword': { - value: '*UI:R*' - } - } - } - ] - } - } + cvssRegexp('containers.cna.metrics.cvssV3_0.vectorString', cvssValue('AV:N', 'AC:L')), + cvssRegexp('containers.cna.metrics.cvssV3_1.vectorString', cvssValue('PR:N', 'UI:R')) ]); }); diff --git a/routes/cpeName.test.js b/routes/cpeName.test.js index 89cc369..4e0d82e 100644 --- a/routes/cpeName.test.js +++ b/routes/cpeName.test.js @@ -35,7 +35,7 @@ describe('POST /search cpeName route behavior', () => { expect(response.body).toStrictEqual({ error: 'INVALID_CPE_NAME', message: - 'The cpeName parameter must be a CPE 2.3 name with non-wildcard part, vendor, product, and version components.' + 'The cpeName parameter must be a CPE 2.3 name with part a, h, or o and non-wildcard vendor, product, and version components.' }); expect(controller.SEARCH).not.toHaveBeenCalled(); }); diff --git a/routes/index.js b/routes/index.js index adca945..62df1e8 100644 --- a/routes/index.js +++ b/routes/index.js @@ -246,7 +246,7 @@ headers: { } } #swagger.responses[400] = { -description: 'Malformed JSON, invalid request body, parameter combination, or version range', +description: 'Malformed JSON or invalid request body, parameter, CVSS vector, date range, or version range', } #swagger.responses[404] = { description: 'Invalid cpeName value', diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index 5319044..6df96c6 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -113,6 +113,60 @@ describe('search route integration', () => { expect(mockSearchOpenSearch).not.toHaveBeenCalled(); }); + test('POST /search rejects reversed date ranges before invoking OpenSearch', async () => { + const response = await request(createApp(), { + method: 'POST', + path: '/search', + body: { + pubStartDate: '2024-02-01T00:00:00.000Z', + pubEndDate: '2024-01-01T00:00:00.000Z' + } + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'DATE_RANGE_ORDER', + message: 'The pubStartDate parameter must be less than or equal to pubEndDate.' + }); + expect(mockSearchOpenSearch).not.toHaveBeenCalled(); + }); + + test('POST /search rejects CVSS headers without metric tokens before invoking OpenSearch', async () => { + const response = await request(createApp(), { + method: 'POST', + path: '/search', + body: { + cvssV3Metrics: 'CVSS:3.1' + } + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'CVSS_METRICS_FORMAT', + message: + 'The cvssV3Metrics parameter must contain at least one unique KEY:VALUE metric token and use a compatible CVSS version header when a header is provided.' + }); + expect(mockSearchOpenSearch).not.toHaveBeenCalled(); + }); + + test('POST /search rejects unsupported CPE parts before invoking OpenSearch', async () => { + const response = await request(createApp(), { + method: 'POST', + path: '/search', + body: { + cpeName: 'cpe:2.3:x:vendor:product:1.0:*:*:*:*:*:*:*' + } + }); + + expect(response.status).toBe(404); + expect(response.body).toStrictEqual({ + error: 'INVALID_CPE_NAME', + message: + 'The cpeName parameter must be a CPE 2.3 name with part a, h, or o and non-wildcard vendor, product, and version components.' + }); + expect(mockSearchOpenSearch).not.toHaveBeenCalled(); + }); + test('POST /search rejects excessive keyword terms before invoking OpenSearch', async () => { const response = await request(createApp(), { method: 'POST', diff --git a/routes/swagger.js b/routes/swagger.js index 1cf65db..a4cfed3 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -42,48 +42,51 @@ const searchRequestProperties = { type: 'boolean', default: false }), - cweId: searchRequestProperty('Returns CVE records that include the provided Common Weakness Enumeration ID.', { - type: 'string', - maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cweId, - example: 'CWE-79' - }), + cweId: searchRequestProperty( + 'Returns CVE records whose CNA data includes the provided Common Weakness Enumeration ID. ADP-provided CWE data is not searched.', + { + type: 'string', + maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cweId, + example: 'CWE-79' + } + ), cvssV2Severity: searchRequestProperty( - 'Returns CVE records that contain the provided CVSS v2 qualitative severity rating.', + 'Returns CVE records whose CNA data contains the provided CVSS v2 qualitative severity rating. ADP-provided CVSS data is not searched.', { type: 'string', enum: ['LOW', 'MEDIUM', 'HIGH'] } ), cvssV3Severity: searchRequestProperty( - 'Returns CVE records that contain the provided CVSS v3.0 or v3.1 qualitative severity rating.', + 'Returns CVE records whose CNA data contains the provided CVSS v3.0 or v3.1 qualitative severity rating. ADP-provided CVSS data is not searched.', { type: 'string', enum: ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL'] } ), cvssV3_0Severity: searchRequestProperty( - 'Returns CVE records that contain the provided CVSS v3.0 qualitative severity rating.', + 'Returns CVE records whose CNA data contains the provided CVSS v3.0 qualitative severity rating. ADP-provided CVSS data is not searched.', { type: 'string', enum: ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL'] } ), cvssV3_1Severity: searchRequestProperty( - 'Returns CVE records that contain the provided CVSS v3.1 qualitative severity rating.', + 'Returns CVE records whose CNA data contains the provided CVSS v3.1 qualitative severity rating. ADP-provided CVSS data is not searched.', { type: 'string', enum: ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL'] } ), cvssV4Severity: searchRequestProperty( - 'Returns CVE records that contain the provided CVSS v4.0 qualitative severity rating.', + 'Returns CVE records whose CNA data contains the provided CVSS v4.0 qualitative severity rating. ADP-provided CVSS data is not searched.', { type: 'string', enum: ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL'] } ), cvssV2Metrics: searchRequestProperty( - `Returns CVE records that contain the provided CVSS v2 vector string. Values are limited to ${constants.MAX_CVSS_METRICS_LENGTH} characters, ${constants.MAX_CVSS_METRIC_TOKENS} slash-delimited metric tokens, and ${constants.MAX_CVSS_METRIC_TOKEN_LENGTH} characters per metric token.`, + `Returns CVE records whose CNA data contains a CVSS v2 vector with every provided metric token in the same stored vector string. ADP-provided CVSS data is not searched. Full and partial vectors are accepted, tokens are order independent, and at least one unique KEY:VALUE token is required. Values are limited to ${constants.MAX_CVSS_METRICS_LENGTH} characters, ${constants.MAX_CVSS_METRIC_TOKENS} slash-delimited metric tokens, and ${constants.MAX_CVSS_METRIC_TOKEN_LENGTH} characters per metric token.`, { type: 'string', maxLength: constants.MAX_CVSS_METRICS_LENGTH, @@ -91,7 +94,7 @@ const searchRequestProperties = { } ), cvssV3Metrics: searchRequestProperty( - `Returns CVE records that contain the provided CVSS v3.0 or v3.1 vector string. Values are limited to ${constants.MAX_CVSS_METRICS_LENGTH} characters, ${constants.MAX_CVSS_METRIC_TOKENS} slash-delimited metric tokens, and ${constants.MAX_CVSS_METRIC_TOKEN_LENGTH} characters per metric token.`, + `Returns CVE records whose CNA data contains a CVSS v3.0 or v3.1 vector with every provided metric token in the same stored vector string. ADP-provided CVSS data is not searched. Full and partial vectors are accepted, tokens are order independent, and at least one unique KEY:VALUE token is required. When present, the header must specify CVSS 3.0 or 3.1 and restricts the searched version. Values are limited to ${constants.MAX_CVSS_METRICS_LENGTH} characters, ${constants.MAX_CVSS_METRIC_TOKENS} slash-delimited metric tokens, and ${constants.MAX_CVSS_METRIC_TOKEN_LENGTH} characters per metric token.`, { type: 'string', maxLength: constants.MAX_CVSS_METRICS_LENGTH, @@ -99,7 +102,7 @@ const searchRequestProperties = { } ), cvssV3_0Metrics: searchRequestProperty( - `Returns CVE records that contain the provided CVSS v3.0 vector string. Values are limited to ${constants.MAX_CVSS_METRICS_LENGTH} characters, ${constants.MAX_CVSS_METRIC_TOKENS} slash-delimited metric tokens, and ${constants.MAX_CVSS_METRIC_TOKEN_LENGTH} characters per metric token.`, + `Returns CVE records whose CNA data contains a CVSS v3.0 vector with every provided metric token in the same stored vector string. ADP-provided CVSS data is not searched. Full and partial vectors are accepted, tokens are order independent, and at least one unique KEY:VALUE token is required. When present, the header must specify CVSS 3.0. Values are limited to ${constants.MAX_CVSS_METRICS_LENGTH} characters, ${constants.MAX_CVSS_METRIC_TOKENS} slash-delimited metric tokens, and ${constants.MAX_CVSS_METRIC_TOKEN_LENGTH} characters per metric token.`, { type: 'string', maxLength: constants.MAX_CVSS_METRICS_LENGTH, @@ -107,7 +110,7 @@ const searchRequestProperties = { } ), cvssV3_1Metrics: searchRequestProperty( - `Returns CVE records that contain the provided CVSS v3.1 vector string. Values are limited to ${constants.MAX_CVSS_METRICS_LENGTH} characters, ${constants.MAX_CVSS_METRIC_TOKENS} slash-delimited metric tokens, and ${constants.MAX_CVSS_METRIC_TOKEN_LENGTH} characters per metric token.`, + `Returns CVE records whose CNA data contains a CVSS v3.1 vector with every provided metric token in the same stored vector string. ADP-provided CVSS data is not searched. Full and partial vectors are accepted, tokens are order independent, and at least one unique KEY:VALUE token is required. When present, the header must specify CVSS 3.1. Values are limited to ${constants.MAX_CVSS_METRICS_LENGTH} characters, ${constants.MAX_CVSS_METRIC_TOKENS} slash-delimited metric tokens, and ${constants.MAX_CVSS_METRIC_TOKEN_LENGTH} characters per metric token.`, { type: 'string', maxLength: constants.MAX_CVSS_METRICS_LENGTH, @@ -115,7 +118,7 @@ const searchRequestProperties = { } ), cvssV4Metrics: searchRequestProperty( - `Returns CVE records that contain the provided CVSS v4.0 vector string. Values are limited to ${constants.MAX_CVSS_METRICS_LENGTH} characters, ${constants.MAX_CVSS_METRIC_TOKENS} slash-delimited metric tokens, and ${constants.MAX_CVSS_METRIC_TOKEN_LENGTH} characters per metric token.`, + `Returns CVE records whose CNA data contains a CVSS v4.0 vector with every provided metric token in the same stored vector string. ADP-provided CVSS data is not searched. Full and partial vectors are accepted, tokens are order independent, and at least one unique KEY:VALUE token is required. When present, the header must specify CVSS 4.0. Values are limited to ${constants.MAX_CVSS_METRICS_LENGTH} characters, ${constants.MAX_CVSS_METRIC_TOKENS} slash-delimited metric tokens, and ${constants.MAX_CVSS_METRIC_TOKEN_LENGTH} characters per metric token.`, { type: 'string', maxLength: constants.MAX_CVSS_METRICS_LENGTH, @@ -123,7 +126,7 @@ const searchRequestProperties = { } ), cpeName: searchRequestProperty( - 'Returns CVE records whose CPE Match Criteria apply to the provided CPE 2.3 asset name. CPE string literal comparisons are insensitive to lexical case. The part, vendor, product, and version components must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches.', + 'Returns CVE records whose CPE Match Criteria apply to the provided CPE 2.3 asset name. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches.', { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cpeName, @@ -138,7 +141,7 @@ const searchRequestProperties = { } ), lastModStartDate: searchRequestProperty( - 'Returns CVE records with dateUpdated timestamps on or after this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ.', + 'Returns CVE records with dateUpdated timestamps on or after this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When lastModEndDate is also provided, this value must be less than or equal to it.', { type: 'string', format: 'date-time', @@ -146,7 +149,7 @@ const searchRequestProperties = { } ), lastModEndDate: searchRequestProperty( - 'Returns CVE records with dateUpdated timestamps on or before this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ.', + 'Returns CVE records with dateUpdated timestamps on or before this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When lastModStartDate is also provided, that value must be less than or equal to this one.', { type: 'string', format: 'date-time', @@ -154,7 +157,7 @@ const searchRequestProperties = { } ), pubStartDate: searchRequestProperty( - 'Returns CVE records with datePublished timestamps on or after this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ.', + 'Returns CVE records with datePublished timestamps on or after this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When pubEndDate is also provided, this value must be less than or equal to it.', { type: 'string', format: 'date-time', @@ -162,7 +165,7 @@ const searchRequestProperties = { } ), pubEndDate: searchRequestProperty( - 'Returns CVE records with datePublished timestamps on or before this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ.', + 'Returns CVE records with datePublished timestamps on or before this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When pubStartDate is also provided, that value must be less than or equal to this one.', { type: 'string', format: 'date-time', @@ -219,7 +222,7 @@ const searchRequestProperties = { } ), virtualMatchString: searchRequestProperty( - 'Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. Omitted trailing components and * components act as broad matches. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component.', + 'Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component.', { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.virtualMatchString, diff --git a/utils/constants.js b/utils/constants.js index 85ae9f3..8f970c3 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -29,6 +29,13 @@ const rejectedCveStates = ['REJECT', 'REJECTED', 'Rejected']; const rejectedCveStateFields = ['cveMetadata.state', 'containers.cna.x_legacyV4Record.CVE_data_meta.STATE']; const cpeVersionRangeTypes = ['including', 'excluding']; +const cvssVectorVersionFields = Object.freeze({ + '2.0': ['containers.cna.metrics.cvssV2_0.vectorString'], + '3.0': ['containers.cna.metrics.cvssV3_0.vectorString'], + 3.1: ['containers.cna.metrics.cvssV3_1.vectorString'], + '4.0': ['containers.cna.metrics.cvssV4_0.vectorString'] +}); + const cpeMatchFields = { criteria: 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria', vulnerable: 'containers.cna.cpeApplicability.nodes.cpeMatch.vulnerable', @@ -68,15 +75,15 @@ const fields = { noRejected: rejectedCveStateFields, cweId: 'containers.cna.problemTypes.descriptions.cweId', cvssV2Severity: 'containers.cna.metrics.cvssV2_0.baseScore', - cvssV2Metrics: 'containers.cna.metrics.cvssV2_0.vectorString', + cvssV2Metrics: cvssVectorVersionFields['2.0'][0], cvssV3_0Severity: 'containers.cna.metrics.cvssV3_0.baseSeverity', - cvssV3_0Metrics: 'containers.cna.metrics.cvssV3_0.vectorString', + cvssV3_0Metrics: cvssVectorVersionFields['3.0'][0], cvssV3_1Severity: 'containers.cna.metrics.cvssV3_1.baseSeverity', - cvssV3_1Metrics: 'containers.cna.metrics.cvssV3_1.vectorString', + cvssV3_1Metrics: cvssVectorVersionFields['3.1'][0], cvssV3Severity: ['containers.cna.metrics.cvssV3_0.baseSeverity', 'containers.cna.metrics.cvssV3_1.baseSeverity'], - cvssV3Metrics: ['containers.cna.metrics.cvssV3_0.vectorString', 'containers.cna.metrics.cvssV3_1.vectorString'], + cvssV3Metrics: [...cvssVectorVersionFields['3.0'], ...cvssVectorVersionFields['3.1']], cvssV4Severity: 'containers.cna.metrics.cvssV4_0.baseSeverity', - cvssV4Metrics: 'containers.cna.metrics.cvssV4_0.vectorString', + cvssV4Metrics: cvssVectorVersionFields['4.0'][0], cpeName: cpeMatchFields.criteria, isVulnerable: cpeMatchFields.vulnerable, virtualMatchString: cpeMatchFields.criteria @@ -149,6 +156,7 @@ module.exports = { REJECTED_CVE_STATES: rejectedCveStates, CPE_VERSION_RANGE_TYPES: cpeVersionRangeTypes, CPE_MATCH_FIELDS: cpeMatchFields, + CVSS_VECTOR_VERSION_FIELDS: cvssVectorVersionFields, FIELDS: fields, SEARCH_CONTROL_PARAMS: searchControlParams, SEARCH_QUERY_PARAMS: searchQueryParams diff --git a/utils/cpe.js b/utils/cpe.js index 7301566..97f0620 100644 --- a/utils/cpe.js +++ b/utils/cpe.js @@ -2,6 +2,7 @@ const CPE_23_PREFIX = ['cpe', '2.3']; const CPE_COMPONENT_COUNT = 11; const CPE_NOT_APPLICABLE = '-'; const REQUIRED_CPE_COMPONENT_INDEXES = [0, 1, 2, 3]; +const VALID_CPE_PARTS = new Set(['a', 'h', 'o']); const VERSION_QUALIFIERS = ['alpha', 'beta', 'milestone', 'rc', 'snapshot', '', 'sp']; const VERSION_QUALIFIER_ALIASES = { a: 'alpha', @@ -27,6 +28,9 @@ function parseCpe23(input) { if (components.length < 4 || components.length > CPE_COMPONENT_COUNT) { return { valid: false, reason: 'component_count' }; } + if (!isValidCpePart(components[0], { allowLogicalValues: true })) { + return { valid: false, reason: 'part' }; + } while (components.length < CPE_COMPONENT_COUNT) { components.push('*'); @@ -72,6 +76,7 @@ function splitOnUnescapedColon(s) { function parseCpeName(input) { const parsed = parseCpe23(input); if (!parsed.valid) return parsed; + if (!isValidCpePart(parsed.part)) return { valid: false, reason: 'part' }; const hasRequiredComponents = REQUIRED_CPE_COMPONENT_INDEXES.every((index) => { const component = parsed.components[index]; @@ -120,6 +125,9 @@ function parseVirtualMatchString(input) { if (rawComponents.length > CPE_COMPONENT_COUNT) { return { valid: false, reason: 'component_count' }; } + if (rawComponents.length && !isValidCpePart(rawComponents[0], { allowLogicalValues: true })) { + return { valid: false, reason: 'part' }; + } const hasVersionComponent = rawComponents.length >= 4; const components = [...rawComponents]; @@ -143,6 +151,13 @@ function isValidCpeName(input) { return parseCpeName(input).valid; } +function isValidCpePart(part, { allowLogicalValues = false } = {}) { + const normalized = String(part ?? '').toLowerCase(); + if (VALID_CPE_PARTS.has(normalized)) return true; + + return allowLogicalValues && ['*', CPE_NOT_APPLICABLE].includes(normalized); +} + function escapeLuceneRegexpLiteral(input) { return String(input ?? '').replace(/([.?+*|{}[\]()"\\#@&<>~])/g, '\\$1'); } diff --git a/utils/cpe.test.js b/utils/cpe.test.js index 916a0d9..caa429e 100644 --- a/utils/cpe.test.js +++ b/utils/cpe.test.js @@ -34,6 +34,7 @@ describe('CPE helpers', () => { test.each([ 'not-a-cpe', 'cpe:2.3:a:vendor:product', + 'cpe:2.3:x:vendor:product:1.0:*:*:*:*:*:*:*', 'cpe:2.3:*:vendor:product:1.0:*:*:*:*:*:*:*', 'cpe:2.3:a:*:product:1.0:*:*:*:*:*:*:*', 'cpe:2.3:a:vendor:*:1.0:*:*:*:*:*:*:*', @@ -44,6 +45,10 @@ describe('CPE helpers', () => { expect(cpe.isValidCpeName(value)).toBe(false); }); + test('parseCpeName accepts a valid uppercase part value', () => { + expect(cpe.parseCpeName('cpe:2.3:A:vendor:product:1.0:*:*:*:*:*:*:*').valid).toBe(true); + }); + test('parseCpeName preserves escaped colons inside required components', () => { const parsed = cpe.parseCpeName('cpe:2.3:a:ven\\:dor:product\\:edition:1.0:*:*:*:*:*:*:*'); @@ -79,6 +84,13 @@ describe('CPE helpers', () => { expect(parsed.components).toStrictEqual(['*', '*', '*', '*', '*', '*', '*', '*', '*', '*', '*']); }); + test('parseVirtualMatchString rejects an unsupported concrete part value', () => { + expect(cpe.parseVirtualMatchString('cpe:2.3:x:vendor:product')).toStrictEqual({ + valid: false, + reason: 'part' + }); + }); + test('splitOnUnescapedColon preserves escaped colons inside CPE components', () => { expect(cpe.splitOnUnescapedColon('a:vendor:product\\:edition:1.0')).toStrictEqual([ 'a', diff --git a/utils/cvss.js b/utils/cvss.js new file mode 100644 index 0000000..4a50d46 --- /dev/null +++ b/utils/cvss.js @@ -0,0 +1,71 @@ +const constants = require('./constants'); + +const CVSS_HEADER_PATTERN = /^CVSS[:#](2\.0|3\.0|3\.1|4\.0)$/i; +const CVSS_METRIC_TOKEN_PATTERN = /^([A-Za-z][A-Za-z0-9_]*):([^/:\s]+)$/; +const CVSS_PARAM_VERSIONS = Object.freeze({ + cvssV2Metrics: ['2.0'], + cvssV3Metrics: ['3.0', '3.1'], + cvssV3_0Metrics: ['3.0'], + cvssV3_1Metrics: ['3.1'], + cvssV4Metrics: ['4.0'] +}); + +function invalid(reason) { + return { + valid: false, + reason + }; +} + +function parseCvssMetrics(value, param) { + const input = String(value ?? '').trim(); + if (input.length > constants.MAX_CVSS_METRICS_LENGTH) return invalid('limit'); + + const parts = input.split('/').map((part) => part.trim()); + if (!parts.length || parts.some((part) => !part)) return invalid('format'); + + let version; + const headerMatch = parts[0].match(CVSS_HEADER_PATTERN); + if (headerMatch) { + version = headerMatch[1]; + parts.shift(); + } else if (/^CVSS[:#]/i.test(parts[0])) { + return invalid('format'); + } + + const allowedVersions = CVSS_PARAM_VERSIONS[param]; + if (!allowedVersions || (version && !allowedVersions.includes(version))) return invalid('format'); + if (!parts.length) return invalid('format'); + if (parts.length > constants.MAX_CVSS_METRIC_TOKENS) return invalid('limit'); + if (parts.some((part) => part.length > constants.MAX_CVSS_METRIC_TOKEN_LENGTH)) return invalid('limit'); + + const metricKeys = new Set(); + const tokens = []; + + for (const part of parts) { + if (/^CVSS[:#]/i.test(part)) return invalid('format'); + + const tokenMatch = part.match(CVSS_METRIC_TOKEN_PATTERN); + if (!tokenMatch) return invalid('format'); + + const key = tokenMatch[1].toUpperCase(); + if (metricKeys.has(key)) return invalid('format'); + + metricKeys.add(key); + tokens.push({ + key, + value: tokenMatch[2], + raw: part + }); + } + + return { + valid: true, + version, + tokens + }; +} + +module.exports = { + parseCvssMetrics +}; diff --git a/utils/cvss.test.js b/utils/cvss.test.js new file mode 100644 index 0000000..6e09cf5 --- /dev/null +++ b/utils/cvss.test.js @@ -0,0 +1,63 @@ +const cvss = require('./cvss'); + +describe('cvss', () => { + test.each([ + ['cvssV2Metrics', 'AV:N/AC:L/Au:N'], + ['cvssV2Metrics', 'CVSS:2.0/AV:N/AC:L'], + ['cvssV3Metrics', 'CVSS:3.0/AV:N/AC:L'], + ['cvssV3Metrics', 'CVSS:3.1/AC:L/AV:N'], + ['cvssV4Metrics', 'CVSS#4.0/AV:N/AC:L'] + ])('parseCvssMetrics accepts %s value %s', (param, value) => { + expect(cvss.parseCvssMetrics(value, param)).toMatchObject({ + valid: true, + tokens: expect.any(Array) + }); + }); + + test.each([ + ['header only', 'cvssV3Metrics', 'CVSS:3.1'], + ['empty slash tokens', 'cvssV3Metrics', '////'], + ['missing metric key', 'cvssV3Metrics', ':N'], + ['missing metric value', 'cvssV3Metrics', 'AV:'], + ['missing metric separator', 'cvssV3Metrics', 'AV'], + ['duplicate metric key', 'cvssV3Metrics', 'AV:N/av:L'], + ['header after a metric', 'cvssV3Metrics', 'AV:N/CVSS:3.1'], + ['unsupported header version', 'cvssV3Metrics', 'CVSS:4.0/AV:N'], + ['wrong parameter version', 'cvssV3_0Metrics', 'CVSS:3.1/AV:N'] + ])('parseCvssMetrics rejects %s', (_caseName, param, value) => { + expect(cvss.parseCvssMetrics(value, param)).toStrictEqual({ + valid: false, + reason: 'format' + }); + }); + + test.each([ + ['overall length', 'A'.repeat(257)], + ['token count', Array.from({ length: 33 }, (_value, index) => `T${index}:A`).join('/')], + ['token length', `AV:${'A'.repeat(33)}`] + ])('parseCvssMetrics enforces the %s limit', (_caseName, value) => { + expect(cvss.parseCvssMetrics(value, 'cvssV3Metrics')).toStrictEqual({ + valid: false, + reason: 'limit' + }); + }); + + test('parseCvssMetrics returns the version and structured tokens', () => { + expect(cvss.parseCvssMetrics('CVSS:3.1/AC:L/AV:N', 'cvssV3Metrics')).toStrictEqual({ + valid: true, + version: '3.1', + tokens: [ + { + key: 'AC', + value: 'L', + raw: 'AC:L' + }, + { + key: 'AV', + value: 'N', + raw: 'AV:N' + } + ] + }); + }); +}); diff --git a/utils/errors.js b/utils/errors.js index 889d702..5017ce0 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -105,7 +105,7 @@ class errors { invalidCpeNameError() { return makeError( 'INVALID_CPE_NAME', - 'The cpeName parameter must be a CPE 2.3 name with non-wildcard part, vendor, product, and version components.' + 'The cpeName parameter must be a CPE 2.3 name with part a, h, or o and non-wildcard vendor, product, and version components.' ); } @@ -225,6 +225,13 @@ class errors { ); } + cvssMetricsFormatError(param) { + return makeError( + 'CVSS_METRICS_FORMAT', + `The ${param} parameter must contain at least one unique KEY:VALUE metric token and use a compatible CVSS version header when a header is provided.` + ); + } + dateTimeError(dateTime) { return makeError( 'DATE_TIME_FORMAT', @@ -232,6 +239,10 @@ class errors { ); } + dateRangeOrderError(startParam, endParam) { + return makeError('DATE_RANGE_ORDER', `The ${startParam} parameter must be less than or equal to ${endParam}.`); + } + webSearchQueryError() { return makeError('WEB_SEARCH_QUERY_REQUIRED', 'The query field is required and must be a non-empty string.'); } diff --git a/utils/errors.test.js b/utils/errors.test.js index ec465b9..f950135 100644 --- a/utils/errors.test.js +++ b/utils/errors.test.js @@ -148,7 +148,7 @@ describe('error response helpers', () => { { error: 'INVALID_CPE_NAME', message: - 'The cpeName parameter must be a CPE 2.3 name with non-wildcard part, vendor, product, and version components.' + 'The cpeName parameter must be a CPE 2.3 name with part a, h, or o and non-wildcard vendor, product, and version components.' } ], [ @@ -331,6 +331,15 @@ describe('error response helpers', () => { message: `The cvssV3Metrics parameter must be ${constants.MAX_CVSS_METRICS_LENGTH} characters or fewer, contain ${constants.MAX_CVSS_METRIC_TOKENS} or fewer slash-delimited metric tokens, and each metric token must be ${constants.MAX_CVSS_METRIC_TOKEN_LENGTH} characters or fewer.` } ], + [ + 'cvssMetricsFormatError', + ['cvssV3Metrics'], + { + error: 'CVSS_METRICS_FORMAT', + message: + 'The cvssV3Metrics parameter must contain at least one unique KEY:VALUE metric token and use a compatible CVSS version header when a header is provided.' + } + ], [ 'dateTimeError', ['pubStartDate'], @@ -339,6 +348,14 @@ describe('error response helpers', () => { message: "The pubStartDate parameter must match the following format: yyyy-MM-ddTHH:mm:ss.SSS'Z'" } ], + [ + 'dateRangeOrderError', + ['pubStartDate', 'pubEndDate'], + { + error: 'DATE_RANGE_ORDER', + message: 'The pubStartDate parameter must be less than or equal to pubEndDate.' + } + ], [ 'webSearchQueryError', [], diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index 07894bb..2bd90aa 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -1,5 +1,6 @@ const constants = require('./constants'); const cpe = require('./cpe'); +const cvss = require('./cvss'); const { MAX_RESULTS_LIMIT, RESULTS_PER_PAGE } = constants; @@ -320,14 +321,12 @@ function getFilterFields(params) { const r = cvssV2SeverityToRange(params[param]); if (!r) return; - fieldsToReturn.push({ - range: { [fieldSpec]: r } - }); + fieldsToReturn.push(buildRangeAcrossFields(r, fieldSpec)); return; } if (cvssVectorParams.has(param)) { - const filter = buildCvssWildcardFilter(params[param], fieldSpec); + const filter = buildCvssRegexpFilter(params[param], param, fieldSpec); if (filter) fieldsToReturn.push(filter); return; } @@ -343,8 +342,12 @@ function getFilterFields(params) { return fieldsToReturn; } -function escapeForWildcard(s) { - return String(s).replace(/([\\*?])/g, '\\$1'); +function escapeForRegexp(s) { + return String(s).replace(/([.?+*|{}[\]()"\\#@&<>~])/g, '\\$1'); +} + +function buildCvssTokenRegexp(token) { + return `(@/)?${escapeForRegexp(token)}(/@)?`; } function parseCveIds(value) { @@ -494,29 +497,30 @@ function buildHasAnyRangeFieldFilter(rangeFields) { }; } -function buildCvssWildcardFilter(inputVectorString, fieldSpec) { - const tokens = String(inputVectorString) - .split('/') - .map((t) => t.trim()) - .filter(Boolean) - .filter((t) => { - const up = t.toUpperCase(); - return !up.startsWith('CVSS:') && !up.startsWith('CVSS#'); - }); +function buildCvssRegexpFilter(inputVectorString, param, fieldSpec) { + const parsed = cvss.parseCvssMetrics(inputVectorString, param); + if (!parsed.valid) { + throw new TypeError(`Invalid ${param} value reached query construction.`); + } - if (!tokens.length) return null; + let fields = Array.isArray(fieldSpec) ? fieldSpec : [fieldSpec]; + if (parsed.version) { + const versionFields = constants.CVSS_VECTOR_VERSION_FIELDS[parsed.version] ?? []; + fields = fields.filter((field) => versionFields.includes(field)); + } - const fields = Array.isArray(fieldSpec) ? fieldSpec : [fieldSpec]; + const value = parsed.tokens.map((token) => buildCvssTokenRegexp(token.raw)).join('&'); const perFieldFilter = (fieldPath) => { const keywordField = `${fieldPath}.keyword`; return { - bool: { - filter: tokens.map((t) => ({ - wildcard: { - [keywordField]: { value: `*${escapeForWildcard(t)}*` } - } - })) + regexp: { + [keywordField]: { + value, + case_insensitive: true, + flags: 'INTERSECTION|ANYSTRING', + rewrite: 'constant_score' + } } }; }; @@ -525,12 +529,30 @@ function buildCvssWildcardFilter(inputVectorString, fieldSpec) { return { bool: { - should: fields.map((f) => perFieldFilter(f)), + should: fields.map((field) => perFieldFilter(field)), minimum_should_match: 1 } }; } +function buildRangeAcrossFields(range, fieldSpec) { + const fields = Array.isArray(fieldSpec) ? fieldSpec : [fieldSpec]; + const clauses = fields.map((field) => ({ + range: { + [field]: range + } + })); + + return clauses.length === 1 + ? clauses[0] + : { + bool: { + should: clauses, + minimum_should_match: 1 + } + }; +} + function buildSeverityAcrossFields(value, fieldSpec) { const fields = Array.isArray(fieldSpec) ? fieldSpec : [fieldSpec]; const v = String(value ?? '').toUpperCase(); diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index b26f879..c7c2915 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -20,6 +20,32 @@ function prefixShould(prefixes) { }; } +function cvssRegexp(field, value) { + return { + regexp: { + [`${field}.keyword`]: { + value, + case_insensitive: true, + flags: 'INTERSECTION|ANYSTRING', + rewrite: 'constant_score' + } + } + }; +} + +function cvssValue(...tokens) { + return tokens.map((token) => `(@/)?${token}(/@)?`).join('&'); +} + +function anyOf(clauses) { + return { + bool: { + should: clauses, + minimum_should_match: 1 + } + }; +} + describe('searchQueryBuilder buildSearchRequest()', () => { test('buildSearchRequest builds filters, query strings, date ranges, and pagination options', () => { const output = queryBuilder.buildSearchRequest({ @@ -426,39 +452,13 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds order-independent wildcard filters for cvssV2 metrics.', () => { + test('getFilterFields() builds a same-vector regexp filter for CNA cvssV2 metrics.', () => { const output = queryBuilder.getFilterFields({ cvssV2Metrics: 'AV:N/AC:L/Au:N' }); expect(output).toStrictEqual([ - { - bool: { - filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*AC:L*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*Au:N*' - } - } - } - ] - } - } + cvssRegexp('containers.cna.metrics.cvssV2_0.vectorString', cvssValue('AV:N', 'AC:L', 'Au:N')) ]); }); @@ -631,96 +631,47 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds order-independent wildcard filters for combined cvssV3 metrics.', () => { + test('getFilterFields() constrains versioned cvssV3 metrics to the matching CNA field.', () => { const output = queryBuilder.getFilterFields({ - cvssV3Metrics: 'CVSS:3.1/AV:N/AC:L' + cvssV3Metrics: 'CVSS:3.1/AC:L/AV:N' }); expect(output).toStrictEqual([ - { - bool: { - should: [ - { - bool: { - filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV3_0.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV3_0.vectorString.keyword': { - value: '*AC:L*' - } - } - } - ] - } - }, - { - bool: { - filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV3_1.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV3_1.vectorString.keyword': { - value: '*AC:L*' - } - } - } - ] - } - } - ], - minimum_should_match: 1 - } - } + cvssRegexp('containers.cna.metrics.cvssV3_1.vectorString', cvssValue('AC:L', 'AV:N')) ]); }); - test('getFilterFields() escapes wildcard metacharacters in CVSS metric filters.', () => { + test('getFilterFields() searches both v3 versions when cvssV3Metrics omits a header.', () => { + const output = queryBuilder.getFilterFields({ + cvssV3Metrics: 'AV:N' + }); + + expect(output).toStrictEqual([ + anyOf([ + cvssRegexp('containers.cna.metrics.cvssV3_0.vectorString', cvssValue('AV:N')), + cvssRegexp('containers.cna.metrics.cvssV3_1.vectorString', cvssValue('AV:N')) + ]) + ]); + }); + + test('getFilterFields() escapes regexp metacharacters in CVSS metric filters.', () => { const output = queryBuilder.getFilterFields({ cvssV4Metrics: 'CVSS#4.0/AV:N/AC:*?/UI:P' }); expect(output).toStrictEqual([ - { - bool: { - filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV4_0.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV4_0.vectorString.keyword': { - value: '*AC:\\*\\?*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV4_0.vectorString.keyword': { - value: '*UI:P*' - } - } - } - ] - } - } + cvssRegexp('containers.cna.metrics.cvssV4_0.vectorString', cvssValue('AV:N', 'AC:\\*\\?', 'UI:P')) ]); + + expect(JSON.stringify(output)).not.toContain('containers.adp'); + }); + + test('getFilterFields() fails closed when invalid CVSS metrics reach query construction.', () => { + expect(() => + queryBuilder.getFilterFields({ + cvssV3Metrics: 'CVSS:3.1' + }) + ).toThrow('Invalid cvssV3Metrics value reached query construction.'); }); test('getFilterFields() does not build filter objects for incorrect parameters.', () => { From a7be791b7979102091d4294fbbed07ade0a766fb Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 12:18:57 -0400 Subject: [PATCH 03/33] Correct search pagination and response contracts --- README.md | 8 + api-docs/openapi.json | 229 ++++++++++++++++++++++++++- api-docs/openapi.test.js | 36 +++++ controllers/searchController.js | 36 +++-- controllers/searchController.test.js | 54 ++++++- routes/index.js | 66 ++++++++ routes/search.integration.test.js | 13 ++ routes/swagger.js | 140 +++++++++++++++- utils/searchQueryBuilder.js | 24 ++- utils/searchQueryBuilder.test.js | 46 ++++++ 10 files changed, 634 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0a837ac..5e2ca5b 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,14 @@ CVSS vector filters accept full or partial vectors containing at least one uniqu Publication and last-modified date filters support one-sided ranges. When both values are supplied, `pubStartDate` must be less than or equal to `pubEndDate`, and `lastModStartDate` must be less than or equal to `lastModEndDate`. +### Search Ordering and Pagination + +`POST /search` uses deterministic ordering so records do not move arbitrarily between offset-based pages. Searches with scored query terms are ordered by relevance descending and then CVE ID ascending. Filter-only searches are ordered by CVE ID ascending. + +`pageSize` reports the number of CVE records actually returned in the current response. `page` is the requested one-based page, except a search with no matches reports page `0`. An accepted page beyond the available matches returns an empty `hits` array and page size `0`. + +Pagination is limited to complete pages inside the first 10,000 results: `from + resultsPerPage` must not exceed 10,000. `totalPages` reports the smaller of the pages implied by the match count and the complete pages accessible inside that window. For example, a page size of 30 allows at most 333 pages. The match total itself is not reduced by this pagination calculation. Point-in-time search and deep pagination remain deferred. + Use `cveIds` to request up to 100 CVE records by comma-separated CVE ID. ```bash diff --git a/api-docs/openapi.json b/api-docs/openapi.json index d3c4010..8e31120 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -33,16 +33,51 @@ "type": "string" } } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SearchResponse" + } + } } }, "400": { - "description": "Malformed JSON or invalid request body, parameter, CVSS vector, date range, or version range" + "description": "Malformed JSON or invalid request body, parameter, CVSS vector, date range, or version range", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ErrorResponse" + }, + { + "$ref": "#/components/schemas/ValidationErrorList" + } + ] + } + } + } }, "404": { - "description": "Invalid cpeName value" + "description": "Invalid cpeName value", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } }, "415": { - "description": "A non-empty request body did not use the application/json media type with UTF-8 encoding" + "description": "A non-empty request body did not use the application/json media type with UTF-8 encoding", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } }, "429": { "description": "Application rate limit exceeded, or the cold CPE search queue is full or exceeded its maximum wait", @@ -65,6 +100,23 @@ "type": "string" } } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "502": { + "description": "OpenSearch returned an invalid response or an unexpected upstream error occurred", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } } }, "503": { @@ -76,10 +128,24 @@ "type": "integer" } } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } } }, "504": { - "description": "Search exceeded its OpenSearch or overall processing deadline" + "description": "Search exceeded its OpenSearch or overall processing deadline", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } }, "requestBody": { @@ -493,6 +559,161 @@ "description": "Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component." } } + }, + "SearchHitTotal": { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "description": "Number of records matched by the active search strategy." + }, + "relation": { + "type": "string", + "enum": [ + "eq", + "gte" + ], + "description": "Whether value is exact or a lower bound when supplied by OpenSearch." + } + } + }, + "SearchHit": { + "type": "object", + "additionalProperties": true, + "properties": { + "_index": { + "type": "string" + }, + "_id": { + "type": "string" + }, + "_score": { + "type": "number", + "nullable": true + }, + "_source": { + "type": "object", + "additionalProperties": true, + "description": "The matching CVE record." + }, + "sort": { + "type": "array", + "items": {} + } + } + }, + "SearchHits": { + "type": "object", + "required": [ + "total", + "hits" + ], + "properties": { + "total": { + "$ref": "#/components/schemas/SearchHitTotal" + }, + "max_score": { + "type": "number", + "nullable": true + }, + "hits": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SearchHit" + } + } + } + }, + "SearchData": { + "type": "object", + "required": [ + "hits" + ], + "additionalProperties": true, + "properties": { + "took": { + "type": "integer", + "minimum": 0, + "description": "Combined OpenSearch processing time in milliseconds when available." + }, + "timed_out": { + "type": "boolean" + }, + "_shards": { + "type": "object", + "additionalProperties": true + }, + "hits": { + "$ref": "#/components/schemas/SearchHits" + } + } + }, + "SearchResponse": { + "type": "object", + "required": [ + "page", + "pageSize", + "totalPages", + "status", + "data" + ], + "additionalProperties": false, + "properties": { + "page": { + "type": "integer", + "minimum": 0, + "description": "Requested one-based page number, or 0 when the search has no matches." + }, + "pageSize": { + "type": "integer", + "minimum": 0, + "maximum": 500, + "description": "Number of CVE records returned in this response." + }, + "totalPages": { + "type": "integer", + "minimum": 0, + "description": "Number of pages accessible within the 10000-result search window." + }, + "status": { + "type": "string", + "enum": [ + "ok" + ] + }, + "data": { + "$ref": "#/components/schemas/SearchData" + } + } + }, + "ErrorResponse": { + "type": "object", + "required": [ + "error", + "message" + ], + "additionalProperties": false, + "properties": { + "error": { + "type": "string", + "description": "Stable machine-readable error code." + }, + "message": { + "type": "string", + "description": "Sanitized explanation of the error." + } + } + }, + "ValidationErrorList": { + "type": "array", + "description": "One or more request validation errors.", + "items": { + "$ref": "#/components/schemas/ErrorResponse" + } } } } diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 5029f32..316c297 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -17,6 +17,13 @@ function jsonRequestBodySchema(path, method) { .reduce((value, key) => value[key], openapi); } +function resolveSchemaRef(ref) { + return ref + .replace('#/', '') + .split('/') + .reduce((value, key) => value[key], openapi); +} + describe('OpenAPI contract', () => { test('documents the supported route surface', () => { expect(openapi.paths['/search'].post.operationId).toBe('search'); @@ -163,6 +170,7 @@ describe('OpenAPI contract', () => { expect(responses['400'].description).toContain('CVSS vector'); expect(responses['400'].description).toContain('date range'); expect(responses['415'].description).toContain('application/json'); + expect(responses['502'].description).toContain('invalid response'); expect(responses['503'].description).toContain('OpenSearch'); expect(responses['503'].headers['Retry-After']).toStrictEqual({ description: 'Seconds before retrying an unavailable OpenSearch dependency', @@ -172,6 +180,34 @@ describe('OpenAPI contract', () => { }); }); + test('documents the complete POST /search success and error response shapes', () => { + const responses = openapi.paths['/search'].post.responses; + const successRef = responses['200'].content['application/json'].schema.$ref; + const successSchema = resolveSchemaRef(successRef); + + expect(successRef).toBe('#/components/schemas/SearchResponse'); + expect(successSchema.required).toStrictEqual(['page', 'pageSize', 'totalPages', 'status', 'data']); + expect(successSchema.properties.pageSize.description).toContain('returned'); + expect(successSchema.properties.totalPages.description).toContain(`${constants.MAX_SEARCH_WINDOW}`); + expect(successSchema.properties.status.enum).toStrictEqual(['ok']); + expect(openapi.components.schemas.SearchHits.required).toStrictEqual(['total', 'hits']); + + expect(responses['400'].content['application/json'].schema.oneOf).toStrictEqual([ + { + $ref: '#/components/schemas/ErrorResponse' + }, + { + $ref: '#/components/schemas/ValidationErrorList' + } + ]); + + ['404', '415', '429', '502', '503', '504'].forEach((status) => { + expect(responses[status].content['application/json'].schema).toStrictEqual({ + $ref: '#/components/schemas/ErrorResponse' + }); + }); + }); + test('documents request correlation headers', () => { expect(openapi.paths['/search'].post.responses['200'].headers['X-Request-ID']).toStrictEqual({ description: 'Request correlation ID', diff --git a/controllers/searchController.js b/controllers/searchController.js index 06c2a77..6d3ea68 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -8,6 +8,7 @@ const searchInFlightRegistry = require('../utils/searchInFlightRegistry'); const searchPostFilterWorkQueue = require('../utils/searchPostFilterWorkQueue'); const searchProvider = require('../utils/searchProvider'); const searchQueryBuilder = require('../utils/searchQueryBuilder'); +const constants = require('../utils/constants'); const error = new errors.errors(); const openSearchUnavailableErrorNames = new Set(['ConnectionError', 'NoLivingConnectionsError']); const openSearchUnavailableStatusCodes = new Set([502, 503, 504]); @@ -90,6 +91,29 @@ function buildPagedSearchResponse(resp, totalValue, pagedHits) { }; } +function buildPaginationMetadata(resp, pagingOptions) { + const totalValue = resp.data.hits.total.value; + const requestedPageSize = pagingOptions.resultsPerPage; + const returnedHits = resp.data.hits.hits; + + if (totalValue === 0) { + return { + page: 0, + pageSize: 0, + totalPages: 0 + }; + } + + return { + page: Math.floor(pagingOptions.from / requestedPageSize) + 1, + pageSize: returnedHits.length, + totalPages: Math.min( + Math.ceil(totalValue / requestedPageSize), + Math.floor(constants.MAX_SEARCH_WINDOW / requestedPageSize) + ) + }; +} + function buildPostFilteredResponse( resp, filteredHits, @@ -326,6 +350,7 @@ async function executeOptimizedPostFilterSearch( const fullPageHits = await fetchFullHitsByCveIds(pageCveIds, metrics, signal); const response = { + status: 'ok', data: { took: combineTook(metrics.openSearchTookMs, metrics.pageFetchTookMs), hits: { @@ -389,17 +414,10 @@ async function searchHandler(req, res, next) { metrics.outcome = 'success'; metrics.statusCode = 200; - //Calculate current page and total pages to append to JSON output - const totalValue = resp.data.hits.total.value; - const requestedPageSize = pagingOptions.resultsPerPage; - const pageSize = Math.min(totalValue, requestedPageSize); - const page = totalValue === 0 ? 0 : Math.floor(pagingOptions.from / requestedPageSize) + 1; - const totalPages = totalValue === 0 ? 0 : Math.ceil(totalValue / requestedPageSize); + const pagination = buildPaginationMetadata(resp, pagingOptions); return res.status(200).json({ - page, - pageSize, - totalPages, + ...pagination, ...resp }); } catch (err) { diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 8bc6aeb..88c1b54 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -257,6 +257,18 @@ describe('Search controller SEARCH handler', () => { ] } }, + sort: [ + { + _score: { + order: 'desc' + } + }, + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ], track_total_hits: true, size: 10, from: 20 @@ -271,7 +283,7 @@ describe('Search controller SEARCH handler', () => { expect(res.status).toHaveBeenCalledWith(200); expect(res.json).toHaveBeenCalledWith({ page: 3, - pageSize: 10, + pageSize: 1, totalPages: 3, ...resp }); @@ -551,6 +563,13 @@ describe('Search controller SEARCH handler', () => { ] } }, + sort: [ + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ], track_total_hits: true, size: 1, from: 0 @@ -565,6 +584,7 @@ describe('Search controller SEARCH handler', () => { page: 1, pageSize: 1, totalPages: 1, + status: 'ok', data: { hits: { total: { @@ -801,6 +821,7 @@ describe('Search controller SEARCH handler', () => { page: 1, pageSize: 1, totalPages: 1, + status: 'ok', data: { hits: { total: { @@ -866,6 +887,7 @@ describe('Search controller SEARCH handler', () => { page: 1, pageSize: 1, totalPages: 1, + status: 'ok', data: { took: undefined, hits: { @@ -926,6 +948,7 @@ describe('Search controller SEARCH handler', () => { page: 1, pageSize: 1, totalPages: 1, + status: 'ok', data: { took: undefined, hits: { @@ -1031,6 +1054,7 @@ describe('Search controller SEARCH handler', () => { page: 1, pageSize: 1, totalPages: 1, + status: 'ok', data: { hits: { total: { @@ -1454,7 +1478,7 @@ describe('Search controller SEARCH handler', () => { }); }); - test('SEARCH calculates an out-of-range page from the requested page size', async () => { + test('SEARCH reports zero returned records for an out-of-range page', async () => { const resp = advancedResponse(5); mockApiSearch.mockResolvedValue(resp); @@ -1472,12 +1496,36 @@ describe('Search controller SEARCH handler', () => { expect(res.status).toHaveBeenCalledWith(200); expect(res.json).toHaveBeenCalledWith({ page: 2, - pageSize: 5, + pageSize: 0, totalPages: 1, ...resp }); }); + test('SEARCH limits advertised pages to complete pages inside the search window', async () => { + const resp = advancedResponse(25000, [{ _id: 'CVE-2024-0001' }]); + mockApiSearch.mockResolvedValue(resp); + + await controller.SEARCH( + { + query: { + resultsPerPage: 30, + pageNumber: 333 + } + }, + res, + next + ); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith({ + page: 333, + pageSize: 1, + totalPages: 333, + ...resp + }); + }); + test.each([ ['ConnectionError', { name: 'ConnectionError' }], ['NoLivingConnectionsError', { name: 'NoLivingConnectionsError' }], diff --git a/routes/index.js b/routes/index.js index 62df1e8..586dda2 100644 --- a/routes/index.js +++ b/routes/index.js @@ -243,16 +243,51 @@ headers: { type: "string" } } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/SearchResponse" + } + } } } #swagger.responses[400] = { description: 'Malformed JSON or invalid request body, parameter, CVSS vector, date range, or version range', +content: { + "application/json": { + schema: { + oneOf: [ + { + $ref: "#/components/schemas/ErrorResponse" + }, + { + $ref: "#/components/schemas/ValidationErrorList" + } + ] + } + } +} } #swagger.responses[404] = { description: 'Invalid cpeName value', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} } #swagger.responses[415] = { description: 'A non-empty request body did not use the application/json media type with UTF-8 encoding', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} } #swagger.responses[429] = { description: 'Application rate limit exceeded, or the cold CPE search queue is full or exceeded its maximum wait', @@ -275,6 +310,23 @@ headers: { type: "string" } } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} +#swagger.responses[502] = { +description: 'OpenSearch returned an invalid response or an unexpected upstream error occurred', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } } } #swagger.responses[503] = { @@ -286,10 +338,24 @@ headers: { type: "integer" } } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } } } #swagger.responses[504] = { description: 'Search exceeded its OpenSearch or overall processing deadline', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} } */ requireSearchJsonContentType, diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index 6df96c6..b9b6918 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -81,6 +81,18 @@ describe('search route integration', () => { filter: [] } }, + sort: [ + { + _score: { + order: 'desc' + } + }, + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ], track_total_hits: true, size: 1, from: 0 @@ -94,6 +106,7 @@ describe('search route integration', () => { expect(response.body.page).toBe(1); expect(response.body.pageSize).toBe(1); expect(response.body.totalPages).toBe(1); + expect(response.body.status).toBe('ok'); }); test('POST /search rejects invalid dates before invoking OpenSearch', async () => { diff --git a/routes/swagger.js b/routes/swagger.js index a4cfed3..ba704cc 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -231,6 +231,143 @@ const searchRequestProperties = { ) }; +const searchResponseSchemas = { + SearchHitTotal: { + type: 'object', + required: ['value'], + properties: { + value: { + type: 'integer', + minimum: 0, + description: 'Number of records matched by the active search strategy.' + }, + relation: { + type: 'string', + enum: ['eq', 'gte'], + description: 'Whether value is exact or a lower bound when supplied by OpenSearch.' + } + } + }, + SearchHit: { + type: 'object', + additionalProperties: true, + properties: { + _index: { + type: 'string' + }, + _id: { + type: 'string' + }, + _score: { + type: 'number', + nullable: true + }, + _source: { + type: 'object', + additionalProperties: true, + description: 'The matching CVE record.' + }, + sort: { + type: 'array', + items: {} + } + } + }, + SearchHits: { + type: 'object', + required: ['total', 'hits'], + properties: { + total: { + $ref: '#/components/schemas/SearchHitTotal' + }, + max_score: { + type: 'number', + nullable: true + }, + hits: { + type: 'array', + items: { + $ref: '#/components/schemas/SearchHit' + } + } + } + }, + SearchData: { + type: 'object', + required: ['hits'], + additionalProperties: true, + properties: { + took: { + type: 'integer', + minimum: 0, + description: 'Combined OpenSearch processing time in milliseconds when available.' + }, + timed_out: { + type: 'boolean' + }, + _shards: { + type: 'object', + additionalProperties: true + }, + hits: { + $ref: '#/components/schemas/SearchHits' + } + } + }, + SearchResponse: { + type: 'object', + required: ['page', 'pageSize', 'totalPages', 'status', 'data'], + additionalProperties: false, + properties: { + page: { + type: 'integer', + minimum: 0, + description: 'Requested one-based page number, or 0 when the search has no matches.' + }, + pageSize: { + type: 'integer', + minimum: 0, + maximum: constants.MAX_RESULTS_LIMIT, + description: 'Number of CVE records returned in this response.' + }, + totalPages: { + type: 'integer', + minimum: 0, + description: `Number of pages accessible within the ${constants.MAX_SEARCH_WINDOW}-result search window.` + }, + status: { + type: 'string', + enum: ['ok'] + }, + data: { + $ref: '#/components/schemas/SearchData' + } + } + }, + ErrorResponse: { + type: 'object', + required: ['error', 'message'], + additionalProperties: false, + properties: { + error: { + type: 'string', + description: 'Stable machine-readable error code.' + }, + message: { + type: 'string', + description: 'Sanitized explanation of the error.' + } + } + }, + ValidationErrorList: { + type: 'array', + description: 'One or more request validation errors.', + items: { + $ref: '#/components/schemas/ErrorResponse' + } + } +}; + const doc = { openapi: '3.0.2', info: { @@ -254,7 +391,8 @@ const doc = { description: 'Optional filters and pagination controls for a CVE record search.', additionalProperties: false, properties: searchRequestProperties - } + }, + ...searchResponseSchemas } } }; diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index 2bd90aa..992bd00 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -17,6 +17,26 @@ const cpeCandidateSourceFields = [ 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndIncluding', 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndExcluding' ]; +const cveIdAscendingSort = { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } +}; + +function buildDeterministicSort(queryStringObjects) { + const sort = []; + + if (queryStringObjects.length) { + sort.push({ + _score: { + order: 'desc' + } + }); + } + + sort.push(cveIdAscendingSort); + return sort; +} function buildSearchRequest(params) { const processedParams = processParams(params); @@ -51,14 +71,16 @@ function buildSearchRequest(params) { } function buildOpenSearchBody(searchOptions, queryStringObjects = [], { sourceIncludes, trackTotalHits = true } = {}) { + const normalizedQueryStringObjects = Array.isArray(queryStringObjects) ? queryStringObjects : []; const queryObj = { - must: Array.isArray(queryStringObjects) ? [...queryStringObjects] : [], + must: [...normalizedQueryStringObjects], filter: [...(searchOptions?.filters ?? []), ...(searchOptions?.rangeObjects ?? [])] }; const body = { query: { bool: queryObj }, + sort: buildDeterministicSort(normalizedQueryStringObjects), track_total_hits: trackTotalHits, size: searchOptions.resultsPerPage, from: searchOptions.from diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index c7c2915..0fca830 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -241,6 +241,18 @@ describe('searchQueryBuilder buildSearchRequest()', () => { filter: output.searchOptions.filters } }, + sort: [ + { + _score: { + order: 'desc' + } + }, + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ], track_total_hits: false, size: 10000, from: 0, @@ -286,12 +298,46 @@ describe('searchQueryBuilder buildSearchRequest()', () => { ] } }, + sort: [ + { + _score: { + order: 'desc' + } + }, + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ], track_total_hits: true, size: 10, from: 20 }); }); + test('buildOpenSearchBody orders filter-only results by CVE ID', () => { + expect( + queryBuilder.buildOpenSearchBody( + { + filters: [], + rangeObjects: [], + resultsPerPage: 20, + from: 0 + }, + [] + ) + ).toMatchObject({ + sort: [ + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ] + }); + }); + test('buildCveIdsSearchRequest builds exact page fetch request for selected CVE IDs', () => { expect(queryBuilder.buildCveIdsSearchRequest(['CVE-2024-0002', 'CVE-2024-0001'])).toStrictEqual({ searchOptions: { From d16f762c97284d6c1cec8c5a2f4564827c55f5d9 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 13:06:12 -0400 Subject: [PATCH 04/33] Harden semantic search resource controls --- .env.example | 2 + README.md | 18 ++- api-docs/openapi.json | 4 +- api-docs/openapi.test.js | 1 + config/custom-environment-variables.jsonc | 2 + config/default.jsonc | 4 + config/runtime-config.test.js | 6 + controllers/searchController.js | 154 ++++++++++++++++------ controllers/searchController.test.js | 83 +++++++++++- routes/index.js | 4 +- routes/search.integration.test.js | 2 + utils/constants.js | 4 + utils/cvss.js | 6 + utils/cvss.test.js | 7 + utils/errors.js | 4 + utils/errors.test.js | 8 ++ utils/searchDeadline.js | 15 +++ utils/searchDeadline.test.js | 18 +++ utils/searchInFlightRegistry.js | 27 +++- utils/searchInFlightRegistry.test.js | 37 ++++++ utils/searchPerformance.js | 5 +- utils/searchPerformance.test.js | 12 ++ utils/searchPostFilterCache.js | 35 +++-- utils/searchPostFilterCache.test.js | 85 ++++++++++++ utils/searchProvider.js | 33 ++++- utils/searchProvider.test.js | 60 +++++++++ utils/searchQueryBuilder.js | 12 +- utils/searchQueryBuilder.test.js | 23 +++- 28 files changed, 596 insertions(+), 75 deletions(-) diff --git a/.env.example b/.env.example index a90f4b1..823f158 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,8 @@ SearchPostFilterCacheMaxEntries=100 SearchPostFilterColdSearchConcurrency=10 SearchPostFilterColdSearchMaxQueueDepth=500 SearchPostFilterColdSearchMaxWaitMs=10000 +SearchInFlightMaxWaitersPerOperation=500 +SearchPostFilterEvaluationChunkSize=250 SearchOpenSearchRequestTimeoutMs=20000 SearchRequestDeadlineMs=30000 SearchReadinessTimeoutMs=5000 diff --git a/README.md b/README.md index 5e2ca5b..1155e76 100644 --- a/README.md +++ b/README.md @@ -259,9 +259,9 @@ Every request receives an `X-Request-ID` response header. A caller-provided `X-R Access logs omit query strings and request bodies. By default, the API retains 10% of successful non-health-check access logs, suppresses successful `/api/health-check` and `/api/ready-check` access logs, and retains every HTTP 4xx or 5xx access log. Set `RequestLogSuccessSampleRate` from `0` through `1` to change successful-request sampling. Set `RequestLogHealthChecks=true` to retain successful health-check logs. -The API always emits a sanitized JSON `search.performance` event for `/search` queue saturation, queue wait timeout, search timeout, client disconnect, OpenSearch connection errors, and upstream search errors. These events include the request ID, outcome, HTTP status, strategy, cache state, total elapsed time, cancellation reason, and queue state when applicable. Search terms, CPE values, request bodies, credentials, and upstream error details are never included. +The API always emits a sanitized JSON `search.performance` event for `/search` queue saturation, shared-operation waiter saturation, queue wait timeout, search timeout, client disconnect, OpenSearch connection errors, and upstream search errors. These events include the request ID, outcome, HTTP status, strategy, cache state, total elapsed time, cancellation reason, and bounded queue or waiter state when applicable. Search terms, CPE values, request bodies, credentials, and upstream error details are never included. -Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, and total handler elapsed time. Leave it disabled during normal operation unless the deployment logging volume is acceptable. +Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, shared waiter counts, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, and total handler elapsed time. Leave it disabled during normal operation unless the deployment logging volume is acceptable. Each `/search` request owns a lifecycle cancellation signal. When its overall deadline expires, the API aborts the active OpenSearch transport request before returning HTTP 504. When the caller disconnects, the API aborts work that is needed only by that caller and does not attempt to write a response. This prevents timed-out or abandoned standard searches from continuing until the transport timeout. @@ -289,13 +289,17 @@ The cache is intentionally small and short-lived: - Cold-search concurrency: cold CPE candidate queries are limited to 10 per process before reaching OpenSearch - Cold-search queue depth: up to 500 unique cold searches can wait per process behind the concurrency limit - Cold-search queue wait: up to 10 seconds before a waiting search returns HTTP 429 +- Shared operation waiters: up to 500 callers can wait on one candidate or page-fetch operation per process +- Post-filter evaluation chunk: 250 candidates between cancellation checks and event-loop yields - OpenSearch request timeout: 20 seconds for each operation - Overall `/search` deadline: 30 seconds - Eviction: oldest entries are removed when the cache exceeds its max size -When the cold-search queue is full, new unique CPE post-filter searches fail closed with HTTP 429 and a `SEARCH_QUEUE_FULL` response. Identical requests for a search that is already in flight still wait on the shared in-flight result and do not consume additional queue slots. +The admission queue applies only to cold CPE candidate searches. Standard `/search` requests and page fetches are not placed in this queue. When the cold-search queue is full, new unique CPE post-filter searches fail closed with HTTP 429 and a `SEARCH_QUEUE_FULL` response. Identical requests for a search that is already in flight still wait on the shared in-flight result and do not consume additional queue slots, up to the per-operation waiter limit. Additional callers fail closed with HTTP 429 and `SEARCH_IN_FLIGHT_WAITER_LIMIT`. -When a cold search waits in the queue for more than 10 seconds, it fails closed with HTTP 429 and a `SEARCH_QUEUE_TIMEOUT` response. OpenSearch operations use one attempt and return a sanitized HTTP 504 `SEARCH_TIMEOUT` response when they exceed 20 seconds. The same response is returned when a `/search` request exceeds the 30-second overall processing deadline, and the active OpenSearch request is aborted. +When a cold search waits in the queue for more than 10 seconds, it fails closed with HTTP 429 and a `SEARCH_QUEUE_TIMEOUT` response. OpenSearch operations use one attempt and return a sanitized HTTP 504 `SEARCH_TIMEOUT` response when they exceed 20 seconds. Standard searches use the shorter of that timeout and the request's remaining overall deadline. Shared candidate and page-fetch operations retain the independent OpenSearch timeout so one caller's remaining time does not shorten work still needed by other callers. Every caller retains its own 30-second overall deadline. + +Semantic filtering runs in bounded chunks and yields to the Node.js event loop between chunks. This lets disconnect and deadline cancellation signals interrupt a large candidate evaluation instead of waiting for the full set to finish synchronously. Healthy dev timing logs showed the impact for repeated identical `virtualMatchString` range searches: @@ -310,9 +314,11 @@ Healthy dev timing logs showed the impact for repeated identical `virtualMatchSt | Avg candidates scanned | 10000 | cached from prior result | | Avg filtered CVEs | 4 | 4 | -This helps most when multiple users submit the same expensive CPE search within the TTL. The first request is a cold cache miss; later identical requests on the same running task are warm cache hits. If identical cold requests arrive concurrently before the first one finishes, the later requests wait on the same candidate query instead of starting duplicate 10,000-candidate OpenSearch requests. Disconnecting one waiter does not cancel work still needed by another waiter; the shared operation is aborted when its final waiter disconnects or times out. Each active request still fetches its requested page of full CVE records after the shared candidate result is available. +This helps most when multiple users submit the same expensive CPE search within the TTL. The first request is a cold cache miss; later equivalent requests on the same running task are warm cache hits. Equivalent requests share keys across JSON property order, CVE ID order and duplicates, CVSS metric token order, and case differences in case-insensitive CPE values. If identical cold requests arrive concurrently before the first one finishes, the later requests wait on the same candidate query instead of starting duplicate 10,000-candidate OpenSearch requests. Disconnecting one waiter does not cancel work still needed by another waiter; the shared operation is aborted when its final waiter disconnects or times out. Each active request still fetches its requested page of full CVE records after the shared candidate result is available. + +The cache and cold-search queue are a temporary per-process proof of concept. They do not share state across ECS tasks, so each task warms and admits work independently depending on load balancer routing. Track cache-hit and in-flight-share rates, cold-search queue waits and rejections, cold-versus-warm latency, and OpenSearch time to decide their long-term value. If the benefit is low, remove this layer; if it is material, replace it with an AWS-managed shared cache and admission-control design. -The cache does not share entries across ECS tasks. With multiple tasks, each task warms independently depending on load balancer routing. Steady repeated traffic can warm every active task, but unique one-off searches still pay the cold-cache cost. The request lifecycle and in-flight waiter registry are separate from this result cache, so the cache can later be replaced with Redis or ElastiCache without changing cancellation behavior. The per-process TTL, entry limit, cold-search concurrency, queue depth, queue wait, OpenSearch timeout, and overall request deadline can be tuned with `SearchPostFilterCacheTtlMs`, `SearchPostFilterCacheMaxEntries`, `SearchPostFilterColdSearchConcurrency`, `SearchPostFilterColdSearchMaxQueueDepth`, `SearchPostFilterColdSearchMaxWaitMs`, `SearchOpenSearchRequestTimeoutMs`, and `SearchRequestDeadlineMs`. +The request lifecycle and in-flight waiter registry are separate from the result cache, so a later shared-cache replacement does not need to change caller cancellation behavior. The per-process TTL, entry limit, cold-search concurrency, queue depth, queue wait, shared waiter limit, evaluation chunk size, OpenSearch timeout, and overall request deadline can be tuned with `SearchPostFilterCacheTtlMs`, `SearchPostFilterCacheMaxEntries`, `SearchPostFilterColdSearchConcurrency`, `SearchPostFilterColdSearchMaxQueueDepth`, `SearchPostFilterColdSearchMaxWaitMs`, `SearchInFlightMaxWaitersPerOperation`, `SearchPostFilterEvaluationChunkSize`, `SearchOpenSearchRequestTimeoutMs`, and `SearchRequestDeadlineMs`. Docker runtime smoke checks can be run after building the image. The default `Dockerfile` avoids BuildKit `RUN --mount` syntax for AWS builders that do not support it: diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 8e31120..22f7452 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -80,10 +80,10 @@ } }, "429": { - "description": "Application rate limit exceeded, or the cold CPE search queue is full or exceeded its maximum wait", + "description": "Application rate limit exceeded, the cold CPE search queue is full or exceeded its maximum wait, or a shared semantic search operation reached its waiter limit", "headers": { "Retry-After": { - "description": "Seconds until a rate-limited request can be retried", + "description": "Seconds until a busy or rate-limited request can be retried", "schema": { "type": "integer" } diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 316c297..35da775 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -161,6 +161,7 @@ describe('OpenAPI contract', () => { expect(cpeName.example).toBe('cpe:2.3:a:freetype:freetype:2.8.1:*:*:*:*:*:*:*'); expect(openapi.paths['/search'].post.responses['404'].description).toBe('Invalid cpeName value'); expect(openapi.paths['/search'].post.responses['429'].description).toContain('rate limit'); + expect(openapi.paths['/search'].post.responses['429'].description).toContain('waiter limit'); }); test('documents fail-closed request and dependency responses for POST /search', () => { diff --git a/config/custom-environment-variables.jsonc b/config/custom-environment-variables.jsonc index 3470a97..8898e50 100644 --- a/config/custom-environment-variables.jsonc +++ b/config/custom-environment-variables.jsonc @@ -17,6 +17,8 @@ "postFilterColdSearchConcurrency": "SearchPostFilterColdSearchConcurrency", "postFilterColdSearchMaxQueueDepth": "SearchPostFilterColdSearchMaxQueueDepth", "postFilterColdSearchMaxWaitMs": "SearchPostFilterColdSearchMaxWaitMs", + "inFlightMaxWaitersPerOperation": "SearchInFlightMaxWaitersPerOperation", + "postFilterEvaluationChunkSize": "SearchPostFilterEvaluationChunkSize", "openSearchRequestTimeoutMs": "SearchOpenSearchRequestTimeoutMs", "requestDeadlineMs": "SearchRequestDeadlineMs", "readinessTimeoutMs": "SearchReadinessTimeoutMs" diff --git a/config/default.jsonc b/config/default.jsonc index 8cdef78..58208d9 100644 --- a/config/default.jsonc +++ b/config/default.jsonc @@ -25,6 +25,10 @@ "postFilterColdSearchMaxQueueDepth": "500", // Limits how long a cold CPE candidate search can wait in the per-process queue. "postFilterColdSearchMaxWaitMs": "10000", + // Limits callers sharing one candidate or page-fetch operation per Node.js process. + "inFlightMaxWaitersPerOperation": "500", + // Allows cancellation checks between batches of semantic CPE candidate evaluations. + "postFilterEvaluationChunkSize": "250", // Limits each request from the API process to OpenSearch. "openSearchRequestTimeoutMs": "20000", // Limits total processing time for a /search request. diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index 350a6b1..d07d258 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -98,6 +98,12 @@ describe('runtime container and compose configuration', () => { expect(customEnvironmentVariables).toContain( '"postFilterColdSearchMaxWaitMs": "SearchPostFilterColdSearchMaxWaitMs"' ); + expect(customEnvironmentVariables).toContain( + '"inFlightMaxWaitersPerOperation": "SearchInFlightMaxWaitersPerOperation"' + ); + expect(customEnvironmentVariables).toContain( + '"postFilterEvaluationChunkSize": "SearchPostFilterEvaluationChunkSize"' + ); expect(customEnvironmentVariables).toContain( '"openSearchRequestTimeoutMs": "SearchOpenSearchRequestTimeoutMs"' ); diff --git a/controllers/searchController.js b/controllers/searchController.js index 6d3ea68..f7e00e0 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -21,47 +21,86 @@ function getHitsContainer(resp) { return resp?.data?.hits; } -function filterSearchHits(resp, postFilters) { - const hitsContainer = getHitsContainer(resp); - const cpeEvaluationContext = cpe.createEvaluationContext(); +function hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext) { + if ( + postFilters?.cpeName && + !cpe.hitAppliesToCpeName(hit, postFilters.cpeName, cpeEvaluationContext, { + requireVulnerable: postFilters.isVulnerable === true + }) + ) { + return false; + } - if (!Array.isArray(hitsContainer?.hits)) { - return { - hitsContainer, - candidateCount: 0, - filteredHits: [] - }; + if ( + postFilters?.virtualMatchString && + !cpe.hitMatchesVirtualMatchString(hit, postFilters.virtualMatchString, cpeEvaluationContext) + ) { + return false; } + if ( + postFilters?.virtualMatchStringRange && + !cpe.hitOverlapsVirtualMatchStringRange(hit, postFilters.virtualMatchStringRange, cpeEvaluationContext) + ) { + return false; + } + + return true; +} + +function buildEmptyFilteredResult(hitsContainer) { + return { + hitsContainer, + candidateCount: 0, + filteredHits: [] + }; +} + +function filterSearchHits(resp, postFilters) { + const hitsContainer = getHitsContainer(resp); + if (!Array.isArray(hitsContainer?.hits)) return buildEmptyFilteredResult(hitsContainer); + + const cpeEvaluationContext = cpe.createEvaluationContext(); return { hitsContainer, candidateCount: hitsContainer.hits.length, - filteredHits: hitsContainer.hits.filter((hit) => { - if ( - postFilters?.cpeName && - !cpe.hitAppliesToCpeName(hit, postFilters.cpeName, cpeEvaluationContext, { - requireVulnerable: postFilters.isVulnerable === true - }) - ) { - return false; - } + filteredHits: hitsContainer.hits.filter((hit) => hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext)) + }; +} - if ( - postFilters?.virtualMatchString && - !cpe.hitMatchesVirtualMatchString(hit, postFilters.virtualMatchString, cpeEvaluationContext) - ) { - return false; - } +function yieldToEventLoop() { + return new Promise((resolve) => setImmediate(resolve)); +} + +async function filterSearchHitsInChunks(resp, postFilters, signal, chunkSize) { + const hitsContainer = getHitsContainer(resp); + if (!Array.isArray(hitsContainer?.hits)) return buildEmptyFilteredResult(hitsContainer); + + const effectiveChunkSize = Math.max(1, chunkSize ?? searchProvider.getPostFilterEvaluationChunkSize()); + const cpeEvaluationContext = cpe.createEvaluationContext(); + const filteredHits = []; - if ( - postFilters?.virtualMatchStringRange && - !cpe.hitOverlapsVirtualMatchStringRange(hit, postFilters.virtualMatchStringRange, cpeEvaluationContext) - ) { - return false; + for (let start = 0; start < hitsContainer.hits.length; start += effectiveChunkSize) { + searchDeadline.throwIfAborted(signal); + const end = Math.min(start + effectiveChunkSize, hitsContainer.hits.length); + + for (let index = start; index < end; index += 1) { + const hit = hitsContainer.hits[index]; + if (hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext)) { + filteredHits.push(hit); } + } - return true; - }) + if (end < hitsContainer.hits.length) { + await yieldToEventLoop(); + } + } + + searchDeadline.throwIfAborted(signal); + return { + hitsContainer, + candidateCount: hitsContainer.hits.length, + filteredHits }; } @@ -212,20 +251,28 @@ function recordOpenSearchStart(metrics, operation, state) { metrics.maxOpenSearchActiveRequests = Math.max(metrics.maxOpenSearchActiveRequests ?? 0, state.activeCount); } -async function executeOpenSearch(body, operation, signal, metrics) { +async function executeOpenSearch(body, operation, signal, metrics, requestTimeoutMs) { return searchProvider.searchOpenSearch(body, { signal, operation, + requestTimeoutMs, onStart: (state) => recordOpenSearchStart(metrics, operation, state) }); } -async function executeStandardSearch(searchOptions, queryStringObjects, postFilters, responseOptions, metrics, signal) { +async function executeStandardSearch( + searchOptions, + queryStringObjects, + postFilters, + responseOptions, + metrics, + lifecycle +) { metrics.strategy = 'standard'; const searchStart = searchPerformance.markTime(); const body = searchQueryBuilder.buildOpenSearchBody(searchOptions, queryStringObjects); - const searchData = await executeOpenSearch(body, 'standard', signal, metrics); + const searchData = await executeOpenSearch(body, 'standard', lifecycle.signal, metrics, lifecycle.getRemainingMs()); const searchResp = { status: 'ok', data: searchData @@ -257,10 +304,14 @@ async function fetchFullHitsByCveIds(cveIds, metrics, signal) { return orderHitsByCveIds(pageData?.hits?.hits ?? [], cveIds); }, - { signal } + { + signal, + maxWaiters: searchProvider.getInFlightMaxWaitersPerOperation() + } ); metrics.pageFetchInFlightHit = inFlight.shared; + metrics.pageFetchInFlightWaiterCount = inFlight.waiterCount; const result = await inFlight.promise; if (inFlight.shared) { metrics.pageFetchWaitElapsedMs = searchPerformance.elapsedMs(waitStart); @@ -279,7 +330,7 @@ async function buildPostFilterCandidateResult(candidateBody, postFilters, metric data: candidateData }; const postFilterStart = searchPerformance.markTime(); - const { candidateCount, filteredHits } = filterSearchHits(candidateResp, postFilters); + const { candidateCount, filteredHits } = await filterSearchHitsInChunks(candidateResp, postFilters, signal); metrics.postFilterElapsedMs = searchPerformance.elapsedMs(postFilterStart); return { @@ -317,10 +368,14 @@ async function getPostFilterCandidateResult(cacheKey, candidateBody, postFilters searchPostFilterCache.set(cacheKey, result); return result; }), - { signal } + { + signal, + maxWaiters: searchProvider.getInFlightMaxWaitersPerOperation() + } ); metrics.inFlightHit = inFlight.shared; + metrics.inFlightWaiterCount = inFlight.waiterCount; const result = await inFlight.promise; if (inFlight.shared) { metrics.inFlightWaitElapsedMs = searchPerformance.elapsedMs(waitStart); @@ -365,9 +420,16 @@ async function executeOptimizedPostFilterSearch( return buildPagedSearchResponse(response, metrics.filteredCount, fullPageHits); } -async function executeSearch(searchOptions, queryStringObjects, postFilters, responseOptions, metrics, signal) { +async function executeSearch(searchOptions, queryStringObjects, postFilters, responseOptions, metrics, lifecycle) { if (!hasSearchPostFilters(postFilters)) { - return executeStandardSearch(searchOptions, queryStringObjects, postFilters, responseOptions, metrics, signal); + return executeStandardSearch( + searchOptions, + queryStringObjects, + postFilters, + responseOptions, + metrics, + lifecycle + ); } return executeOptimizedPostFilterSearch( @@ -376,7 +438,7 @@ async function executeSearch(searchOptions, queryStringObjects, postFilters, res postFilters, responseOptions, metrics, - signal + lifecycle.signal ); } @@ -407,7 +469,7 @@ async function searchHandler(req, res, next) { postFilters, responseOptions, metrics, - lifecycle.signal + lifecycle ); lifecycle.throwIfAborted(); const pagingOptions = responseOptions ?? searchOptions; @@ -447,6 +509,15 @@ async function searchHandler(req, res, next) { return res.status(429).json(error.searchQueueTimeoutError()); } + if (searchInFlightRegistry.isWaiterLimitError(err)) { + metrics.outcome = 'in_flight_waiter_limit'; + metrics.statusCode = 429; + metrics.inFlightState = err.state; + if (typeof res.set === 'function') res.set('Retry-After', '5'); + + return res.status(429).json(error.searchInFlightWaiterLimitError()); + } + if (searchProvider.isOpenSearchResponseError?.(err)) { const failureType = err?.failureType ?? 'malformed_response'; recordUpstreamFailure(metrics, err, failureType); @@ -572,6 +643,7 @@ async function webSearchHandler(req, res, next) { module.exports = { applySearchPostFilters, buildWebSearchSort, + filterSearchHitsInChunks, SEARCH: searchHandler, WEB_SEARCH: webSearchHandler }; diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 88c1b54..69fb4f2 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -21,6 +21,8 @@ const mockGetPostFilterCacheMaxEntries = jest.fn(() => 100); const mockGetPostFilterColdSearchConcurrency = jest.fn(() => 10); const mockGetPostFilterColdSearchMaxQueueDepth = jest.fn(() => 500); const mockGetPostFilterColdSearchMaxWaitMs = jest.fn(() => 10000); +const mockGetInFlightMaxWaitersPerOperation = jest.fn(() => 500); +const mockGetPostFilterEvaluationChunkSize = jest.fn(() => 250); const mockGetSearchRequestDeadlineMs = jest.fn(() => 30000); const mockIsOpenSearchResponseError = jest.fn((err) => err?.code === 'OPEN_SEARCH_RESPONSE_INVALID'); const mockIsSearchProviderContractError = jest.fn((err) => err?.code === 'SEARCH_PROVIDER_CONTRACT'); @@ -43,6 +45,8 @@ jest.mock('../utils/searchProvider', () => ({ getPostFilterColdSearchConcurrency: mockGetPostFilterColdSearchConcurrency, getPostFilterColdSearchMaxQueueDepth: mockGetPostFilterColdSearchMaxQueueDepth, getPostFilterColdSearchMaxWaitMs: mockGetPostFilterColdSearchMaxWaitMs, + getInFlightMaxWaitersPerOperation: mockGetInFlightMaxWaitersPerOperation, + getPostFilterEvaluationChunkSize: mockGetPostFilterEvaluationChunkSize, getSearchRequestDeadlineMs: mockGetSearchRequestDeadlineMs, isOpenSearchResponseError: mockIsOpenSearchResponseError, isSearchProviderContractError: mockIsSearchProviderContractError, @@ -176,6 +180,8 @@ describe('Search controller SEARCH handler', () => { mockGetPostFilterColdSearchConcurrency.mockClear(); mockGetPostFilterColdSearchMaxQueueDepth.mockClear(); mockGetPostFilterColdSearchMaxWaitMs.mockClear(); + mockGetInFlightMaxWaitersPerOperation.mockClear(); + mockGetPostFilterEvaluationChunkSize.mockClear(); mockGetSearchRequestDeadlineMs.mockClear(); mockIsOpenSearchResponseError.mockClear(); mockIsSearchProviderContractError.mockClear(); @@ -184,6 +190,8 @@ describe('Search controller SEARCH handler', () => { mockGetPostFilterColdSearchConcurrency.mockReturnValue(10); mockGetPostFilterColdSearchMaxQueueDepth.mockReturnValue(500); mockGetPostFilterColdSearchMaxWaitMs.mockReturnValue(10000); + mockGetInFlightMaxWaitersPerOperation.mockReturnValue(500); + mockGetPostFilterEvaluationChunkSize.mockReturnValue(250); mockGetSearchRequestDeadlineMs.mockReturnValue(30000); searchPostFilterCache.clear(); searchInFlightRegistry.clear(); @@ -1066,6 +1074,30 @@ describe('Search controller SEARCH handler', () => { }); }); + test('chunked semantic filtering stops after cancellation between batches', async () => { + const hits = [ + cpeHit('CVE-2024-0001', { + criteria: 'cpe:2.3:o:microsoft:windows_10_1809:-:*:*:*:*:*:x86:*' + }), + cpeHit('CVE-2024-0002', { + criteria: 'cpe:2.3:o:microsoft:windows_10_1809:-:*:*:*:*:*:x86:*' + }) + ]; + const abortController = new AbortController(); + const filtering = controller.filterSearchHitsInChunks( + advancedResponse(2, hits), + { + virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809' + }, + abortController.signal, + 1 + ); + + abortController.abort(new Error('cancelled')); + + await expect(filtering).rejects.toThrow('cancelled'); + }); + test('SEARCH returns 429 when the cold semantic post-filter queue is full', async () => { const candidateSearch = deferred(); const firstRes = createResponse(); @@ -1188,6 +1220,53 @@ describe('Search controller SEARCH handler', () => { await firstSearch; }); + test('SEARCH bounds callers sharing one semantic candidate operation', async () => { + const candidateSearch = deferred(); + const firstRes = createResponse(); + const secondRes = createResponse(); + const req = { + query: { + virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', + resultsPerPage: '1', + pageNumber: '1' + } + }; + + mockGetInFlightMaxWaitersPerOperation.mockReturnValue(1); + mockSearchRaw.mockReturnValue(candidateSearch.promise); + + const firstSearch = controller.SEARCH(req, firstRes, next); + await Promise.resolve(); + await controller.SEARCH( + { + requestId: 'waiter-limit-request', + ...req + }, + secondRes, + next + ); + + expect(secondRes.set).toHaveBeenCalledWith('Retry-After', '5'); + expect(secondRes.status).toHaveBeenCalledWith(429); + expect(secondRes.json).toHaveBeenCalledWith({ + error: 'SEARCH_IN_FLIGHT_WAITER_LIMIT', + message: 'Search service is busy. Please retry later.' + }); + expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ + event: 'search.performance', + requestId: 'waiter-limit-request', + outcome: 'in_flight_waiter_limit', + statusCode: 429, + inFlightState: { + waiterCount: 1, + maxWaiters: 1 + } + }); + + candidateSearch.resolve(rawResponse(0, [])); + await firstSearch; + }); + test('SEARCH returns sanitized upstream errors when optimized post-filter search fails', async () => { mockSearchRaw.mockRejectedValue(new Error('candidate search failed')); @@ -1319,7 +1398,7 @@ describe('Search controller SEARCH handler', () => { ); expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - cvssRegexp('containers.cna.metrics.cvssV2_0.vectorString', cvssValue('AV:N', 'AC:L', 'Au:N')) + cvssRegexp('containers.cna.metrics.cvssV2_0.vectorString', cvssValue('AC:L', 'Au:N', 'AV:N')) ]); }); @@ -1338,7 +1417,7 @@ describe('Search controller SEARCH handler', () => { ); expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - cvssRegexp('containers.cna.metrics.cvssV3_0.vectorString', cvssValue('AV:N', 'AC:L')), + cvssRegexp('containers.cna.metrics.cvssV3_0.vectorString', cvssValue('AC:L', 'AV:N')), cvssRegexp('containers.cna.metrics.cvssV3_1.vectorString', cvssValue('PR:N', 'UI:R')) ]); }); diff --git a/routes/index.js b/routes/index.js index 586dda2..658d590 100644 --- a/routes/index.js +++ b/routes/index.js @@ -290,10 +290,10 @@ content: { } } #swagger.responses[429] = { -description: 'Application rate limit exceeded, or the cold CPE search queue is full or exceeded its maximum wait', +description: 'Application rate limit exceeded, the cold CPE search queue is full or exceeded its maximum wait, or a shared semantic search operation reached its waiter limit', headers: { "Retry-After": { - description: "Seconds until a rate-limited request can be retried", + description: "Seconds until a busy or rate-limited request can be retried", schema: { type: "integer" } diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index b9b6918..d4ff51a 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -8,6 +8,8 @@ jest.mock('../utils/searchProvider', () => ({ getBasicSearchManager: jest.fn(() => ({ search: mockBasicSearch })), + getInFlightMaxWaitersPerOperation: jest.fn(() => 500), + getPostFilterEvaluationChunkSize: jest.fn(() => 250), getSearchRequestDeadlineMs: jest.fn(() => 30000), searchOpenSearch: mockSearchOpenSearch, shouldLogSearchPerformance: jest.fn(() => false) diff --git a/utils/constants.js b/utils/constants.js index 8f970c3..696bebd 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -21,6 +21,8 @@ const SEARCH_POST_FILTER_CACHE_MAX_ENTRIES = 100; const SEARCH_POST_FILTER_COLD_SEARCH_CONCURRENCY = 10; const SEARCH_POST_FILTER_COLD_SEARCH_MAX_QUEUE_DEPTH = 500; const SEARCH_POST_FILTER_COLD_SEARCH_MAX_WAIT_MS = 10 * 1000; +const SEARCH_IN_FLIGHT_MAX_WAITERS_PER_OPERATION = 500; +const SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE = 250; const SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS = 20 * 1000; const SEARCH_REQUEST_DEADLINE_MS = 30 * 1000; const SEARCH_READINESS_TIMEOUT_MS = 5 * 1000; @@ -149,6 +151,8 @@ module.exports = { SEARCH_POST_FILTER_COLD_SEARCH_CONCURRENCY, SEARCH_POST_FILTER_COLD_SEARCH_MAX_QUEUE_DEPTH, SEARCH_POST_FILTER_COLD_SEARCH_MAX_WAIT_MS, + SEARCH_IN_FLIGHT_MAX_WAITERS_PER_OPERATION, + SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE, SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS, SEARCH_REQUEST_DEADLINE_MS, SEARCH_READINESS_TIMEOUT_MS, diff --git a/utils/cvss.js b/utils/cvss.js index 4a50d46..e56a291 100644 --- a/utils/cvss.js +++ b/utils/cvss.js @@ -59,6 +59,12 @@ function parseCvssMetrics(value, param) { }); } + tokens.sort((left, right) => { + if (left.key < right.key) return -1; + if (left.key > right.key) return 1; + return 0; + }); + return { valid: true, version, diff --git a/utils/cvss.test.js b/utils/cvss.test.js index 6e09cf5..17bdbbc 100644 --- a/utils/cvss.test.js +++ b/utils/cvss.test.js @@ -60,4 +60,11 @@ describe('cvss', () => { ] }); }); + + test('parseCvssMetrics canonicalizes metric token order', () => { + const first = cvss.parseCvssMetrics('AV:N/AC:L', 'cvssV3Metrics'); + const second = cvss.parseCvssMetrics('AC:L/AV:N', 'cvssV3Metrics'); + + expect(first).toStrictEqual(second); + }); }); diff --git a/utils/errors.js b/utils/errors.js index 5017ce0..824b804 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -297,6 +297,10 @@ class errors { return makeError('SEARCH_QUEUE_TIMEOUT', 'Search service is busy. Please retry later.'); } + searchInFlightWaiterLimitError() { + return makeError('SEARCH_IN_FLIGHT_WAITER_LIMIT', 'Search service is busy. Please retry later.'); + } + searchTimeoutError() { return makeError('SEARCH_TIMEOUT', 'Search request timed out. Please try again later.'); } diff --git a/utils/errors.test.js b/utils/errors.test.js index f950135..3258bf4 100644 --- a/utils/errors.test.js +++ b/utils/errors.test.js @@ -191,6 +191,14 @@ describe('error response helpers', () => { message: 'Search service is busy. Please retry later.' } ], + [ + 'searchInFlightWaiterLimitError', + [], + { + error: 'SEARCH_IN_FLIGHT_WAITER_LIMIT', + message: 'Search service is busy. Please retry later.' + } + ], [ 'searchTimeoutError', [], diff --git a/utils/searchDeadline.js b/utils/searchDeadline.js index ecb33c4..d30efec 100644 --- a/utils/searchDeadline.js +++ b/utils/searchDeadline.js @@ -44,6 +44,7 @@ function addOnceListener(emitter, eventName, listener) { function createSearchRequestLifecycle(req, res, timeoutMs) { const controller = new AbortController(); + const deadlineAt = Date.now() + timeoutMs; let completed = false; const abort = (reason) => { @@ -82,8 +83,22 @@ function createSearchRequestLifecycle(req, res, timeoutMs) { throwIfAborted(controller.signal); } + function getRemainingMs() { + throwIfRequestAborted(); + + const remainingMs = deadlineAt - Date.now(); + if (remainingMs <= 0) { + abort(new SearchDeadlineExceededError()); + throwIfAborted(controller.signal); + } + + return Math.max(1, Math.floor(remainingMs)); + } + return { complete, + deadlineAt, + getRemainingMs, signal: controller.signal, throwIfAborted: throwIfRequestAborted }; diff --git a/utils/searchDeadline.test.js b/utils/searchDeadline.test.js index cafbf3c..53b962c 100644 --- a/utils/searchDeadline.test.js +++ b/utils/searchDeadline.test.js @@ -103,6 +103,24 @@ describe('searchDeadline', () => { expect(() => lifecycle.throwIfAborted()).toThrow(searchDeadline.SearchClientDisconnectedError); }); + test('reports the absolute deadline and remaining request budget', () => { + captureTimers(); + jest.spyOn(Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(11000); + const { lifecycle } = createLifecycle(); + + expect(lifecycle.deadlineAt).toBe(31000); + expect(lifecycle.getRemainingMs()).toBe(20000); + }); + + test('fails closed when the remaining request budget has elapsed', () => { + captureTimers(); + jest.spyOn(Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(31000); + const { lifecycle } = createLifecycle(); + + expect(() => lifecycle.getRemainingMs()).toThrow(searchDeadline.SearchDeadlineExceededError); + expect(lifecycle.signal.aborted).toBe(true); + }); + test.each([ [new searchDeadline.SearchDeadlineExceededError()], [{ code: 'SEARCH_TIMEOUT' }], diff --git a/utils/searchInFlightRegistry.js b/utils/searchInFlightRegistry.js index 07f4e85..4a20c59 100644 --- a/utils/searchInFlightRegistry.js +++ b/utils/searchInFlightRegistry.js @@ -10,6 +10,15 @@ class SearchOperationAbandonedError extends Error { } } +class SearchInFlightWaiterLimitError extends Error { + constructor(state) { + super('Shared search operation reached its waiter limit.'); + this.name = 'SearchInFlightWaiterLimitError'; + this.code = 'SEARCH_IN_FLIGHT_WAITER_LIMIT'; + this.state = state; + } +} + function buildKey(type, value) { return `${type}:${JSON.stringify(value)}`; } @@ -78,7 +87,7 @@ function waitForEntry(entry, signal) { }); } -function acquire(key, createOperation, { signal } = {}) { +function acquire(key, createOperation, { signal, maxWaiters } = {}) { searchDeadline.throwIfAborted(signal); let entry = entries.get(key); @@ -87,12 +96,24 @@ function acquire(key, createOperation, { signal } = {}) { entry = createEntry(key, createOperation); } + if (Number.isInteger(maxWaiters) && maxWaiters > 0 && entry.waiterCount >= maxWaiters) { + throw new SearchInFlightWaiterLimitError({ + waiterCount: entry.waiterCount, + maxWaiters + }); + } + return { promise: waitForEntry(entry, signal), - shared + shared, + waiterCount: entry.waiterCount }; } +function isWaiterLimitError(err) { + return err instanceof SearchInFlightWaiterLimitError || err?.code === 'SEARCH_IN_FLIGHT_WAITER_LIMIT'; +} + function clear() { for (const entry of entries.values()) { if (!entry.controller.signal.aborted) { @@ -114,5 +135,7 @@ module.exports = { buildKey, clear, getState, + isWaiterLimitError, + SearchInFlightWaiterLimitError, SearchOperationAbandonedError }; diff --git a/utils/searchInFlightRegistry.test.js b/utils/searchInFlightRegistry.test.js index aed7ad9..a5c256d 100644 --- a/utils/searchInFlightRegistry.test.js +++ b/utils/searchInFlightRegistry.test.js @@ -27,7 +27,9 @@ describe('searchInFlightRegistry', () => { await Promise.resolve(); expect(first.shared).toBe(false); + expect(first.waiterCount).toBe(1); expect(second.shared).toBe(true); + expect(second.waiterCount).toBe(2); expect(createOperation).toHaveBeenCalledTimes(1); expect(searchInFlightRegistry.getState()).toStrictEqual({ operationCount: 1, @@ -125,6 +127,41 @@ describe('searchInFlightRegistry', () => { expect(createOperation).not.toHaveBeenCalled(); }); + test('rejects callers beyond the per-operation waiter limit without cancelling active work', async () => { + const operation = deferred(); + const createOperation = jest.fn(() => operation.promise); + const first = searchInFlightRegistry.acquire('same-key', createOperation, { + maxWaiters: 1 + }); + + let limitError; + try { + searchInFlightRegistry.acquire('same-key', createOperation, { + maxWaiters: 1 + }); + } catch (err) { + limitError = err; + } + + expect(limitError).toMatchObject({ + code: 'SEARCH_IN_FLIGHT_WAITER_LIMIT', + state: { + waiterCount: 1, + maxWaiters: 1 + } + }); + expect( + searchInFlightRegistry.isWaiterLimitError(new searchInFlightRegistry.SearchInFlightWaiterLimitError({})) + ).toBe(true); + expect(searchInFlightRegistry.getState()).toStrictEqual({ + operationCount: 1, + waiterCount: 1 + }); + + operation.resolve('result'); + await expect(first.promise).resolves.toBe('result'); + }); + test('buildKey preserves operation type boundaries', () => { expect(searchInFlightRegistry.buildKey('candidate', ['CVE-2024-0001'])).not.toBe( searchInFlightRegistry.buildKey('pageFetch', ['CVE-2024-0001']) diff --git a/utils/searchPerformance.js b/utils/searchPerformance.js index f666aec..19c690f 100644 --- a/utils/searchPerformance.js +++ b/utils/searchPerformance.js @@ -37,6 +37,7 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte upstreamFailureType: metrics.upstreamFailureType, strategy: metrics.strategy, cacheStatus: getCacheStatus(metrics, postFilters), + inFlightState: metrics.inFlightState, coldSearchQueueWaitElapsedMs: roundMs(metrics.coldSearchQueueWaitElapsedMs), coldSearchQueueState: metrics.coldSearchQueueState, totalElapsedMs: roundMs(metrics.totalElapsedMs) @@ -62,8 +63,10 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte openSearchElapsedMs: roundMs(metrics.openSearchElapsedMs), postFilterElapsedMs: roundMs(metrics.postFilterElapsedMs), inFlightWaitElapsedMs: roundMs(metrics.inFlightWaitElapsedMs), + inFlightWaiterCount: metrics.inFlightWaiterCount, pageFetchElapsedMs: roundMs(metrics.pageFetchElapsedMs), - pageFetchWaitElapsedMs: roundMs(metrics.pageFetchWaitElapsedMs) + pageFetchWaitElapsedMs: roundMs(metrics.pageFetchWaitElapsedMs), + pageFetchInFlightWaiterCount: metrics.pageFetchInFlightWaiterCount }); } diff --git a/utils/searchPerformance.test.js b/utils/searchPerformance.test.js index 5ec42f1..b4d9a75 100644 --- a/utils/searchPerformance.test.js +++ b/utils/searchPerformance.test.js @@ -44,8 +44,10 @@ describe('searchPerformance', () => { maxConcurrent: 4 }, inFlightWaitElapsedMs: 5.678, + inFlightWaiterCount: 4, pageFetchElapsedMs: 10.234, pageFetchWaitElapsedMs: 3.456, + pageFetchInFlightWaiterCount: 2, totalElapsedMs: 150.567 }, { @@ -94,8 +96,10 @@ describe('searchPerformance', () => { maxConcurrent: 4 }, inFlightWaitElapsedMs: 5.68, + inFlightWaiterCount: 4, pageFetchElapsedMs: 10.23, pageFetchWaitElapsedMs: 3.46, + pageFetchInFlightWaiterCount: 2, totalElapsedMs: 150.57 }); }); @@ -132,6 +136,10 @@ describe('searchPerformance', () => { maxQueueDepth: 500, maxWaitMs: 10000 }, + inFlightState: { + waiterCount: 500, + maxWaiters: 500 + }, totalElapsedMs: 10001.456 }, undefined, @@ -150,6 +158,10 @@ describe('searchPerformance', () => { statusCode: 429, strategy: 'cpe-candidate', cacheStatus: 'miss', + inFlightState: { + waiterCount: 500, + maxWaiters: 500 + }, coldSearchQueueWaitElapsedMs: 10000.12, coldSearchQueueState: { activeCount: 10, diff --git a/utils/searchPostFilterCache.js b/utils/searchPostFilterCache.js index f7ed9d8..adfd906 100644 --- a/utils/searchPostFilterCache.js +++ b/utils/searchPostFilterCache.js @@ -1,31 +1,48 @@ const searchProvider = require('./searchProvider'); const entries = new Map(); +const UNORDERED_QUERY_ARRAY_KEYS = new Set(['filter', 'must', 'must_not', 'should']); -function normalizeForKey(value) { - if (Array.isArray(value)) return value.map(normalizeForKey); +function compareNormalizedValues(left, right) { + const leftValue = JSON.stringify(left); + const rightValue = JSON.stringify(right); + + if (leftValue < rightValue) return -1; + if (leftValue > rightValue) return 1; + return 0; +} + +function normalizeForKey(value, { parentKey, lowercaseStrings = false } = {}) { + if (Array.isArray(value)) { + const normalized = value.map((item) => normalizeForKey(item, { lowercaseStrings })); + return UNORDERED_QUERY_ARRAY_KEYS.has(parentKey) ? normalized.sort(compareNormalizedValues) : normalized; + } if (value && typeof value === 'object') { + const caseInsensitiveValue = value.case_insensitive === true; return Object.keys(value) .sort() .reduce((normalized, key) => { if (value[key] !== undefined) { - normalized[key] = normalizeForKey(value[key]); + normalized[key] = normalizeForKey(value[key], { + parentKey: key, + lowercaseStrings: lowercaseStrings || (caseInsensitiveValue && key === 'value') + }); } return normalized; }, {}); } + if (lowercaseStrings && typeof value === 'string') return value.toLowerCase(); + return value; } function buildCacheKey(candidateBody, postFilters) { - return JSON.stringify( - normalizeForKey({ - candidateBody, - postFilters - }) - ); + return JSON.stringify({ + candidateBody: normalizeForKey(candidateBody), + postFilters: normalizeForKey(postFilters, { lowercaseStrings: true }) + }); } function get(cacheKey, now = Date.now()) { diff --git a/utils/searchPostFilterCache.test.js b/utils/searchPostFilterCache.test.js index 77aff79..4fd81c6 100644 --- a/utils/searchPostFilterCache.test.js +++ b/utils/searchPostFilterCache.test.js @@ -46,6 +46,91 @@ describe('searchPostFilterCache', () => { expect(left).toBe(right); }); + test('buildCacheKey is stable for equivalent boolean clause ordering', () => { + const left = searchPostFilterCache.buildCacheKey( + { + query: { + bool: { + filter: [{ term: { first: true } }, { term: { second: true } }] + } + } + }, + {} + ); + const right = searchPostFilterCache.buildCacheKey( + { + query: { + bool: { + filter: [{ term: { second: true } }, { term: { first: true } }] + } + } + }, + {} + ); + + expect(left).toBe(right); + }); + + test('buildCacheKey normalizes case-insensitive CPE values', () => { + const left = searchPostFilterCache.buildCacheKey( + { + query: { + prefix: { + criteria: { + value: 'cpe:2.3:A:Vendor:Product:', + case_insensitive: true + } + } + } + }, + { + virtualMatchString: { + raw: 'cpe:2.3:A:Vendor:Product' + } + } + ); + const right = searchPostFilterCache.buildCacheKey( + { + query: { + prefix: { + criteria: { + value: 'cpe:2.3:a:vendor:product:', + case_insensitive: true + } + } + } + }, + { + virtualMatchString: { + raw: 'cpe:2.3:a:vendor:product' + } + } + ); + + expect(left).toBe(right); + }); + + test('buildCacheKey preserves positional array ordering', () => { + const left = searchPostFilterCache.buildCacheKey( + {}, + { + cpeName: { + components: ['a', 'vendor', 'product'] + } + } + ); + const right = searchPostFilterCache.buildCacheKey( + {}, + { + cpeName: { + components: ['vendor', 'a', 'product'] + } + } + ); + + expect(left).not.toBe(right); + }); + test('get returns cached values until the entry expires', () => { const key = 'cache-key'; const value = { diff --git a/utils/searchProvider.js b/utils/searchProvider.js index 5b45207..18c0ea2 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -241,7 +241,18 @@ async function executeOpenSearchRequest(createRequest, { signal, operation, onSt } } -async function searchOpenSearch(body, { signal, operation = 'search', onStart } = {}) { +function getEffectiveOpenSearchRequestTimeoutMs(requestTimeoutMs) { + const configuredTimeoutMs = getOpenSearchRequestTimeoutMs(); + const requestedTimeoutMs = Number(requestTimeoutMs); + + if (!Number.isFinite(requestedTimeoutMs) || requestedTimeoutMs < 1) { + return configuredTimeoutMs; + } + + return Math.min(configuredTimeoutMs, Math.max(1, Math.floor(requestedTimeoutMs))); +} + +async function searchOpenSearch(body, { signal, operation = 'search', onStart, requestTimeoutMs } = {}) { searchDeadline.throwIfAborted(signal); const reader = getSearchReader(); const response = await executeOpenSearchRequest( @@ -254,7 +265,7 @@ async function searchOpenSearch(body, { signal, operation = 'search', onStart } }, { maxRetries: 0, - requestTimeout: getOpenSearchRequestTimeoutMs() + requestTimeout: getEffectiveOpenSearchRequestTimeoutMs(requestTimeoutMs) } ), { @@ -352,6 +363,22 @@ function getPostFilterColdSearchMaxWaitMs() { ); } +function getInFlightMaxWaitersPerOperation() { + return getIntegerConfig( + 'search.inFlightMaxWaitersPerOperation', + constants.SEARCH_IN_FLIGHT_MAX_WAITERS_PER_OPERATION, + { min: 1 } + ); +} + +function getPostFilterEvaluationChunkSize() { + return getIntegerConfig( + 'search.postFilterEvaluationChunkSize', + constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE, + { min: 1 } + ); +} + function getOpenSearchRequestTimeoutMs() { return getIntegerConfig('search.openSearchRequestTimeoutMs', constants.SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS, { min: 1 @@ -377,11 +404,13 @@ module.exports = { getIntegerConfig, getOpenSearchRequestState, getOpenSearchReadinessTimeoutMs, + getInFlightMaxWaitersPerOperation, getPostFilterCacheMaxEntries, getPostFilterCacheTtlMs, getPostFilterColdSearchConcurrency, getPostFilterColdSearchMaxQueueDepth, getPostFilterColdSearchMaxWaitMs, + getPostFilterEvaluationChunkSize, getOpenSearchRequestTimeoutMs, getSearchRequestDeadlineMs, getSearchProviderConfig, diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index e3d7c74..8d06bed 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -167,6 +167,58 @@ describe('searchProvider', () => { }); }); + test('caps an OpenSearch request timeout to the remaining request budget', async () => { + mockClientSearch.mockResolvedValue({ + body: { + hits: { + hits: [] + } + } + }); + + await searchProvider.searchOpenSearch( + { + query: { + match_all: {} + } + }, + { + requestTimeoutMs: 1500 + } + ); + + expect(mockClientSearch.mock.calls[0][1]).toStrictEqual({ + maxRetries: 0, + requestTimeout: 1500 + }); + }); + + test('does not extend an OpenSearch request beyond the configured timeout', async () => { + mockClientSearch.mockResolvedValue({ + body: { + hits: { + hits: [] + } + } + }); + + await searchProvider.searchOpenSearch( + { + query: { + match_all: {} + } + }, + { + requestTimeoutMs: 30000 + } + ); + + expect(mockClientSearch.mock.calls[0][1]).toStrictEqual({ + maxRetries: 0, + requestTimeout: 20000 + }); + }); + test.each([ [ 'timed-out', @@ -466,6 +518,8 @@ describe('searchProvider', () => { 'search.postFilterColdSearchConcurrency', 'search.postFilterColdSearchMaxQueueDepth', 'search.postFilterColdSearchMaxWaitMs', + 'search.inFlightMaxWaitersPerOperation', + 'search.postFilterEvaluationChunkSize', 'search.openSearchRequestTimeoutMs', 'search.requestDeadlineMs', 'search.readinessTimeoutMs' @@ -478,6 +532,8 @@ describe('searchProvider', () => { 'search.postFilterColdSearchConcurrency': '6', 'search.postFilterColdSearchMaxQueueDepth': '50', 'search.postFilterColdSearchMaxWaitMs': '15000', + 'search.inFlightMaxWaitersPerOperation': '125', + 'search.postFilterEvaluationChunkSize': '100', 'search.openSearchRequestTimeoutMs': '25000', 'search.requestDeadlineMs': '45000', 'search.readinessTimeoutMs': '7000' @@ -490,6 +546,8 @@ describe('searchProvider', () => { expect(searchProvider.getPostFilterColdSearchConcurrency()).toBe(6); expect(searchProvider.getPostFilterColdSearchMaxQueueDepth()).toBe(50); expect(searchProvider.getPostFilterColdSearchMaxWaitMs()).toBe(15000); + expect(searchProvider.getInFlightMaxWaitersPerOperation()).toBe(125); + expect(searchProvider.getPostFilterEvaluationChunkSize()).toBe(100); expect(searchProvider.getOpenSearchRequestTimeoutMs()).toBe(25000); expect(searchProvider.getSearchRequestDeadlineMs()).toBe(45000); expect(searchProvider.getOpenSearchReadinessTimeoutMs()).toBe(7000); @@ -504,6 +562,8 @@ describe('searchProvider', () => { expect(searchProvider.getPostFilterColdSearchConcurrency()).toBe(10); expect(searchProvider.getPostFilterColdSearchMaxQueueDepth()).toBe(500); expect(searchProvider.getPostFilterColdSearchMaxWaitMs()).toBe(10000); + expect(searchProvider.getInFlightMaxWaitersPerOperation()).toBe(500); + expect(searchProvider.getPostFilterEvaluationChunkSize()).toBe(250); expect(searchProvider.getOpenSearchRequestTimeoutMs()).toBe(20000); expect(searchProvider.getSearchRequestDeadlineMs()).toBe(30000); expect(searchProvider.getOpenSearchReadinessTimeoutMs()).toBe(5000); diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index 992bd00..f5c59ec 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -205,13 +205,17 @@ function buildDateRangeObjects(params) { return rangeObjects; } +function getOrderedSearchParamNames(params) { + return constants.SEARCH_QUERY_PARAMS.filter((param) => Object.prototype.hasOwnProperty.call(params, param)); +} + function processParams(params) { let allParams = { queryStrings: {}, matchPhrases: {} }; - Object.keys(params).forEach((param) => { + getOrderedSearchParamNames(params).forEach((param) => { if ( param === 'keywordSearch' && (!params['keywordExactMatch'] || String(params['keywordExactMatch']).toLowerCase() === 'false') @@ -254,7 +258,7 @@ function escapeQueryStringTerm(term, { escapeWildcards = false } = {}) { function buildQueryStrings(params) { const queryStringObjects = []; - Object.keys(params).forEach((param) => { + getOrderedSearchParamNames(params).forEach((param) => { if (param === 'virtualMatchString') return; const fieldSpec = constants.FIELDS[param]; @@ -297,7 +301,7 @@ function getFilterFields(params) { const cvssSeverityParams = new Set(['cvssV3Severity', 'cvssV3_0Severity', 'cvssV3_1Severity', 'cvssV4Severity']); - Object.keys(params).forEach((param) => { + getOrderedSearchParamNames(params).forEach((param) => { if (!validFields.includes(param)) return; const fieldSpec = constants.FIELDS[param]; @@ -380,7 +384,7 @@ function parseCveIds(value) { } function buildCveIdsFilter(value, field) { - const cveIds = [...new Set(parseCveIds(value))]; + const cveIds = [...new Set(parseCveIds(value))].sort(); if (!cveIds.length) return null; return { diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index 0fca830..e8f4b88 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -101,6 +101,21 @@ describe('searchQueryBuilder buildSearchRequest()', () => { }); }); + test('buildSearchRequest is stable across request property ordering', () => { + const first = queryBuilder.buildSearchRequest({ + keywordSearch: 'remote code', + cveIds: 'CVE-2024-0002,CVE-2024-0001', + cvssV3Metrics: 'AV:N/AC:L' + }); + const second = queryBuilder.buildSearchRequest({ + cvssV3Metrics: 'AC:L/AV:N', + cveIds: 'CVE-2024-0001,CVE-2024-0002', + keywordSearch: 'remote code' + }); + + expect(first).toStrictEqual(second); + }); + test('buildSearchRequest over-fetches bounded CPE candidates and keeps caller pagination for post-filtering', () => { const output = queryBuilder.buildSearchRequest({ cpeName: 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*', @@ -344,7 +359,7 @@ describe('searchQueryBuilder buildSearchRequest()', () => { filters: [ { terms: { - 'cveMetadata.cveId.keyword': ['CVE-2024-0002', 'CVE-2024-0001'] + 'cveMetadata.cveId.keyword': ['CVE-2024-0001', 'CVE-2024-0002'] } } ], @@ -432,7 +447,7 @@ describe('searchQueryBuilder getFilterFields() function', () => { test('getFilterFields() builds exact terms filter for comma-separated cveIds.', () => { const output = queryBuilder.getFilterFields({ - cveIds: 'CVE-2024-0001, cve-2024-0002,CVE-2024-0001' + cveIds: 'cve-2024-0002, CVE-2024-0001,CVE-2024-0002' }); expect(output).toStrictEqual([ @@ -504,7 +519,7 @@ describe('searchQueryBuilder getFilterFields() function', () => { }); expect(output).toStrictEqual([ - cvssRegexp('containers.cna.metrics.cvssV2_0.vectorString', cvssValue('AV:N', 'AC:L', 'Au:N')) + cvssRegexp('containers.cna.metrics.cvssV2_0.vectorString', cvssValue('AC:L', 'Au:N', 'AV:N')) ]); }); @@ -706,7 +721,7 @@ describe('searchQueryBuilder getFilterFields() function', () => { }); expect(output).toStrictEqual([ - cvssRegexp('containers.cna.metrics.cvssV4_0.vectorString', cvssValue('AV:N', 'AC:\\*\\?', 'UI:P')) + cvssRegexp('containers.cna.metrics.cvssV4_0.vectorString', cvssValue('AC:\\*\\?', 'AV:N', 'UI:P')) ]); expect(JSON.stringify(output)).not.toContain('containers.adp'); From 4440cb75ce1c4e12a03866eb021547cb6da7ade4 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 13:28:25 -0400 Subject: [PATCH 05/33] Add graceful runtime shutdown handling --- .env.example | 1 + README.md | 13 +- api-docs/openapi.json | 14 +- api-docs/openapi.test.js | 5 +- config/custom-environment-variables.jsonc | 3 + config/default.jsonc | 4 + config/runtime-config.test.js | 1 + routes/index.js | 20 ++- routes/index.test.js | 18 +++ server.js | 177 ++++++++++++++++++++- server.test.js | 179 ++++++++++++++++++++++ utils/constants.js | 2 + utils/errors.js | 8 + utils/errors.test.js | 16 ++ utils/runtimeLifecycle.js | 64 ++++++++ utils/runtimeLifecycle.test.js | 72 +++++++++ utils/searchProvider.js | 16 ++ utils/searchProvider.test.js | 21 +++ 18 files changed, 627 insertions(+), 7 deletions(-) create mode 100644 utils/runtimeLifecycle.js create mode 100644 utils/runtimeLifecycle.test.js diff --git a/.env.example b/.env.example index 823f158..795db04 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,7 @@ SearchRequestDeadlineMs=30000 SearchReadinessTimeoutMs=5000 RequestLogSuccessSampleRate=0.1 RequestLogHealthChecks=false +ShutdownTimeoutMs=35000 WebSearchEnabled=true SearchRateLimitEnabled=false SearchRateLimitWindowMs=60000 diff --git a/README.md b/README.md index 1155e76..429fe3d 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,14 @@ Rename `.env.example` to `.env` and replace or check the following values. - `SearchPostFilterColdSearchConcurrency` - optional per-process concurrent cold CPE candidate search limit, default `10` - `SearchPostFilterColdSearchMaxQueueDepth` - optional per-process waiting queue depth for cold CPE candidate searches, default `500` - `SearchPostFilterColdSearchMaxWaitMs` - optional maximum cold CPE candidate queue wait, default `10000` +- `SearchInFlightMaxWaitersPerOperation` - optional per-process caller limit for one shared semantic search operation, default `500` +- `SearchPostFilterEvaluationChunkSize` - optional semantic CPE cancellation/yield interval, default `250` - `SearchOpenSearchRequestTimeoutMs` - optional timeout for each OpenSearch request, default `20000` - `SearchRequestDeadlineMs` - optional overall `/search` processing deadline, default `30000` - `SearchReadinessTimeoutMs` - optional timeout for the OpenSearch index readiness check, default `5000` - `RequestLogSuccessSampleRate` - optional fraction of successful non-health-check access logs to retain, default `0.1` - `RequestLogHealthChecks` - optional flag to retain successful health-check access logs, default `false` +- `ShutdownTimeoutMs` - optional graceful HTTP shutdown deadline, default `35000` - `WebSearchEnabled` - optional deprecated `/webSearch` feature flag, default `true` - `SearchRateLimitEnabled` - optional per-process `/search` rate-limit flag, default `false` - `SearchRateLimitWindowMs` - optional rate-limit window in milliseconds, default `60000` @@ -89,10 +92,18 @@ The Search API will be served at `localhost:3000`. `GET /api/health-check` is a process liveness check. It returns HTTP 200 when the Node.js process is running and deliberately does not contact OpenSearch. Use it where a dependency outage should not restart otherwise healthy API tasks. -`GET /api/ready-check` sends a minimal zero-result search to the configured CVE index. This verifies connectivity, index availability, and the same search permission used by the API without retrieving CVE records or counting all matches. It returns HTTP 200 when search requests can be served. Configuration errors, connection failures, timeouts, incompatible `cve-core` SearchReader contracts, and a missing index return a sanitized HTTP 503 `SEARCH_NOT_READY` response with `Retry-After: 5`. The OpenSearch request is limited to five seconds by default and can be changed with `SearchReadinessTimeoutMs`. +`GET /api/ready-check` sends a minimal zero-result search to the configured CVE index. This verifies connectivity, index availability, and the same search permission used by the API without retrieving CVE records or counting all matches. It returns HTTP 200 when search requests can be served. Configuration errors, connection failures, timeouts, incompatible `cve-core` SearchReader contracts, a missing index, and an in-progress process shutdown return a sanitized HTTP 503 `SEARCH_NOT_READY` response with `Retry-After: 5`. The OpenSearch request is limited to five seconds by default and can be changed with `SearchReadinessTimeoutMs`. Once shutdown starts, readiness fails immediately without contacting OpenSearch. An AWS load balancer can use `/api/ready-check` to stop routing search traffic to an unready task while the container orchestrator continues using `/api/health-check` for process liveness. Coordinate that probe selection with the infrastructure team before changing deployed health-check behavior. +## Graceful Shutdown + +When the runtime receives `SIGTERM` or `SIGINT`, it marks the process as shutting down before closing the HTTP listener. Readiness changes to HTTP 503, while liveness remains HTTP 200 for as long as the process is running. Existing requests are allowed to drain. New non-probe requests attempted over existing keep-alive connections receive HTTP 503 `SERVICE_SHUTTING_DOWN`, `Retry-After: 5`, and `Connection: close`. + +The server closes idle connections and waits up to `ShutdownTimeoutMs`, which defaults to 35 seconds. A successful drain exits normally. If the deadline expires, the process force-closes remaining connections and exits with a failure code. Configure the container orchestrator's stop timeout to exceed `ShutdownTimeoutMs`; otherwise the orchestrator can terminate the process before its application-level drain completes. + +Shutdown lifecycle events are emitted as sanitized `runtime.lifecycle` JSON logs. Unexpected Express errors emit a sanitized `request.error` event and return HTTP 500 `INTERNAL_SERVER_ERROR`. Neither event includes exception messages, stacks, query strings, request bodies, credentials, or upstream response details. + ## API Input Validation `POST /search` accepts an optional JSON object request body and validates supported fields, boolean parameters, pagination values, CVSS severity values, required parameter combinations, CPE Name requirements, and date-time formats. A non-empty body must use the `application/json` media type with UTF-8 encoding; unsupported media types or encodings return HTTP 415, and malformed JSON returns HTTP 400. If OpenSearch is unavailable, the endpoint returns HTTP 503 with a `Retry-After` header. diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 22f7452..3d69e9e 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -109,6 +109,16 @@ } } }, + "500": { + "description": "An unexpected internal server error occurred", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, "502": { "description": "OpenSearch returned an invalid response or an unexpected upstream error occurred", "content": { @@ -120,7 +130,7 @@ } }, "503": { - "description": "OpenSearch is unavailable", + "description": "The service is shutting down or OpenSearch is unavailable", "headers": { "Retry-After": { "description": "Seconds before retrying an unavailable OpenSearch dependency", @@ -258,7 +268,7 @@ } }, "503": { - "description": "Search service is not ready", + "description": "Search service is shutting down or its OpenSearch dependency is not ready", "headers": { "Retry-After": { "description": "Seconds before retrying the readiness check", diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 35da775..3bee554 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -202,11 +202,13 @@ describe('OpenAPI contract', () => { } ]); - ['404', '415', '429', '502', '503', '504'].forEach((status) => { + ['404', '415', '429', '500', '502', '503', '504'].forEach((status) => { expect(responses[status].content['application/json'].schema).toStrictEqual({ $ref: '#/components/schemas/ErrorResponse' }); }); + + expect(responses['503'].description).toContain('shutting down'); }); test('documents request correlation headers', () => { @@ -219,6 +221,7 @@ describe('OpenAPI contract', () => { expect(openapi.paths['/api/health-check'].get.responses['200'].headers['X-Request-ID']).toBeDefined(); expect(openapi.paths['/api/ready-check'].get.responses['200'].headers['X-Request-ID']).toBeDefined(); expect(openapi.paths['/api/ready-check'].get.responses['503'].headers['Retry-After']).toBeDefined(); + expect(openapi.paths['/api/ready-check'].get.responses['503'].description).toContain('shutting down'); expect(openapi.paths['/webSearch'].post.responses['200'].headers['X-Request-ID']).toBeDefined(); }); diff --git a/config/custom-environment-variables.jsonc b/config/custom-environment-variables.jsonc index 8898e50..205aa1f 100644 --- a/config/custom-environment-variables.jsonc +++ b/config/custom-environment-variables.jsonc @@ -27,6 +27,9 @@ "successSampleRate": "RequestLogSuccessSampleRate", "healthChecks": "RequestLogHealthChecks" }, + "runtime": { + "shutdownTimeoutMs": "ShutdownTimeoutMs" + }, "features": { "webSearchEnabled": "WebSearchEnabled" }, diff --git a/config/default.jsonc b/config/default.jsonc index 58208d9..33561f2 100644 --- a/config/default.jsonc +++ b/config/default.jsonc @@ -43,6 +43,10 @@ // Set to TRUE to retain successful health-check access logs. "healthChecks": "FALSE" }, + "runtime": { + // Must remain shorter than the container orchestrator's stop timeout. + "shutdownTimeoutMs": "35000" + }, // Feature flags used for controlled endpoint migrations. "features": { // Keep enabled until existing consumers have migrated to POST /search. diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index d07d258..2ee6a0f 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -111,6 +111,7 @@ describe('runtime container and compose configuration', () => { expect(customEnvironmentVariables).toContain('"readinessTimeoutMs": "SearchReadinessTimeoutMs"'); expect(customEnvironmentVariables).toContain('"successSampleRate": "RequestLogSuccessSampleRate"'); expect(customEnvironmentVariables).toContain('"healthChecks": "RequestLogHealthChecks"'); + expect(customEnvironmentVariables).toContain('"shutdownTimeoutMs": "ShutdownTimeoutMs"'); expect(customEnvironmentVariables).toContain('"webSearchEnabled": "WebSearchEnabled"'); expect(customEnvironmentVariables).toContain('"enabled": "SearchRateLimitEnabled"'); expect(customEnvironmentVariables).toContain('"windowMs": "SearchRateLimitWindowMs"'); diff --git a/routes/index.js b/routes/index.js index 658d590..7a2ff47 100644 --- a/routes/index.js +++ b/routes/index.js @@ -8,6 +8,7 @@ const featureFlags = require('../utils/featureFlags'); const searchDeadline = require('../utils/searchDeadline'); const searchProvider = require('../utils/searchProvider'); const searchRateLimiter = require('../utils/searchRateLimiter'); +const runtimeLifecycle = require('../utils/runtimeLifecycle'); const error = new errors.errors(); const limitSearchRequests = searchRateLimiter.createSearchRateLimiter(); @@ -24,6 +25,11 @@ function healthCheck(req, res) { } async function readinessCheck(req, res) { + if (runtimeLifecycle.isShuttingDown()) { + if (typeof res.set === 'function') res.set('Retry-After', '5'); + return res.status(503).json(error.searchNotReadyError()); + } + const lifecycle = searchDeadline.createSearchRequestLifecycle( req, res, @@ -330,7 +336,7 @@ content: { } } #swagger.responses[503] = { -description: 'OpenSearch is unavailable', +description: 'The service is shutting down or OpenSearch is unavailable', headers: { "Retry-After": { description: "Seconds before retrying an unavailable OpenSearch dependency", @@ -357,6 +363,16 @@ content: { } } } +#swagger.responses[500] = { +description: 'An unexpected internal server error occurred', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} */ requireSearchJsonContentType, limitSearchRequests, @@ -432,7 +448,7 @@ headers: { } } #swagger.responses[503] = { -description: 'Search service is not ready', +description: 'Search service is shutting down or its OpenSearch dependency is not ready', headers: { "Retry-After": { description: "Seconds before retrying the readiness check", diff --git a/routes/index.test.js b/routes/index.test.js index c1cd095..9b7aff1 100644 --- a/routes/index.test.js +++ b/routes/index.test.js @@ -38,6 +38,7 @@ jest.mock('../utils/searchProvider', () => ({ const mw = require('../controllers/middleware'); const controller = require('../controllers/searchController'); const featureFlags = require('../utils/featureFlags'); +const runtimeLifecycle = require('../utils/runtimeLifecycle'); const router = require('./index'); describe('routes/index router', () => { @@ -47,6 +48,7 @@ describe('routes/index router', () => { app = createRouterApp(router); jest.clearAllMocks(); featureFlags.isWebSearchEnabled.mockReturnValue(true); + runtimeLifecycle.resetForTests(); mockLimitSearchRequests.mockImplementation((req, res, next) => next()); mockCheckOpenSearchReadiness.mockResolvedValue(true); }); @@ -89,6 +91,22 @@ describe('routes/index router', () => { expect(response.text).not.toContain('sensitive internal OpenSearch details'); }); + test('GET /api/ready-check fails without contacting OpenSearch during shutdown', async () => { + runtimeLifecycle.beginShutdown(); + + const response = await request(app, { + path: '/api/ready-check' + }); + + expect(response.status).toBe(503); + expect(response.headers['retry-after']).toBe('5'); + expect(response.body).toStrictEqual({ + error: 'SEARCH_NOT_READY', + message: 'Search service is not ready. Please try again later.' + }); + expect(mockCheckOpenSearchReadiness).not.toHaveBeenCalled(); + }); + test('POST /search reaches middleware and controller for valid JSON body fields', async () => { const response = await request(app, { method: 'POST', diff --git a/server.js b/server.js index 4c94a56..1a34142 100644 --- a/server.js +++ b/server.js @@ -9,10 +9,13 @@ const path = require('path'); const router = require('./routes/index'); const errors = require('./utils/errors'); const requestLogging = require('./utils/requestLogging'); +const runtimeLifecycle = require('./utils/runtimeLifecycle'); +const searchProvider = require('./utils/searchProvider'); dotenv.load(); const error = new errors.errors(); +const SHUTDOWN_PROBE_PATHS = new Set(['/api/health-check', '/api/ready-check']); const swaggerSetupOptions = { swaggerOptions: { @@ -50,12 +53,51 @@ function handleRequestError(err, req, res, next) { next(err); } +function getRouteCategory(req) { + const requestPath = requestLogging.getRedactedRequestPath(req); + if (['/search', '/webSearch', '/api/health-check', '/api/ready-check'].includes(requestPath)) { + return requestPath; + } + if (requestPath.startsWith('/api-docs')) return '/api-docs'; + + return 'other'; +} + +function rejectRequestsDuringShutdown(req, res, next) { + const requestPath = requestLogging.getRedactedRequestPath(req); + if (!runtimeLifecycle.isShuttingDown() || SHUTDOWN_PROBE_PATHS.has(requestPath)) { + return next(); + } + + res.set('Retry-After', '5'); + res.set('Connection', 'close'); + return res.status(503).json(error.serviceShuttingDownError()); +} + +function handleUnexpectedError(err, req, res, next) { + console.error( + JSON.stringify({ + event: 'request.error', + outcome: 'internal_error', + requestId: req.requestId, + route: getRouteCategory(req), + statusCode: 500 + }) + ); + + if (res.headersSent) return next(err); + + return res.status(500).json(error.internalServerError()); +} + function createApp() { const app = express(); app.use(requestLogging.assignRequestId); app.use(createRequestLogger()); app.use(cors()); + app.use(runtimeLifecycle.trackRequest); + app.use(rejectRequestsDuringShutdown); app.use(express.static(path.join(__dirname, 'public'))); app.use(express.json()); @@ -64,6 +106,7 @@ function createApp() { app.use('/', router); app.use(handleRequestError); + app.use(handleUnexpectedError); return app; } @@ -83,15 +126,147 @@ function startServer(port = process.env.PORT || 3000) { }); } +function logRuntimeEvent(log, event) { + log(JSON.stringify(event)); +} + +function shutdownServer( + server, + { + closeDependencies = searchProvider.closeSearchProviderClients, + exit = (code) => process.exit(code), + log = console.log, + setExitCode = (code) => { + if (process.exitCode === undefined) process.exitCode = code; + }, + signal = 'unknown', + timeoutMs = runtimeLifecycle.getShutdownTimeoutMs() + } = {} +) { + const startState = runtimeLifecycle.beginShutdown(); + logRuntimeEvent(log, { + event: 'runtime.lifecycle', + outcome: 'shutdown_started', + signal, + activeRequestCount: startState.activeRequestCount, + timeoutMs + }); + + return new Promise((resolve) => { + let settled = false; + + const complete = (outcome, exitCode, forceExit = false) => { + if (settled) return; + + settled = true; + clearTimeout(timeoutId); + const state = runtimeLifecycle.getState(); + const result = { + outcome, + exitCode, + activeRequestCount: state.activeRequestCount + }; + logRuntimeEvent(log, { + event: 'runtime.lifecycle', + ...result + }); + resolve(result); + + if (forceExit) { + exit(exitCode); + } else { + setExitCode(exitCode); + } + }; + + const timeoutId = setTimeout(() => { + if (settled) return; + + settled = true; + server.closeAllConnections?.(); + const state = runtimeLifecycle.getState(); + const result = { + outcome: 'shutdown_timeout', + exitCode: 1, + activeRequestCount: state.activeRequestCount + }; + logRuntimeEvent(log, { + event: 'runtime.lifecycle', + ...result + }); + resolve(result); + exit(1); + }, timeoutMs); + timeoutId.unref?.(); + + try { + server.close(async (err) => { + if (err) { + complete('shutdown_error', 1, true); + return; + } + + try { + await closeDependencies(); + complete('shutdown_complete', 0); + } catch (_err) { + complete('shutdown_error', 1, true); + } + }); + server.closeIdleConnections?.(); + } catch (_err) { + complete('shutdown_error', 1, true); + } + }); +} + +function installShutdownHandlers(server, { processRef = process, shutdownOptions = {} } = {}) { + let shutdownPromise; + + const handleSignal = (signal) => { + if (!shutdownPromise) { + shutdownPromise = shutdownServer(server, { + exit: (code) => processRef.exit(code), + setExitCode: (code) => { + if (processRef.exitCode === undefined) processRef.exitCode = code; + }, + ...shutdownOptions, + signal + }); + } + + return shutdownPromise; + }; + const handleSigterm = () => void handleSignal('SIGTERM'); + const handleSigint = () => void handleSignal('SIGINT'); + + processRef.on('SIGTERM', handleSigterm); + processRef.on('SIGINT', handleSigint); + + return { + dispose() { + processRef.removeListener('SIGTERM', handleSigterm); + processRef.removeListener('SIGINT', handleSigint); + }, + getShutdownPromise: () => shutdownPromise, + handleSignal + }; +} + if (require.main === module) { - startServer(); + const server = startServer(); + installShutdownHandlers(server); } module.exports = { createApp, createRequestLogger, + handleUnexpectedError, + installShutdownHandlers, getRedactedRequestPath: requestLogging.getRedactedRequestPath, getRuntimeConfig, + rejectRequestsDuringShutdown, + shutdownServer, startServer, swaggerSetupOptions }; diff --git a/server.test.js b/server.test.js index 51ce3c8..c847901 100644 --- a/server.test.js +++ b/server.test.js @@ -1,4 +1,5 @@ const http = require('http'); +const { EventEmitter } = require('events'); const { request } = require('./test/http'); const mockValidateParams = jest.fn((req, res, next) => next()); @@ -34,9 +35,13 @@ const { createRequestLogger, getRedactedRequestPath, getRuntimeConfig, + handleUnexpectedError, + installShutdownHandlers, + shutdownServer, startServer, swaggerSetupOptions } = require('./server'); +const runtimeLifecycle = require('./utils/runtimeLifecycle'); function restoreEnv(name, value) { if (value === undefined) { @@ -56,6 +61,8 @@ describe('server application bootstrap', () => { restoreEnv('BASE_URL', originalBaseUrl); restoreEnv('NODE_ENV', originalNodeEnv); restoreEnv('PORT', originalPort); + runtimeLifecycle.resetForTests(); + jest.useRealTimers(); jest.restoreAllMocks(); jest.clearAllMocks(); }); @@ -142,6 +149,37 @@ describe('server application bootstrap', () => { }); }); + test('createApp rejects non-probe requests after shutdown starts', async () => { + runtimeLifecycle.beginShutdown(); + + const response = await request(createApp(), { + method: 'POST', + path: '/search', + body: { + keywordSearch: 'apache' + } + }); + + expect(response.status).toBe(503); + expect(response.headers['retry-after']).toBe('5'); + expect(response.headers.connection).toBe('close'); + expect(response.body).toStrictEqual({ + error: 'SERVICE_SHUTTING_DOWN', + message: 'Search service is shutting down. Please retry later.' + }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test('createApp keeps process liveness available during shutdown', async () => { + runtimeLifecycle.beginShutdown(); + + const response = await request(createApp(), { + path: '/api/health-check' + }); + + expect(response.status).toBe(200); + }); + test('createApp returns a sanitized error for malformed JSON', async () => { const response = await request(createApp(), { method: 'POST', @@ -266,6 +304,147 @@ describe('server application bootstrap', () => { expect(fakeServer.listen).toHaveBeenCalledWith('4567', expect.any(Function)); }); + test('shutdownServer marks readiness unavailable and drains the HTTP server', async () => { + const fakeServer = { + close: jest.fn((callback) => callback()), + closeIdleConnections: jest.fn() + }; + const exit = jest.fn(); + const setExitCode = jest.fn(); + const log = jest.fn(); + const closeDependencies = jest.fn().mockResolvedValue(); + + await expect( + shutdownServer(fakeServer, { + closeDependencies, + exit, + log, + setExitCode, + signal: 'SIGTERM', + timeoutMs: 1000 + }) + ).resolves.toStrictEqual({ + outcome: 'shutdown_complete', + exitCode: 0, + activeRequestCount: 0 + }); + + expect(runtimeLifecycle.isShuttingDown()).toBe(true); + expect(fakeServer.close).toHaveBeenCalledTimes(1); + expect(fakeServer.closeIdleConnections).toHaveBeenCalledTimes(1); + expect(closeDependencies).toHaveBeenCalledTimes(1); + expect(setExitCode).toHaveBeenCalledWith(0); + expect(exit).not.toHaveBeenCalled(); + expect(log.mock.calls.map(([entry]) => JSON.parse(entry).outcome)).toEqual([ + 'shutdown_started', + 'shutdown_complete' + ]); + }); + + test('shutdownServer force-closes connections after its deadline', async () => { + const fakeServer = { + close: jest.fn(), + closeAllConnections: jest.fn(), + closeIdleConnections: jest.fn() + }; + const exit = jest.fn(); + const shutdown = shutdownServer(fakeServer, { + closeDependencies: jest.fn(), + exit, + log: jest.fn(), + setExitCode: jest.fn(), + timeoutMs: 5 + }); + + await expect(shutdown).resolves.toStrictEqual({ + outcome: 'shutdown_timeout', + exitCode: 1, + activeRequestCount: 0 + }); + expect(fakeServer.closeAllConnections).toHaveBeenCalledTimes(1); + expect(exit).toHaveBeenCalledWith(1); + }); + + test('installShutdownHandlers coalesces repeated shutdown signals', async () => { + const processRef = new EventEmitter(); + processRef.exit = jest.fn(); + processRef.exitCode = undefined; + let closeServer; + const fakeServer = { + close: jest.fn((callback) => { + closeServer = callback; + }), + closeIdleConnections: jest.fn() + }; + const handlers = installShutdownHandlers(fakeServer, { + processRef, + shutdownOptions: { + closeDependencies: jest.fn().mockResolvedValue(), + log: jest.fn(), + timeoutMs: 1000 + } + }); + + processRef.emit('SIGTERM'); + const firstShutdown = handlers.getShutdownPromise(); + processRef.emit('SIGTERM'); + processRef.emit('SIGINT'); + + expect(handlers.getShutdownPromise()).toBe(firstShutdown); + expect(fakeServer.close).toHaveBeenCalledTimes(1); + + closeServer(); + await expect(firstShutdown).resolves.toMatchObject({ + outcome: 'shutdown_complete' + }); + handlers.dispose(); + }); + + test('handleUnexpectedError returns and logs a sanitized internal error', () => { + const res = {}; + res.headersSent = false; + res.status = jest.fn(() => res); + res.json = jest.fn(() => res); + const next = jest.fn(); + jest.spyOn(console, 'error').mockImplementation(() => {}); + + handleUnexpectedError( + new Error('sensitive stack and credential details'), + { + path: '/private/value', + requestId: 'request-123' + }, + res, + next + ); + + expect(res.status).toHaveBeenCalledWith(500); + expect(res.json).toHaveBeenCalledWith({ + error: 'INTERNAL_SERVER_ERROR', + message: 'An unexpected server error occurred.' + }); + expect(next).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledTimes(1); + expect(JSON.parse(console.error.mock.calls[0][0])).toStrictEqual({ + event: 'request.error', + outcome: 'internal_error', + requestId: 'request-123', + route: 'other', + statusCode: 500 + }); + expect(console.error.mock.calls[0][0]).not.toContain('sensitive'); + }); + + test('handleUnexpectedError delegates after response headers were sent', () => { + const err = new Error('request failed after headers'); + const next = jest.fn(); + jest.spyOn(console, 'error').mockImplementation(() => {}); + + handleUnexpectedError(err, { path: '/search' }, { headersSent: true }, next); + + expect(next).toHaveBeenCalledWith(err); + }); + test('importing server does not start listening', () => { const createServer = jest.spyOn(http, 'createServer'); diff --git a/utils/constants.js b/utils/constants.js index 696bebd..6233699 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -26,6 +26,7 @@ const SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE = 250; const SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS = 20 * 1000; const SEARCH_REQUEST_DEADLINE_MS = 30 * 1000; const SEARCH_READINESS_TIMEOUT_MS = 5 * 1000; +const RUNTIME_SHUTDOWN_TIMEOUT_MS = 35 * 1000; const rejectedCveStates = ['REJECT', 'REJECTED', 'Rejected']; const rejectedCveStateFields = ['cveMetadata.state', 'containers.cna.x_legacyV4Record.CVE_data_meta.STATE']; @@ -156,6 +157,7 @@ module.exports = { SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS, SEARCH_REQUEST_DEADLINE_MS, SEARCH_READINESS_TIMEOUT_MS, + RUNTIME_SHUTDOWN_TIMEOUT_MS, RESULTS_PER_PAGE, REJECTED_CVE_STATES: rejectedCveStates, CPE_VERSION_RANGE_TYPES: cpeVersionRangeTypes, diff --git a/utils/errors.js b/utils/errors.js index 824b804..6b46440 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -285,6 +285,14 @@ class errors { return makeError('SEARCH_NOT_READY', 'Search service is not ready. Please try again later.'); } + serviceShuttingDownError() { + return makeError('SERVICE_SHUTTING_DOWN', 'Search service is shutting down. Please retry later.'); + } + + internalServerError() { + return makeError('INTERNAL_SERVER_ERROR', 'An unexpected server error occurred.'); + } + upstreamSearchError() { return makeError('UPSTREAM_SEARCH_ERROR', 'Search backend returned an error. Please try again later.'); } diff --git a/utils/errors.test.js b/utils/errors.test.js index 3258bf4..3d09f08 100644 --- a/utils/errors.test.js +++ b/utils/errors.test.js @@ -223,6 +223,22 @@ describe('error response helpers', () => { message: 'Search service is not ready. Please try again later.' } ], + [ + 'serviceShuttingDownError', + [], + { + error: 'SERVICE_SHUTTING_DOWN', + message: 'Search service is shutting down. Please retry later.' + } + ], + [ + 'internalServerError', + [], + { + error: 'INTERNAL_SERVER_ERROR', + message: 'An unexpected server error occurred.' + } + ], [ 'searchBodyError', [], diff --git a/utils/runtimeLifecycle.js b/utils/runtimeLifecycle.js new file mode 100644 index 0000000..cc2c86f --- /dev/null +++ b/utils/runtimeLifecycle.js @@ -0,0 +1,64 @@ +const { AppConfig } = require('cve-core'); +const constants = require('./constants'); + +let shuttingDown = false; +let activeRequestCount = 0; + +function getShutdownTimeoutMs() { + if (!AppConfig.has('runtime.shutdownTimeoutMs')) { + return constants.RUNTIME_SHUTDOWN_TIMEOUT_MS; + } + + const value = Number(AppConfig.get('runtime.shutdownTimeoutMs')); + if (Number.isInteger(value) && value >= 1) return value; + + return constants.RUNTIME_SHUTDOWN_TIMEOUT_MS; +} + +function getState() { + return { + activeRequestCount, + shuttingDown + }; +} + +function isShuttingDown() { + return shuttingDown; +} + +function beginShutdown() { + shuttingDown = true; + return getState(); +} + +function trackRequest(_req, res, next) { + activeRequestCount += 1; + let completed = false; + + const complete = () => { + if (completed) return; + + completed = true; + activeRequestCount = Math.max(0, activeRequestCount - 1); + res.removeListener?.('finish', complete); + res.removeListener?.('close', complete); + }; + + res.once('finish', complete); + res.once('close', complete); + next(); +} + +function resetForTests() { + shuttingDown = false; + activeRequestCount = 0; +} + +module.exports = { + beginShutdown, + getShutdownTimeoutMs, + getState, + isShuttingDown, + resetForTests, + trackRequest +}; diff --git a/utils/runtimeLifecycle.test.js b/utils/runtimeLifecycle.test.js new file mode 100644 index 0000000..a62c806 --- /dev/null +++ b/utils/runtimeLifecycle.test.js @@ -0,0 +1,72 @@ +const { EventEmitter } = require('events'); + +const mockAppConfigHas = jest.fn(); +const mockAppConfigGet = jest.fn(); + +jest.mock('cve-core', () => ({ + AppConfig: { + get: mockAppConfigGet, + has: mockAppConfigHas + } +})); + +const runtimeLifecycle = require('./runtimeLifecycle'); + +describe('runtimeLifecycle', () => { + beforeEach(() => { + runtimeLifecycle.resetForTests(); + mockAppConfigHas.mockReset(); + mockAppConfigGet.mockReset(); + mockAppConfigHas.mockReturnValue(false); + }); + + test('beginShutdown changes process readiness state idempotently', () => { + expect(runtimeLifecycle.getState()).toStrictEqual({ + activeRequestCount: 0, + shuttingDown: false + }); + + expect(runtimeLifecycle.beginShutdown()).toStrictEqual({ + activeRequestCount: 0, + shuttingDown: true + }); + expect(runtimeLifecycle.beginShutdown()).toStrictEqual({ + activeRequestCount: 0, + shuttingDown: true + }); + }); + + test('trackRequest counts a request until its response finishes or closes', () => { + const res = new EventEmitter(); + const next = jest.fn(); + + runtimeLifecycle.trackRequest({}, res, next); + + expect(next).toHaveBeenCalledTimes(1); + expect(runtimeLifecycle.getState().activeRequestCount).toBe(1); + + res.emit('finish'); + res.emit('close'); + + expect(runtimeLifecycle.getState().activeRequestCount).toBe(0); + expect(res.listenerCount('finish')).toBe(0); + expect(res.listenerCount('close')).toBe(0); + }); + + test('reads a positive shutdown timeout from config', () => { + mockAppConfigHas.mockImplementation((path) => path === 'runtime.shutdownTimeoutMs'); + mockAppConfigGet.mockReturnValue('45000'); + + expect(runtimeLifecycle.getShutdownTimeoutMs()).toBe(45000); + }); + + test.each(['not-a-number', '0', '-1', '1.5'])( + 'falls back to the shutdown timeout default for invalid value %s', + (value) => { + mockAppConfigHas.mockReturnValue(true); + mockAppConfigGet.mockReturnValue(value); + + expect(runtimeLifecycle.getShutdownTimeoutMs()).toBe(35000); + } + ); +}); diff --git a/utils/searchProvider.js b/utils/searchProvider.js index 18c0ea2..e9244f1 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -170,6 +170,21 @@ function getSearchReader() { return searchReader; } +async function closeSearchProviderClients() { + const clients = new Set( + [basicSearchManager?._searchReader?._client, searchReader?._client].filter( + (client) => typeof client?.close === 'function' + ) + ); + + const results = await Promise.allSettled([...clients].map((client) => client.close())); + basicSearchManager = undefined; + searchReader = undefined; + + const failedClose = results.find((result) => result.status === 'rejected'); + if (failedClose) throw failedClose.reason; +} + function getOpenSearchRequestState() { return { activeCount: activeOpenSearchRequestCount, @@ -399,6 +414,7 @@ function resetSearchProviderForTests() { module.exports = { checkOpenSearchReadiness, + closeSearchProviderClients, getBasicSearchManager, getAllowUnknownSslCerts, getIntegerConfig, diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index 8d06bed..f9ceae7 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -2,6 +2,7 @@ const mockBasicSearchManager = jest.fn(function BasicSearchManager(spec) { this.spec = spec; this._searchReader = { _client: { + close: jest.fn().mockResolvedValue(), transport: { maxRetries: 3, requestTimeout: 30000 @@ -10,11 +11,13 @@ const mockBasicSearchManager = jest.fn(function BasicSearchManager(spec) { }; }); const mockClientSearch = jest.fn(); +const mockClientClose = jest.fn(); const mockSearchReader = jest.fn(function SearchReader(endpoint, index, options) { this._openSearchDomainEndpoint = endpoint; this._cveIndex = index; this._options = options; this._client = { + close: mockClientClose, search: mockClientSearch, transport: { maxRetries: 3, @@ -57,6 +60,8 @@ describe('searchProvider', () => { mockBasicSearchManager.mockClear(); mockSearchReader.mockClear(); mockClientSearch.mockReset(); + mockClientClose.mockReset(); + mockClientClose.mockResolvedValue(); mockGetDefaultSearchProviderSpec.mockReset(); mockGetDefaultSearchProviderSpec.mockReturnValue({ providerEndpoint: 'https://search.example', @@ -167,6 +172,22 @@ describe('searchProvider', () => { }); }); + test('closes initialized OpenSearch clients and allows them to be recreated', async () => { + const manager = searchProvider.getBasicSearchManager(); + searchProvider.getSearchReader(); + + await searchProvider.closeSearchProviderClients(); + + expect(manager._searchReader._client.close).toHaveBeenCalledTimes(1); + expect(mockClientClose).toHaveBeenCalledTimes(1); + + searchProvider.getBasicSearchManager(); + searchProvider.getSearchReader(); + + expect(mockBasicSearchManager).toHaveBeenCalledTimes(2); + expect(mockSearchReader).toHaveBeenCalledTimes(2); + }); + test('caps an OpenSearch request timeout to the remaining request budget', async () => { mockClientSearch.mockResolvedValue({ body: { From e0388f4d027d14209d810fbb921a1a05904beea1 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 14:07:08 -0400 Subject: [PATCH 06/33] Expand CI runtime validation coverage --- .github/workflows/quality.yml | 50 +++++++++++++++++++++ README.md | 19 ++++++++ config/runtime-config.test.js | 12 +++++ package.json | 3 +- server.test.js | 80 ++++++++++++++++++++++++++++++++++ test/contract/opensearch.js | 65 +++++++++++++++++++++++++++ utils/runtimeLifecycle.test.js | 5 +++ utils/searchProvider.test.js | 18 ++++++++ 8 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 test/contract/opensearch.js diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 95eff65..8de9c02 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -31,3 +31,53 @@ jobs: - name: Run quality checks run: npm run quality + + container: + name: Runtime Container + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Build production image + run: docker build --tag cve-search-api:pr . + + - name: Start production image + run: | + docker run --detach \ + --name cve-search-api-pr \ + --publish 4100:3000 \ + --env NODE_CONFIG_ENV=prod \ + --env OpenSearchDomainEndpoint=https://search.example.com \ + --env OpenSearchCveIndex=cves \ + cve-search-api:pr + + - name: Validate runtime image + run: | + for attempt in $(seq 1 30); do + if curl --fail --silent --show-error http://127.0.0.1:4100/api/health-check; then + break + fi + if [ "$attempt" -eq 30 ]; then + exit 1 + fi + sleep 1 + done + + curl --fail --silent --show-error http://127.0.0.1:4100/api-docs/openapi.json \ + | jq --exit-status \ + '.openapi == "3.0.2" and .info.title == "CVE Search API"' \ + > /dev/null + + test "$(docker image inspect cve-search-api:pr --format '{{.Config.User}}')" = "65532:65532" + test "$(docker image inspect cve-search-api:pr --format '{{json .Config.Cmd}}')" = '["server.js"]' + + - name: Capture container logs + if: always() + run: docker logs cve-search-api-pr || true + + - name: Remove test container + if: always() + run: docker rm --force cve-search-api-pr || true diff --git a/README.md b/README.md index 429fe3d..cf7fa7e 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,25 @@ npm run coverage npm run quality ``` +Pull requests run the quality gate and independently build the production `Dockerfile`. The container job starts the built image, checks process liveness and the packaged OpenAPI document, and verifies the runtime user and command. It intentionally does not call `/api/ready-check` or `/search`, because pull-request CI does not own or provision an OpenSearch index. + +An opt-in, read-only OpenSearch contract check is available for an approved environment that can reach an existing index: + +```bash +npm run test:opensearch-contract +``` + +Set `OpenSearchDomainEndpoint` and `OpenSearchCveIndex` in the environment or local `.env` before running it. `OpenSearchAllowUnknownSslCerts` remains optional and should be enabled only for local development against a self-signed certificate. The contract check sends a minimal readiness query and one single-hit search, validates the response shape, closes its OpenSearch client, and emits sanitized JSON output. It never creates, updates, maps, or deletes indexes. + +The contract check is not part of public pull-request CI. A future automated invocation should use a protected GitHub environment or self-hosted runner with network access to an approved test index. Connection values and credentials must remain in environment secrets rather than source files or workflow text. + +The validation tiers have separate purposes: + +- Jest and `npm run quality` verify application behavior, API contracts, and regression coverage with controlled test doubles. +- Pull-request container validation verifies production-image packaging, non-root execution, startup, liveness, and packaged API documentation. +- The optional OpenSearch contract check verifies that the installed client can communicate with an externally managed index using the response contract expected by this application. +- Load and performance testing remain in the separate automation performance-testing repository. + ### Operational Logging Every request receives an `X-Request-ID` response header. A caller-provided `X-Request-ID` is preserved when it contains only letters, numbers, `.`, `_`, `:`, or `-` and is no longer than 128 characters; otherwise the API generates a UUID. The same value appears as `request-id` in access logs and `requestId` in structured `/search` events. diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index 2ee6a0f..4c969ec 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -69,6 +69,18 @@ describe('runtime container and compose configuration', () => { expect(dockerignore).toContain('**/reports'); }); + test('pull request CI builds and smoke tests the production runtime image', () => { + const workflow = readProjectFile('.github', 'workflows', 'quality.yml'); + + expect(workflow).toContain('name: Runtime Container'); + expect(workflow).toContain('docker build --tag cve-search-api:pr .'); + expect(workflow).toContain('http://127.0.0.1:4100/api/health-check'); + expect(workflow).toContain('http://127.0.0.1:4100/api-docs/openapi.json'); + expect(workflow).toContain(`--format '{{.Config.User}}'`); + expect(workflow).toContain(`--format '{{json .Config.Cmd}}'`); + expect(workflow).not.toContain('/api/ready-check'); + }); + test('server-only compose file joins the existing OpenSearch 3.5 network', () => { const compose = readProjectFile('docker-compose.yaml'); diff --git a/package.json b/package.json index 2a59a9f..346a0c2 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "quality": "npm run lint && npm run format:check && npm run coverage", "start": "node server.js", "swagger-autogen": "node routes/swagger.js", - "test": "jest" + "test": "jest", + "test:opensearch-contract": "node test/contract/opensearch.js" }, "dependencies": { "cors": "^2.8.5", diff --git a/server.test.js b/server.test.js index c847901..bb829df 100644 --- a/server.test.js +++ b/server.test.js @@ -365,6 +365,86 @@ describe('server application bootstrap', () => { expect(exit).toHaveBeenCalledWith(1); }); + test('shutdownServer exits with failure when the HTTP server reports a close error', async () => { + const closeDependencies = jest.fn(); + const exit = jest.fn(); + const log = jest.fn(); + const fakeServer = { + close: jest.fn((callback) => callback(new Error('sensitive server close failure'))), + closeIdleConnections: jest.fn() + }; + + await expect( + shutdownServer(fakeServer, { + closeDependencies, + exit, + log, + setExitCode: jest.fn(), + timeoutMs: 1000 + }) + ).resolves.toMatchObject({ + outcome: 'shutdown_error', + exitCode: 1 + }); + + expect(closeDependencies).not.toHaveBeenCalled(); + expect(exit).toHaveBeenCalledWith(1); + expect(log.mock.calls.map(([entry]) => entry).join(' ')).not.toContain('sensitive'); + }); + + test('shutdownServer exits with failure when dependency cleanup rejects', async () => { + const exit = jest.fn(); + const log = jest.fn(); + const fakeServer = { + close: jest.fn((callback) => callback()), + closeIdleConnections: jest.fn() + }; + + await expect( + shutdownServer(fakeServer, { + closeDependencies: jest.fn().mockRejectedValue(new Error('sensitive dependency close failure')), + exit, + log, + setExitCode: jest.fn(), + timeoutMs: 1000 + }) + ).resolves.toMatchObject({ + outcome: 'shutdown_error', + exitCode: 1 + }); + + expect(exit).toHaveBeenCalledWith(1); + expect(log.mock.calls.map(([entry]) => entry).join(' ')).not.toContain('sensitive'); + }); + + test('shutdownServer exits with failure when closing the HTTP server throws', async () => { + const exit = jest.fn(); + const log = jest.fn(); + const fakeServer = { + close: jest.fn(() => { + throw new Error('sensitive synchronous close failure'); + }), + closeIdleConnections: jest.fn() + }; + + await expect( + shutdownServer(fakeServer, { + closeDependencies: jest.fn(), + exit, + log, + setExitCode: jest.fn(), + timeoutMs: 1000 + }) + ).resolves.toMatchObject({ + outcome: 'shutdown_error', + exitCode: 1 + }); + + expect(fakeServer.closeIdleConnections).not.toHaveBeenCalled(); + expect(exit).toHaveBeenCalledWith(1); + expect(log.mock.calls.map(([entry]) => entry).join(' ')).not.toContain('sensitive'); + }); + test('installShutdownHandlers coalesces repeated shutdown signals', async () => { const processRef = new EventEmitter(); processRef.exit = jest.fn(); diff --git a/test/contract/opensearch.js b/test/contract/opensearch.js new file mode 100644 index 0000000..0467d2d --- /dev/null +++ b/test/contract/opensearch.js @@ -0,0 +1,65 @@ +const assert = require('assert'); +require('dotenv').config(); + +const REQUIRED_ENVIRONMENT_VARIABLES = ['OpenSearchDomainEndpoint', 'OpenSearchCveIndex']; + +function requireConfiguration() { + const missing = REQUIRED_ENVIRONMENT_VARIABLES.filter((name) => !process.env[name]?.trim()); + if (missing.length > 0) { + throw new Error(`Missing required environment variables: ${missing.join(', ')}`); + } +} + +function getSanitizedFailure(err) { + const statusCode = Number(err?.meta?.statusCode ?? err?.statusCode); + + return { + event: 'opensearch.contract', + outcome: 'failed', + errorCode: typeof err?.code === 'string' ? err.code : undefined, + failureType: typeof err?.failureType === 'string' ? err.failureType : undefined, + statusCode: Number.isInteger(statusCode) ? statusCode : undefined + }; +} + +async function run() { + requireConfiguration(); + + const searchProvider = require('../../utils/searchProvider'); + try { + await searchProvider.checkOpenSearchReadiness(); + const body = await searchProvider.searchOpenSearch( + { + query: { + match_all: {} + }, + size: 1, + track_total_hits: false, + _source: ['cveMetadata.cveId'] + }, + { + operation: 'contract-check' + } + ); + + assert.ok(Array.isArray(body.hits.hits)); + if (body.hits.hits.length > 0) { + assert.strictEqual(typeof body.hits.hits[0]._id, 'string'); + } + + process.stdout.write( + `${JSON.stringify({ + event: 'opensearch.contract', + outcome: 'passed', + returnedHits: body.hits.hits.length + })}\n` + ); + } finally { + await searchProvider.closeSearchProviderClients(); + } +} + +run().catch((err) => { + process.stderr.write(`${JSON.stringify(getSanitizedFailure(err))}\n`); + process.exitCode = 1; +}); diff --git a/utils/runtimeLifecycle.test.js b/utils/runtimeLifecycle.test.js index a62c806..47b2571 100644 --- a/utils/runtimeLifecycle.test.js +++ b/utils/runtimeLifecycle.test.js @@ -60,6 +60,11 @@ describe('runtimeLifecycle', () => { expect(runtimeLifecycle.getShutdownTimeoutMs()).toBe(45000); }); + test('uses the default shutdown timeout when no override is configured', () => { + expect(runtimeLifecycle.getShutdownTimeoutMs()).toBe(35000); + expect(mockAppConfigGet).not.toHaveBeenCalled(); + }); + test.each(['not-a-number', '0', '-1', '1.5'])( 'falls back to the shutdown timeout default for invalid value %s', (value) => { diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index f9ceae7..159aac2 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -188,6 +188,24 @@ describe('searchProvider', () => { expect(mockSearchReader).toHaveBeenCalledTimes(2); }); + test('attempts to close every initialized OpenSearch client before reporting a close failure', async () => { + const manager = searchProvider.getBasicSearchManager(); + searchProvider.getSearchReader(); + const closeFailure = new Error('sensitive client close failure'); + manager._searchReader._client.close.mockRejectedValue(closeFailure); + + await expect(searchProvider.closeSearchProviderClients()).rejects.toBe(closeFailure); + + expect(manager._searchReader._client.close).toHaveBeenCalledTimes(1); + expect(mockClientClose).toHaveBeenCalledTimes(1); + + searchProvider.getBasicSearchManager(); + searchProvider.getSearchReader(); + + expect(mockBasicSearchManager).toHaveBeenCalledTimes(2); + expect(mockSearchReader).toHaveBeenCalledTimes(2); + }); + test('caps an OpenSearch request timeout to the remaining request budget', async () => { mockClientSearch.mockResolvedValue({ body: { From 200538a5e1458775935245ea5a40ff35ad1fd854 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 14:28:11 -0400 Subject: [PATCH 07/33] Remove stale search maintenance paths --- .github/workflows/quality.yml | 6 +- controllers/middleware.js | 2 +- controllers/middleware.test.js | 4 +- controllers/searchController.js | 64 +---------- controllers/searchController.test.js | 156 ++++++++++++--------------- jest.config.js | 8 +- package-lock.json | 45 -------- package.json | 2 - server.js | 2 - utils/constants.js | 15 --- utils/searchPerformance.js | 1 - utils/searchPerformance.test.js | 1 - 12 files changed, 81 insertions(+), 225 deletions(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 8de9c02..7671e70 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -18,10 +18,10 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 24.14.1 cache: npm @@ -39,7 +39,7 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Build production image run: docker build --tag cve-search-api:pr . diff --git a/controllers/middleware.js b/controllers/middleware.js index 524d052..cdbba5a 100644 --- a/controllers/middleware.js +++ b/controllers/middleware.js @@ -117,7 +117,7 @@ function isValidDateTime(value) { //Ensures parameters are valid and checks for parameters that can't be called together function validateParams(req, res, next) { try { - const params = req.searchParams ?? req.query ?? {}; + const params = req.searchParams ?? {}; //Ensures all specified parameter names are valid for (const param of Object.keys(params)) { diff --git a/controllers/middleware.test.js b/controllers/middleware.test.js index 0261e9b..c793a8f 100644 --- a/controllers/middleware.test.js +++ b/controllers/middleware.test.js @@ -926,8 +926,8 @@ describe('middleware validation function', () => { ); test('ValidateParams delegates unexpected errors to next()', () => { - const err = new Error('query failure'); - Object.defineProperty(req, 'query', { + const err = new Error('search parameter failure'); + Object.defineProperty(req, 'searchParams', { get: () => { throw err; } diff --git a/controllers/searchController.js b/controllers/searchController.js index f7e00e0..e954f28 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -56,18 +56,6 @@ function buildEmptyFilteredResult(hitsContainer) { }; } -function filterSearchHits(resp, postFilters) { - const hitsContainer = getHitsContainer(resp); - if (!Array.isArray(hitsContainer?.hits)) return buildEmptyFilteredResult(hitsContainer); - - const cpeEvaluationContext = cpe.createEvaluationContext(); - return { - hitsContainer, - candidateCount: hitsContainer.hits.length, - filteredHits: hitsContainer.hits.filter((hit) => hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext)) - }; -} - function yieldToEventLoop() { return new Promise((resolve) => setImmediate(resolve)); } @@ -153,15 +141,6 @@ function buildPaginationMetadata(resp, pagingOptions) { }; } -function buildPostFilteredResponse( - resp, - filteredHits, - responseOptions, - pagedHits = paginateHits(filteredHits, responseOptions) -) { - return buildPagedSearchResponse(resp, filteredHits.length, pagedHits); -} - function getOpenSearchErrorStatusCode(err) { return Number(err?.statusCode ?? err?.meta?.statusCode ?? err?.meta?.body?.status); } @@ -208,22 +187,6 @@ function buildWebSearchSort(sort) { return undefined; } -function applySearchPostFilters(resp, postFilters, responseOptions, metrics) { - if (!hasSearchPostFilters(postFilters)) return resp; - - const { hitsContainer, candidateCount, filteredHits } = filterSearchHits(resp, postFilters); - if (!Array.isArray(hitsContainer?.hits)) return resp; - - if (metrics) { - const pagedHits = paginateHits(filteredHits, responseOptions); - metrics.candidateCount = candidateCount; - metrics.filteredCount = filteredHits.length; - metrics.returnedCount = pagedHits.length; - } - - return buildPostFilteredResponse(resp, filteredHits, responseOptions); -} - function getHitCveId(hit) { return hit?._source?.cveMetadata?.cveId ?? hit?._id; } @@ -260,14 +223,7 @@ async function executeOpenSearch(body, operation, signal, metrics, requestTimeou }); } -async function executeStandardSearch( - searchOptions, - queryStringObjects, - postFilters, - responseOptions, - metrics, - lifecycle -) { +async function executeStandardSearch(searchOptions, queryStringObjects, metrics, lifecycle) { metrics.strategy = 'standard'; const searchStart = searchPerformance.markTime(); @@ -280,11 +236,7 @@ async function executeStandardSearch( metrics.openSearchElapsedMs = searchPerformance.elapsedMs(searchStart); metrics.openSearchTookMs = searchData?.took; - const postFilterStart = searchPerformance.markTime(); - const resp = applySearchPostFilters(searchResp, postFilters, responseOptions, metrics); - metrics.postFilterElapsedMs = searchPerformance.elapsedMs(postFilterStart); - - return resp; + return searchResp; } async function fetchFullHitsByCveIds(cveIds, metrics, signal) { @@ -422,14 +374,7 @@ async function executeOptimizedPostFilterSearch( async function executeSearch(searchOptions, queryStringObjects, postFilters, responseOptions, metrics, lifecycle) { if (!hasSearchPostFilters(postFilters)) { - return executeStandardSearch( - searchOptions, - queryStringObjects, - postFilters, - responseOptions, - metrics, - lifecycle - ); + return executeStandardSearch(searchOptions, queryStringObjects, metrics, lifecycle); } return executeOptimizedPostFilterSearch( @@ -443,7 +388,7 @@ async function executeSearch(searchOptions, queryStringObjects, postFilters, res } async function searchHandler(req, res, next) { - const params = req.searchParams ?? req.query ?? {}; + const params = req.searchParams ?? {}; const metrics = { requestId: req.requestId }; @@ -641,7 +586,6 @@ async function webSearchHandler(req, res, next) { } module.exports = { - applySearchPostFilters, buildWebSearchSort, filterSearchHitsInChunks, SEARCH: searchHandler, diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 69fb4f2..245415d 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -212,7 +212,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { keywordSearch: 'remote code', cveIds: 'CVE-2024-0001', pubStartDate: '2024-01-01T00:00:00.000Z', @@ -302,7 +302,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { keywordSearch: 'remote code', keywordExactMatch: 'true' } @@ -328,7 +328,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { cveIds: 'CVE-2024-0001,CVE-2024-0002' } }, @@ -351,7 +351,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { noRejected: true } }, @@ -389,7 +389,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:a:microsoft:windows:*:*:*:*:*:*:*:*' } }, @@ -438,7 +438,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { cweId: 'CWE-79', cpeName: 'cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*', isVulnerable: 'true' @@ -503,7 +503,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { cpeName: 'cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*', isVulnerable: true } @@ -547,7 +547,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { cpeName: 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*', resultsPerPage: '1', pageNumber: '1' @@ -604,7 +604,7 @@ describe('Search controller SEARCH handler', () => { }); }); - test('applySearchPostFilters does not combine platform and version matches from different CPE entries', () => { + test('chunked filtering does not combine platform and version matches from different CPE entries', async () => { const candidate = cpeHit('CVE-2026-0250', { criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:macos:*:*:*:*:*', vulnerable: true, @@ -632,23 +632,20 @@ describe('Search controller SEARCH handler', () => { } ); - const response = controller.applySearchPostFilters( + const { filteredHits } = await controller.filterSearchHitsInChunks( advancedResponse(1, [candidate]), { cpeName: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:6.1.1:*:macos:*:*:*:*:*', isVulnerable: true }, - { - from: 0, - resultsPerPage: 20 - } + undefined, + 100 ); - expect(response.data.hits.total.value).toBe(0); - expect(response.data.hits.hits).toStrictEqual([]); + expect(filteredHits).toStrictEqual([]); }); - test('applySearchPostFilters matches CPE string literals without lexical case', () => { + test('chunked filtering matches CPE string literals without lexical case', async () => { const candidate = cpeHit('CVE-2026-0251', { criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:*:*:*:macOS:*:*', vulnerable: true, @@ -656,23 +653,20 @@ describe('Search controller SEARCH handler', () => { versionEndExcluding: '6.1' }); - const response = controller.applySearchPostFilters( + const { filteredHits } = await controller.filterSearchHitsInChunks( advancedResponse(1, [candidate]), { cpeName: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:6.0.1:*:*:*:*:macos:*:*', isVulnerable: true }, - { - from: 0, - resultsPerPage: 20 - } + undefined, + 100 ); - expect(response.data.hits.total.value).toBe(1); - expect(response.data.hits.hits).toStrictEqual([candidate]); + expect(filteredHits).toStrictEqual([candidate]); }); - test('applySearchPostFilters preserves exact version padding while normalizing numeric range segments', () => { + test('chunked filtering preserves exact version padding while normalizing numeric range segments', async () => { const exactCandidate = cpeHit('CVE-2025-9392', { criteria: 'cpe:2.3:o:linksys:re6250_firmware:1.0.04.001:*:*:*:*:*:*:*', vulnerable: true @@ -683,46 +677,40 @@ describe('Search controller SEARCH handler', () => { versionEndExcluding: '6.02' }); - const exactResponse = controller.applySearchPostFilters( + const exactResult = await controller.filterSearchHitsInChunks( advancedResponse(1, [exactCandidate]), { cpeName: 'cpe:2.3:o:linksys:re6250_firmware:1.0.4.1:*:*:*:*:*:*:*', isVulnerable: true }, - { - from: 0, - resultsPerPage: 20 - } + undefined, + 100 ); - const includedRangeResponse = controller.applySearchPostFilters( + const includedRangeResult = await controller.filterSearchHitsInChunks( advancedResponse(1, [rangeCandidate]), { cpeName: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:6.1:*:*:*:*:*:*:*', isVulnerable: true }, - { - from: 0, - resultsPerPage: 20 - } + undefined, + 100 ); - const excludedRangeResponse = controller.applySearchPostFilters( + const excludedRangeResult = await controller.filterSearchHitsInChunks( advancedResponse(1, [rangeCandidate]), { cpeName: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:6.2:*:*:*:*:*:*:*', isVulnerable: true }, - { - from: 0, - resultsPerPage: 20 - } + undefined, + 100 ); - expect(exactResponse.data.hits.total.value).toBe(0); - expect(includedRangeResponse.data.hits.hits).toStrictEqual([rangeCandidate]); - expect(excludedRangeResponse.data.hits.total.value).toBe(0); + expect(exactResult.filteredHits).toStrictEqual([]); + expect(includedRangeResult.filteredHits).toStrictEqual([rangeCandidate]); + expect(excludedRangeResult.filteredHits).toStrictEqual([]); }); - test('applySearchPostFilters preserves CPE NA matches across numeric version ranges', () => { + test('chunked filtering preserves CPE NA matches across numeric version ranges', async () => { const criteria = 'cpe:2.3:a:microsoft:edge:*:*:*:*:*:android:*:*'; const candidates = [ cpeHit('CVE-2020-17153', { @@ -744,20 +732,17 @@ describe('Search controller SEARCH handler', () => { }) ]; - const response = controller.applySearchPostFilters( + const { filteredHits } = await controller.filterSearchHitsInChunks( advancedResponse(candidates.length, candidates), { cpeName: 'cpe:2.3:a:microsoft:edge:-:*:*:*:*:android:*:*', isVulnerable: true }, - { - from: 0, - resultsPerPage: 20 - } + undefined, + 100 ); - expect(response.data.hits.total.value).toBe(3); - expect(response.data.hits.hits.map((hit) => hit._id)).toStrictEqual([ + expect(filteredHits.map((hit) => hit._id)).toStrictEqual([ 'CVE-2020-17153', 'CVE-2024-38208', 'CVE-2026-35429' @@ -790,7 +775,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', versionStart: '2.2', versionStartType: 'including', @@ -857,7 +842,7 @@ describe('Search controller SEARCH handler', () => { } }; const req = { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', versionStart: '2.2', versionStartType: 'including', @@ -925,7 +910,7 @@ describe('Search controller SEARCH handler', () => { } }; const req = { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', versionStart: '2.2', versionStartType: 'including', @@ -984,13 +969,13 @@ describe('Search controller SEARCH handler', () => { }; const firstReq = new EventEmitter(); const secondReq = new EventEmitter(); - const query = { + const searchParams = { virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', resultsPerPage: 1, pageNumber: 1 }; - firstReq.query = query; - secondReq.query = query; + firstReq.searchParams = searchParams; + secondReq.searchParams = searchParams; const firstRes = createResponse(); const secondRes = createResponse(); let operationSignal; @@ -1045,7 +1030,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809', resultsPerPage: '1', pageNumber: '1' @@ -1109,7 +1094,7 @@ describe('Search controller SEARCH handler', () => { const firstSearch = controller.SEARCH( { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', resultsPerPage: '1', pageNumber: '1' @@ -1123,7 +1108,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { requestId: 'queue-full-request', - query: { + searchParams: { virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809', resultsPerPage: '1', pageNumber: '1' @@ -1172,7 +1157,7 @@ describe('Search controller SEARCH handler', () => { const firstSearch = controller.SEARCH( { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', resultsPerPage: '1', pageNumber: '1' @@ -1186,7 +1171,7 @@ describe('Search controller SEARCH handler', () => { const secondSearch = controller.SEARCH( { requestId: 'queue-timeout-request', - query: { + searchParams: { virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809', resultsPerPage: '1', pageNumber: '1' @@ -1225,7 +1210,7 @@ describe('Search controller SEARCH handler', () => { const firstRes = createResponse(); const secondRes = createResponse(); const req = { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', resultsPerPage: '1', pageNumber: '1' @@ -1272,7 +1257,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809', resultsPerPage: '1', pageNumber: '1' @@ -1296,7 +1281,7 @@ describe('Search controller SEARCH handler', () => { failureType: 'partial_shards' }); const req = { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809', resultsPerPage: '1', pageNumber: '1' @@ -1324,7 +1309,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { requestId: 'request-123', - query: { + searchParams: { keywordSearch: 'remote code', resultsPerPage: '1' } @@ -1343,7 +1328,6 @@ describe('Search controller SEARCH handler', () => { statusCode: 200, strategy: 'standard', cacheStatus: 'not_applicable', - optimizedFallback: false, cacheHit: false, inFlightHit: false, openSearchOperations: ['standard'], @@ -1361,7 +1345,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { cvssV3_0Severity: 'high', cvssV3_1Severity: 'critical' } @@ -1389,7 +1373,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { cvssV2Metrics: 'AV:N/AC:L/Au:N' } }, @@ -1407,7 +1391,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { cvssV3_0Metrics: 'CVSS:3.0/AV:N/AC:L', cvssV3_1Metrics: 'CVSS:3.1/PR:N/UI:R' } @@ -1427,7 +1411,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { keywordSearch: 'c++ path:/tmp (test)' } }, @@ -1451,7 +1435,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { keywordSearch: ' ' } }, @@ -1475,7 +1459,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:a:vendor:product\\:edition:*:*:*:*:*:*:*:*' } }, @@ -1515,7 +1499,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { virtualMatchString: 'cpe:2.3:*:*:*:*:*:*:*:*:*:*:*' } }, @@ -1542,7 +1526,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: {} + searchParams: {} }, res, next @@ -1563,7 +1547,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { resultsPerPage: 10, pageNumber: 2 } @@ -1587,7 +1571,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { - query: { + searchParams: { resultsPerPage: 30, pageNumber: 333 } @@ -1616,7 +1600,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { requestId: 'connection-error-request', - query: {} + searchParams: {} }, res, next @@ -1651,7 +1635,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { requestId: 'throttled-request', - query: {} + searchParams: {} }, res, next @@ -1685,7 +1669,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { requestId: 'response-timeout-request', - query: {} + searchParams: {} }, res, next @@ -1717,7 +1701,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { requestId: `${failureType}-request`, - query: {} + searchParams: {} }, res, next @@ -1747,7 +1731,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { requestId: 'timeout-request', - query: {} + searchParams: {} }, res, next @@ -1779,7 +1763,7 @@ describe('Search controller SEARCH handler', () => { const search = controller.SEARCH( { requestId: 'request-deadline-request', - query: {} + searchParams: {} }, res, next @@ -1808,7 +1792,7 @@ describe('Search controller SEARCH handler', () => { test('SEARCH stops without writing a response when the client disconnects', async () => { const req = new EventEmitter(); req.requestId = 'client-disconnect-request'; - req.query = {}; + req.searchParams = {}; mockApiSearch.mockImplementation( (_body, { signal }) => new Promise((resolve, reject) => { @@ -1836,7 +1820,7 @@ describe('Search controller SEARCH handler', () => { test('SEARCH does not start a page fetch after a client disconnects from a CPE search', async () => { const req = new EventEmitter(); req.requestId = 'cpe-client-disconnect-request'; - req.query = { + req.searchParams = { virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', resultsPerPage: 1, pageNumber: 1 @@ -1871,7 +1855,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { requestId: 'upstream-error-request', - query: {} + searchParams: {} }, res, next diff --git a/jest.config.js b/jest.config.js index dd834cb..36247ba 100644 --- a/jest.config.js +++ b/jest.config.js @@ -2,13 +2,7 @@ const config = { clearMocks: true, collectCoverage: false, - collectCoverageFrom: [ - 'controllers/**/*.js', - 'routes/index.js', - 'server.js', - 'utils/**/*.js', - '!controllers/wrapper.js' - ], + collectCoverageFrom: ['controllers/**/*.js', 'routes/index.js', 'server.js', 'utils/**/*.js'], coverageDirectory: 'coverage', coverageProvider: 'v8', coverageReporters: ['json', 'text', 'lcov'], diff --git a/package-lock.json b/package-lock.json index 61f936e..5a06297 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,12 +10,10 @@ "dependencies": { "cors": "^2.8.5", "cve-core": "github:CVEProject/cve-core.git#dev-package-test", - "debug": "~3.2.7", "dotenv": "^6.1.0", "express": "^4.22.2", "express-rate-limit": "^8.6.1", "express-validator": "^7.2.1", - "http-errors": "~1.8.1", "morgan": "~1.11.0", "swagger-ui-express": "^5.0.1" }, @@ -2323,15 +2321,6 @@ "date-fns": "2.x" } }, - "node_modules/debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.1" - } - }, "node_modules/dedent": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz", @@ -3467,40 +3456,6 @@ "dev": true, "license": "MIT" }, - "node_modules/http-errors": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", - "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", - "license": "MIT", - "dependencies": { - "depd": "~1.1.2", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/http-errors/node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/http-errors/node_modules/statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", diff --git a/package.json b/package.json index 346a0c2..1b752e5 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,10 @@ "dependencies": { "cors": "^2.8.5", "cve-core": "github:CVEProject/cve-core.git#dev-package-test", - "debug": "~3.2.7", "dotenv": "^6.1.0", "express": "^4.22.2", "express-rate-limit": "^8.6.1", "express-validator": "^7.2.1", - "http-errors": "~1.8.1", "morgan": "~1.11.0", "swagger-ui-express": "^5.0.1" }, diff --git a/server.js b/server.js index 1a34142..a388c54 100644 --- a/server.js +++ b/server.js @@ -5,7 +5,6 @@ const swaggerUi = require('swagger-ui-express'); const openApiSpecification = require('./api-docs/openapi.json'); const http = require('http'); const logger = require('morgan'); -const path = require('path'); const router = require('./routes/index'); const errors = require('./utils/errors'); const requestLogging = require('./utils/requestLogging'); @@ -98,7 +97,6 @@ function createApp() { app.use(cors()); app.use(runtimeLifecycle.trackRequest); app.use(rejectRequestsDuringShutdown); - app.use(express.static(path.join(__dirname, 'public'))); app.use(express.json()); app.get('/api-docs/openapi.json', (req, res) => res.json(openApiSpecification)); diff --git a/utils/constants.js b/utils/constants.js index 6233699..6b3c979 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -92,20 +92,6 @@ const fields = { virtualMatchString: cpeMatchFields.criteria }; -const searchControlParams = [ - 'keywordExactMatch', - 'resultsPerPage', - 'pageNumber', - 'pubStartDate', - 'pubEndDate', - 'versionStart', - 'versionStartType', - 'versionEnd', - 'versionEndType', - 'lastModStartDate', - 'lastModEndDate' -]; - const searchQueryParams = [ 'keywordSearch', 'keywordExactMatch', @@ -164,6 +150,5 @@ module.exports = { CPE_MATCH_FIELDS: cpeMatchFields, CVSS_VECTOR_VERSION_FIELDS: cvssVectorVersionFields, FIELDS: fields, - SEARCH_CONTROL_PARAMS: searchControlParams, SEARCH_QUERY_PARAMS: searchQueryParams }; diff --git a/utils/searchPerformance.js b/utils/searchPerformance.js index 19c690f..7dfa764 100644 --- a/utils/searchPerformance.js +++ b/utils/searchPerformance.js @@ -45,7 +45,6 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte if (includeDetailedTimings) { Object.assign(event, { - optimizedFallback: metrics.optimizedFallback === true, cacheHit: metrics.cacheHit === true, inFlightHit: metrics.inFlightHit === true, pageFetchInFlightHit: metrics.pageFetchInFlightHit === true, diff --git a/utils/searchPerformance.test.js b/utils/searchPerformance.test.js index b4d9a75..4d93959 100644 --- a/utils/searchPerformance.test.js +++ b/utils/searchPerformance.test.js @@ -72,7 +72,6 @@ describe('searchPerformance', () => { statusCode: 200, strategy: 'cpe-candidate', cacheStatus: 'hit', - optimizedFallback: false, cacheHit: true, inFlightHit: true, pageFetchInFlightHit: true, From b5f5901575d181f6df94d4c86f2e24260dd9ee28 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 15:30:52 -0400 Subject: [PATCH 08/33] Fail closed on malformed search candidates --- controllers/searchController.js | 35 +++++-- controllers/searchController.test.js | 56 +++++++++++- routes/search.integration.test.js | 38 ++++++++ utils/searchProvider.js | 78 +++++++++++++++- utils/searchProvider.test.js | 131 ++++++++++++++++++++++++++- 5 files changed, 319 insertions(+), 19 deletions(-) diff --git a/controllers/searchController.js b/controllers/searchController.js index e954f28..249ca2b 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -11,6 +11,7 @@ const searchQueryBuilder = require('../utils/searchQueryBuilder'); const constants = require('../utils/constants'); const error = new errors.errors(); const openSearchUnavailableErrorNames = new Set(['ConnectionError', 'NoLivingConnectionsError']); +const openSearchUpstreamErrorNames = new Set(['ResponseError', 'SearchError']); const openSearchUnavailableStatusCodes = new Set([502, 503, 504]); function hasSearchPostFilters(postFilters) { @@ -156,11 +157,25 @@ function recordUpstreamFailure(metrics, err, failureType) { function isOpenSearchUnavailableError(err) { return ( searchProvider.isSearchProviderContractError?.(err) === true || + searchProvider.isSearchProviderConfigurationError?.(err) === true || openSearchUnavailableErrorNames.has(err?.name) || openSearchUnavailableStatusCodes.has(getOpenSearchErrorStatusCode(err)) ); } +function getOpenSearchUnavailableFailureType(err) { + if (searchProvider.isSearchProviderContractError?.(err)) return 'provider_contract'; + if (searchProvider.isSearchProviderConfigurationError?.(err)) return 'provider_configuration'; + + return 'unavailable'; +} + +function isOpenSearchUpstreamError(err) { + const statusCode = getOpenSearchErrorStatusCode(err); + + return openSearchUpstreamErrorNames.has(err?.name) || (Number.isInteger(statusCode) && statusCode >= 400); +} + function buildWebSearchSort(sort) { if (!sort) return undefined; @@ -500,11 +515,7 @@ async function searchHandler(req, res, next) { } if (isOpenSearchUnavailableError(err)) { - recordUpstreamFailure( - metrics, - err, - searchProvider.isSearchProviderContractError?.(err) ? 'provider_contract' : 'unavailable' - ); + recordUpstreamFailure(metrics, err, getOpenSearchUnavailableFailureType(err)); metrics.outcome = 'opensearch_unavailable'; metrics.statusCode = 503; if (typeof res.set === 'function') res.set('Retry-After', '5'); @@ -512,10 +523,16 @@ async function searchHandler(req, res, next) { return res.status(503).json(error.openSearchError()); } - recordUpstreamFailure(metrics, err, 'upstream_error'); - metrics.outcome = 'upstream_error'; - metrics.statusCode = 502; - return res.status(502).json(error.upstreamSearchError()); + if (isOpenSearchUpstreamError(err)) { + recordUpstreamFailure(metrics, err, 'upstream_error'); + metrics.outcome = 'upstream_error'; + metrics.statusCode = 502; + return res.status(502).json(error.upstreamSearchError()); + } + + metrics.outcome = 'internal_error'; + metrics.statusCode = 500; + return next(err); } finally { lifecycle.complete(); metrics.totalElapsedMs = searchPerformance.elapsedMs(handlerStart); diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 245415d..5fdaf3c 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -25,6 +25,7 @@ const mockGetInFlightMaxWaitersPerOperation = jest.fn(() => 500); const mockGetPostFilterEvaluationChunkSize = jest.fn(() => 250); const mockGetSearchRequestDeadlineMs = jest.fn(() => 30000); const mockIsOpenSearchResponseError = jest.fn((err) => err?.code === 'OPEN_SEARCH_RESPONSE_INVALID'); +const mockIsSearchProviderConfigurationError = jest.fn((err) => err?.code === 'SEARCH_PROVIDER_CONFIGURATION'); const mockIsSearchProviderContractError = jest.fn((err) => err?.code === 'SEARCH_PROVIDER_CONTRACT'); const mockGetBasicSearchManager = jest.fn(() => ({ search: mockBasicSearch @@ -49,6 +50,7 @@ jest.mock('../utils/searchProvider', () => ({ getPostFilterEvaluationChunkSize: mockGetPostFilterEvaluationChunkSize, getSearchRequestDeadlineMs: mockGetSearchRequestDeadlineMs, isOpenSearchResponseError: mockIsOpenSearchResponseError, + isSearchProviderConfigurationError: mockIsSearchProviderConfigurationError, isSearchProviderContractError: mockIsSearchProviderContractError, searchOpenSearch: mockSearchOpenSearch, shouldLogSearchPerformance: mockShouldLogSearchPerformance @@ -149,12 +151,20 @@ function cpeHit(id, cpeMatch) { return { _id: id, _source: { + cveMetadata: { + cveId: id + }, containers: { cna: { cpeApplicability: { nodes: [ { - cpeMatch: [cpeMatch] + cpeMatch: [ + { + vulnerable: true, + ...cpeMatch + } + ] } ] } @@ -184,6 +194,7 @@ describe('Search controller SEARCH handler', () => { mockGetPostFilterEvaluationChunkSize.mockClear(); mockGetSearchRequestDeadlineMs.mockClear(); mockIsOpenSearchResponseError.mockClear(); + mockIsSearchProviderConfigurationError.mockClear(); mockIsSearchProviderContractError.mockClear(); mockGetPostFilterCacheTtlMs.mockReturnValue(60000); mockGetPostFilterCacheMaxEntries.mockReturnValue(100); @@ -1253,7 +1264,13 @@ describe('Search controller SEARCH handler', () => { }); test('SEARCH returns sanitized upstream errors when optimized post-filter search fails', async () => { - mockSearchRaw.mockRejectedValue(new Error('candidate search failed')); + mockSearchRaw.mockRejectedValue({ + name: 'ResponseError', + message: 'candidate search failed', + meta: { + statusCode: 500 + } + }); await controller.SEARCH( { @@ -1273,12 +1290,16 @@ describe('Search controller SEARCH handler', () => { error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); + expect(next).not.toHaveBeenCalled(); }); - test('SEARCH does not cache a partial OpenSearch candidate response', async () => { + test.each([ + ['partial', 'partial_shards'], + ['malformed', 'malformed_candidate'] + ])('SEARCH does not cache a %s OpenSearch candidate response', async (_caseName, failureType) => { mockSearchRaw.mockRejectedValue({ code: 'OPEN_SEARCH_RESPONSE_INVALID', - failureType: 'partial_shards' + failureType }); const req = { searchParams: { @@ -1592,6 +1613,7 @@ describe('Search controller SEARCH handler', () => { test.each([ ['ConnectionError', { name: 'ConnectionError' }], ['NoLivingConnectionsError', { name: 'NoLivingConnectionsError' }], + ['search provider configuration error', { code: 'SEARCH_PROVIDER_CONFIGURATION' }], ['cve-core provider contract error', { code: 'SEARCH_PROVIDER_CONTRACT' }], ['OpenSearch 503 response', { name: 'ResponseError', meta: { statusCode: 503 } }] ])('SEARCH returns a retryable 503 when OpenSearch is unavailable through %s', async (_caseName, thrownError) => { @@ -1872,6 +1894,32 @@ describe('Search controller SEARCH handler', () => { outcome: 'upstream_error', statusCode: 502 }); + expect(next).not.toHaveBeenCalled(); + }); + + test('SEARCH delegates unexpected application errors to the sanitized Express error handler', async () => { + const err = new TypeError('sensitive internal implementation details'); + mockApiSearch.mockRejectedValue(err); + + await controller.SEARCH( + { + requestId: 'internal-error-request', + searchParams: {} + }, + res, + next + ); + + expect(next).toHaveBeenCalledWith(err); + expect(res.status).not.toHaveBeenCalled(); + expect(res.json).not.toHaveBeenCalled(); + expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ + event: 'search.performance', + requestId: 'internal-error-request', + outcome: 'internal_error', + statusCode: 500 + }); + expect(console.log.mock.calls[0][0]).not.toContain('sensitive'); }); }); diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index d4ff51a..9aab62a 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -53,6 +53,10 @@ describe('search route integration', () => { mockBasicSearch.mockReset(); }); + afterEach(() => { + jest.restoreAllMocks(); + }); + test('POST /search validates, builds the OpenSearch request, and formats pagination metadata', async () => { mockSearchOpenSearch.mockResolvedValue(advancedResponse(1, [{ _id: 'CVE-2024-0001' }]).data); @@ -111,6 +115,40 @@ describe('search route integration', () => { expect(response.body.status).toBe('ok'); }); + test('POST /search returns a sanitized 500 for unexpected application errors', async () => { + const internalError = new TypeError('sensitive internal implementation details'); + const performanceLog = jest.spyOn(console, 'log').mockImplementation(() => {}); + const requestErrorLog = jest.spyOn(console, 'error').mockImplementation(() => {}); + mockSearchOpenSearch.mockRejectedValue(internalError); + + const response = await request(createApp(), { + method: 'POST', + path: '/search', + body: { + keywordSearch: 'remote code' + } + }); + + expect(response.status).toBe(500); + expect(response.body).toStrictEqual({ + error: 'INTERNAL_SERVER_ERROR', + message: 'An unexpected server error occurred.' + }); + expect(JSON.parse(performanceLog.mock.calls[0][0])).toMatchObject({ + event: 'search.performance', + outcome: 'internal_error', + statusCode: 500 + }); + expect(JSON.parse(requestErrorLog.mock.calls[0][0])).toMatchObject({ + event: 'request.error', + outcome: 'internal_error', + route: '/search', + statusCode: 500 + }); + expect(performanceLog.mock.calls[0][0]).not.toContain('sensitive'); + expect(requestErrorLog.mock.calls[0][0]).not.toContain('sensitive'); + }); + test('POST /search rejects invalid dates before invoking OpenSearch', async () => { const response = await request(createApp(), { method: 'POST', diff --git a/utils/searchProvider.js b/utils/searchProvider.js index e9244f1..4d8b9eb 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -1,5 +1,6 @@ const { AppConfig, BasicSearchManager, SearchProviderSpec, SearchReader } = require('cve-core'); const constants = require('./constants'); +const cpe = require('./cpe'); const searchDeadline = require('./searchDeadline'); class SearchProviderContractError extends Error { @@ -10,11 +11,28 @@ class SearchProviderContractError extends Error { } } +class SearchProviderConfigurationError extends Error { + constructor() { + super('OpenSearch providerEndpoint and index must be configured.'); + this.name = 'SearchProviderConfigurationError'; + this.code = 'SEARCH_PROVIDER_CONFIGURATION'; + } +} + const OPEN_SEARCH_RESPONSE_FAILURE_TYPES = Object.freeze({ + MALFORMED_CANDIDATE: 'malformed_candidate', MALFORMED_RESPONSE: 'malformed_response', PARTIAL_SHARDS: 'partial_shards', TIMEOUT: 'timeout' }); +const CPE_CANDIDATE_OPERATION = 'cpe-candidate'; +const CVE_ID_PATTERN = /^CVE-\d{4}-\d{4,19}$/; +const CPE_RANGE_FIELDS = [ + 'versionStartIncluding', + 'versionStartExcluding', + 'versionEndIncluding', + 'versionEndExcluding' +]; class OpenSearchResponseError extends Error { constructor(failureType) { @@ -79,7 +97,7 @@ function normalizeSearchProviderConfig(spec) { const index = spec?.index; if (!providerEndpoint || !index) { - throw new Error('OpenSearch providerEndpoint and index must be configured.'); + throw new SearchProviderConfigurationError(); } return Object.freeze({ @@ -123,7 +141,51 @@ function throwOpenSearchResponseError(failureType) { throw new OpenSearchResponseError(failureType); } -function validateOpenSearchResponse(body) { +function isObject(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function isValidCpeMatch(match) { + if (!isObject(match) || typeof match.criteria !== 'string' || match.criteria.trim() === '') return false; + if (!cpe.parseCpeMatchCriteria(match.criteria).valid) return false; + if (typeof match.vulnerable !== 'boolean') return false; + + return CPE_RANGE_FIELDS.every( + (field) => match[field] === undefined || (typeof match[field] === 'string' && match[field].trim() !== '') + ); +} + +function isValidCpeCandidateHit(hit) { + if (!isObject(hit) || !isObject(hit._source)) return false; + + const cveId = hit._source.cveMetadata?.cveId; + if (typeof cveId !== 'string' || !CVE_ID_PATTERN.test(cveId)) return false; + + const cpeApplicability = hit._source.containers?.cna?.cpeApplicability; + const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; + if (!applicabilityItems.length || applicabilityItems.some((item) => !isObject(item))) return false; + + let matchCount = 0; + for (const applicability of applicabilityItems) { + if (!Array.isArray(applicability.nodes)) return false; + + for (const node of applicability.nodes) { + if (!isObject(node) || !Array.isArray(node.cpeMatch)) return false; + if (!node.cpeMatch.every(isValidCpeMatch)) return false; + matchCount += node.cpeMatch.length; + } + } + + return matchCount > 0; +} + +function validateCpeCandidateHits(hits) { + if (hits.every(isValidCpeCandidateHit)) return; + + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_CANDIDATE); +} + +function validateOpenSearchResponse(body, { operation } = {}) { if (!body || typeof body !== 'object' || Array.isArray(body)) { throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); } @@ -152,6 +214,10 @@ function validateOpenSearchResponse(body) { throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); } + if (operation === CPE_CANDIDATE_OPERATION) { + validateCpeCandidateHits(body.hits.hits); + } + return body; } @@ -290,7 +356,7 @@ async function searchOpenSearch(body, { signal, operation = 'search', onStart, r } ); - return validateOpenSearchResponse(response.body); + return validateOpenSearchResponse(response.body, { operation }); } async function checkOpenSearchReadiness({ signal } = {}) { @@ -330,6 +396,10 @@ function isSearchProviderContractError(err) { return err instanceof SearchProviderContractError || err?.code === 'SEARCH_PROVIDER_CONTRACT'; } +function isSearchProviderConfigurationError(err) { + return err instanceof SearchProviderConfigurationError || err?.code === 'SEARCH_PROVIDER_CONFIGURATION'; +} + function isOpenSearchResponseError(err) { return err instanceof OpenSearchResponseError || err?.code === 'OPEN_SEARCH_RESPONSE_INVALID'; } @@ -432,6 +502,7 @@ module.exports = { getSearchProviderConfig, getSearchReader, isOpenSearchResponseError, + isSearchProviderConfigurationError, isSearchProviderContractError, normalizeBoolean, resetSearchProviderForTests, @@ -441,5 +512,6 @@ module.exports = { validateSearchReader, OPEN_SEARCH_RESPONSE_FAILURE_TYPES, OpenSearchResponseError, + SearchProviderConfigurationError, SearchProviderContractError }; diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index 159aac2..8abc85d 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -54,6 +54,36 @@ function deferred() { return state; } +function cpeCandidateHit(cpeMatch = {}) { + return { + _id: 'CVE-2024-0001', + _source: { + cveMetadata: { + cveId: 'CVE-2024-0001' + }, + containers: { + cna: { + cpeApplicability: [ + { + nodes: [ + { + cpeMatch: [ + { + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true, + ...cpeMatch + } + ] + } + ] + } + ] + } + } + } + }; +} + describe('searchProvider', () => { beforeEach(() => { searchProvider.resetSearchProviderForTests(); @@ -297,6 +327,103 @@ describe('searchProvider', () => { }); }); + test('accepts schema-shaped CPE candidate hits for candidate operations', async () => { + const body = { + hits: { + hits: [cpeCandidateHit()] + } + }; + mockClientSearch.mockResolvedValue({ body }); + + await expect( + searchProvider.searchOpenSearch( + { + query: { + match_all: {} + } + }, + { + operation: 'cpe-candidate' + } + ) + ).resolves.toBe(body); + }); + + test('accepts an empty CPE candidate result', async () => { + const body = { + hits: { + hits: [] + } + }; + mockClientSearch.mockResolvedValue({ body }); + + await expect( + searchProvider.searchOpenSearch( + { + query: { + match_all: {} + } + }, + { + operation: 'cpe-candidate' + } + ) + ).resolves.toBe(body); + }); + + test.each([ + [ + 'missing projected CVE ID', + { + ...cpeCandidateHit(), + _source: { + ...cpeCandidateHit()._source, + cveMetadata: {} + } + } + ], + [ + 'missing projected CPE applicability', + { + ...cpeCandidateHit(), + _source: { + ...cpeCandidateHit()._source, + containers: { + cna: {} + } + } + } + ], + ['missing CPE criteria', cpeCandidateHit({ criteria: undefined })], + ['invalid CPE criteria', cpeCandidateHit({ criteria: 'not-a-cpe-name' })], + ['missing CPE vulnerability flag', cpeCandidateHit({ vulnerable: undefined })], + ['invalid CPE range boundary', cpeCandidateHit({ versionEndExcluding: 2 })] + ])('rejects CPE candidate hits with %s', async (_caseName, hit) => { + mockClientSearch.mockResolvedValue({ + body: { + hits: { + hits: [hit] + } + } + }); + + await expect( + searchProvider.searchOpenSearch( + { + query: { + match_all: {} + } + }, + { + operation: 'cpe-candidate' + } + ) + ).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_candidate' + }); + }); + test('checks OpenSearch readiness through the configured CVE index', async () => { mockClientSearch.mockResolvedValue({ body: { @@ -513,9 +640,7 @@ describe('searchProvider', () => { index: 'test-index' }); - expect(() => searchProvider.getSearchProviderConfig()).toThrow( - 'OpenSearch providerEndpoint and index must be configured.' - ); + expect(() => searchProvider.getSearchProviderConfig()).toThrow(searchProvider.SearchProviderConfigurationError); }); test('normalizes string FALSE from AppConfig before constructing managers', () => { From fa2eabde4fa6a6ff49b097d7c54ea9c005d6ab0a Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 15:38:45 -0400 Subject: [PATCH 09/33] Handle oversized JSON requests explicitly --- README.md | 2 +- api-docs/openapi.json | 11 +++++++++++ api-docs/openapi.test.js | 2 ++ routes/index.js | 11 +++++++++++ server.js | 7 ++++++- server.test.js | 23 +++++++++++++++++++++++ utils/constants.js | 4 ++++ utils/errors.js | 7 +++++++ utils/errors.test.js | 8 ++++++++ 9 files changed, 73 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cf7fa7e..9993afd 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Shutdown lifecycle events are emitted as sanitized `runtime.lifecycle` JSON logs ## API Input Validation -`POST /search` accepts an optional JSON object request body and validates supported fields, boolean parameters, pagination values, CVSS severity values, required parameter combinations, CPE Name requirements, and date-time formats. A non-empty body must use the `application/json` media type with UTF-8 encoding; unsupported media types or encodings return HTTP 415, and malformed JSON returns HTTP 400. If OpenSearch is unavailable, the endpoint returns HTTP 503 with a `Retry-After` header. +`POST /search` accepts an optional JSON object request body and validates supported fields, boolean parameters, pagination values, CVSS severity values, required parameter combinations, CPE Name requirements, and date-time formats. The JSON request body cannot exceed 100 KB; larger bodies return HTTP 413 before routing or OpenSearch access. A non-empty body must use the `application/json` media type with UTF-8 encoding; unsupported media types or encodings return HTTP 415, and malformed JSON returns HTTP 400. If OpenSearch is unavailable, the endpoint returns HTTP 503 with a `Retry-After` header. String inputs are bounded before query construction: diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 3d69e9e..1b67784 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -69,6 +69,16 @@ } } }, + "413": { + "description": "The JSON request body exceeded the 100 KB limit", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, "415": { "description": "A non-empty request body did not use the application/json media type with UTF-8 encoding", "content": { @@ -160,6 +170,7 @@ }, "requestBody": { "required": false, + "description": "Optional JSON filters and pagination controls. The JSON request body cannot exceed 100 KB.", "content": { "application/json": { "schema": { diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 3bee554..f7c9d24 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -63,7 +63,9 @@ describe('OpenAPI contract', () => { const mediaType = operation.requestBody.content['application/json']; expect(operation.parameters || []).toStrictEqual([]); + expect(operation.requestBody.description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`); expect(mediaType.schema).toStrictEqual({ $ref: '#/components/schemas/SearchRequest' }); + expect(operation.responses['413'].description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`); expect(schema.additionalProperties).toBe(false); expect(Object.keys(schema.properties)).toStrictEqual(constants.SEARCH_QUERY_PARAMS); Object.values(schema.properties).forEach((property) => { diff --git a/routes/index.js b/routes/index.js index 7a2ff47..ff71317 100644 --- a/routes/index.js +++ b/routes/index.js @@ -172,6 +172,7 @@ router.post(

Retrieves all matching CVE records after applying the JSON request body as filters

#swagger.requestBody = { required: false, + description: "Optional JSON filters and pagination controls. The JSON request body cannot exceed 100 KB.", content: { "application/json": { schema: { @@ -295,6 +296,16 @@ content: { } } } +#swagger.responses[413] = { +description: 'The JSON request body exceeded the 100 KB limit', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} #swagger.responses[429] = { description: 'Application rate limit exceeded, the cold CPE search queue is full or exceeded its maximum wait, or a shared semantic search operation reached its waiter limit', headers: { diff --git a/server.js b/server.js index a388c54..1340374 100644 --- a/server.js +++ b/server.js @@ -6,6 +6,7 @@ const openApiSpecification = require('./api-docs/openapi.json'); const http = require('http'); const logger = require('morgan'); const router = require('./routes/index'); +const constants = require('./utils/constants'); const errors = require('./utils/errors'); const requestLogging = require('./utils/requestLogging'); const runtimeLifecycle = require('./utils/runtimeLifecycle'); @@ -41,6 +42,10 @@ function createRequestLogger() { } function handleRequestError(err, req, res, next) { + if (err?.type === 'entity.too.large' && err?.status === 413) { + return res.status(413).json(error.requestBodyTooLargeError()); + } + if (err?.type === 'entity.parse.failed') { return res.status(400).json(error.malformedJsonError()); } @@ -97,7 +102,7 @@ function createApp() { app.use(cors()); app.use(runtimeLifecycle.trackRequest); app.use(rejectRequestsDuringShutdown); - app.use(express.json()); + app.use(express.json({ limit: constants.JSON_REQUEST_BODY_LIMIT_BYTES })); app.get('/api-docs/openapi.json', (req, res) => res.json(openApiSpecification)); app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(openApiSpecification, swaggerSetupOptions)); diff --git a/server.test.js b/server.test.js index bb829df..61248a8 100644 --- a/server.test.js +++ b/server.test.js @@ -1,6 +1,7 @@ const http = require('http'); const { EventEmitter } = require('events'); const { request } = require('./test/http'); +const constants = require('./utils/constants'); const mockValidateParams = jest.fn((req, res, next) => next()); const mockSearch = jest.fn((req, res) => @@ -199,6 +200,28 @@ describe('server application bootstrap', () => { expect(mockSearch).not.toHaveBeenCalled(); }); + test('createApp returns a sanitized error for oversized JSON before routing', async () => { + const oversizedValue = 'x'.repeat(constants.JSON_REQUEST_BODY_LIMIT_BYTES); + const response = await request(createApp(), { + method: 'POST', + path: '/search', + body: { + keywordSearch: oversizedValue + } + }); + + expect(response.status).toBe(413); + expect(response.body).toStrictEqual({ + error: 'REQUEST_BODY_TOO_LARGE', + message: `The JSON request body cannot exceed ${constants.JSON_REQUEST_BODY_LIMIT_KB} KB.` + }); + expect(response.text).not.toContain(oversizedValue); + expect(response.text).not.toContain('entity.too.large'); + expect(response.headers['access-control-allow-origin']).toBe('*'); + expect(mockValidateParams).not.toHaveBeenCalled(); + expect(mockSearch).not.toHaveBeenCalled(); + }); + test('createApp returns a sanitized error for unsupported JSON encoding', async () => { const response = await request(createApp(), { method: 'POST', diff --git a/utils/constants.js b/utils/constants.js index 6b3c979..0093369 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -7,6 +7,8 @@ const MAX_CVSS_METRIC_TOKEN_LENGTH = 32; const MAX_CVE_IDS = 100; const MAX_CVE_ID_LENGTH = 64; const MAX_KEYWORD_SEARCH_TOKENS = 64; +const JSON_REQUEST_BODY_LIMIT_KB = 100; +const JSON_REQUEST_BODY_LIMIT_BYTES = JSON_REQUEST_BODY_LIMIT_KB * 1024; const SEARCH_STRING_LENGTH_LIMITS = Object.freeze({ keywordSearch: 512, cveIds: 4096, @@ -132,6 +134,8 @@ module.exports = { MAX_CVE_IDS, MAX_CVE_ID_LENGTH, MAX_KEYWORD_SEARCH_TOKENS, + JSON_REQUEST_BODY_LIMIT_KB, + JSON_REQUEST_BODY_LIMIT_BYTES, SEARCH_STRING_LENGTH_LIMITS, SEARCH_POST_FILTER_CACHE_TTL_MS, SEARCH_POST_FILTER_CACHE_MAX_ENTRIES, diff --git a/utils/errors.js b/utils/errors.js index 6b46440..d001e55 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -139,6 +139,13 @@ class errors { return makeError('MALFORMED_JSON', 'The request body contains malformed JSON.'); } + requestBodyTooLargeError() { + return makeError( + 'REQUEST_BODY_TOO_LARGE', + `The JSON request body cannot exceed ${constants.JSON_REQUEST_BODY_LIMIT_KB} KB.` + ); + } + searchQueryUnsupportedError() { return makeError( 'SEARCH_QUERY_UNSUPPORTED', diff --git a/utils/errors.test.js b/utils/errors.test.js index 3d09f08..b40efc8 100644 --- a/utils/errors.test.js +++ b/utils/errors.test.js @@ -263,6 +263,14 @@ describe('error response helpers', () => { message: 'The request body contains malformed JSON.' } ], + [ + 'requestBodyTooLargeError', + [], + { + error: 'REQUEST_BODY_TOO_LARGE', + message: `The JSON request body cannot exceed ${constants.JSON_REQUEST_BODY_LIMIT_KB} KB.` + } + ], [ 'searchQueryUnsupportedError', [], From c0c077b8c7300af037734ac9b2195dfd95fc3b27 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 15:43:03 -0400 Subject: [PATCH 10/33] Prevent readiness success during shutdown --- README.md | 2 +- api-docs/openapi.json | 2 +- api-docs/openapi.test.js | 3 +++ routes/index.js | 17 ++++++++++++----- routes/index.test.js | 35 +++++++++++++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9993afd..fb9c40d 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ The Search API will be served at `localhost:3000`. `GET /api/health-check` is a process liveness check. It returns HTTP 200 when the Node.js process is running and deliberately does not contact OpenSearch. Use it where a dependency outage should not restart otherwise healthy API tasks. -`GET /api/ready-check` sends a minimal zero-result search to the configured CVE index. This verifies connectivity, index availability, and the same search permission used by the API without retrieving CVE records or counting all matches. It returns HTTP 200 when search requests can be served. Configuration errors, connection failures, timeouts, incompatible `cve-core` SearchReader contracts, a missing index, and an in-progress process shutdown return a sanitized HTTP 503 `SEARCH_NOT_READY` response with `Retry-After: 5`. The OpenSearch request is limited to five seconds by default and can be changed with `SearchReadinessTimeoutMs`. Once shutdown starts, readiness fails immediately without contacting OpenSearch. +`GET /api/ready-check` sends a minimal zero-result search to the configured CVE index. This verifies connectivity, index availability, and the same search permission used by the API without retrieving CVE records or counting all matches. It returns HTTP 200 only when search requests can still be served after the dependency check completes. Configuration errors, connection failures, timeouts, incompatible `cve-core` SearchReader contracts, a missing index, and an in-progress process shutdown return a sanitized HTTP 503 `SEARCH_NOT_READY` response with `Retry-After: 5`. The OpenSearch request is limited to five seconds by default and can be changed with `SearchReadinessTimeoutMs`. If shutdown is already active, readiness fails immediately without contacting OpenSearch. If shutdown begins while the dependency check is pending, the completed check is not reported as ready. An AWS load balancer can use `/api/ready-check` to stop routing search traffic to an unready task while the container orchestrator continues using `/api/health-check` for process liveness. Coordinate that probe selection with the infrastructure team before changing deployed health-check behavior. diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 1b67784..6b1f60e 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -264,7 +264,7 @@ "/api/ready-check": { "get": { "summary": "OpenSearch readiness check", - "description": "Sends a minimal zero-result search to the configured CVE index. Returns 200 when the index is available with search permission, or a sanitized 503 response when configuration, connectivity, timeout, permission, or index availability checks fail.", + "description": "Sends a minimal zero-result search to the configured CVE index. Returns 200 only when the index is available with search permission and the process is still accepting traffic after the dependency check. Returns a sanitized 503 response when shutdown is already active, shutdown begins while the dependency check is pending, or configuration, connectivity, timeout, permission, or index availability checks fail.", "operationId": "readinessCheck", "responses": { "200": { diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index f7c9d24..4f7e846 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -29,6 +29,9 @@ describe('OpenAPI contract', () => { expect(openapi.paths['/search'].post.operationId).toBe('search'); expect(openapi.paths['/api/health-check'].get.operationId).toBe('healthCheck'); expect(openapi.paths['/api/ready-check'].get.operationId).toBe('readinessCheck'); + expect(openapi.paths['/api/ready-check'].get.description).toContain( + 'shutdown begins while the dependency check is pending' + ); expect(openapi.paths['/webSearch'].post.operationId).toBe('webSearch'); }); diff --git a/routes/index.js b/routes/index.js index ff71317..2d30dbc 100644 --- a/routes/index.js +++ b/routes/index.js @@ -24,10 +24,14 @@ function healthCheck(req, res) { res.status(200).send(); } +function sendSearchNotReady(res) { + if (typeof res.set === 'function') res.set('Retry-After', '5'); + return res.status(503).json(error.searchNotReadyError()); +} + async function readinessCheck(req, res) { if (runtimeLifecycle.isShuttingDown()) { - if (typeof res.set === 'function') res.set('Retry-After', '5'); - return res.status(503).json(error.searchNotReadyError()); + return sendSearchNotReady(res); } const lifecycle = searchDeadline.createSearchRequestLifecycle( @@ -42,12 +46,15 @@ async function readinessCheck(req, res) { }); lifecycle.throwIfAborted(); + if (runtimeLifecycle.isShuttingDown()) { + return sendSearchNotReady(res); + } + return res.status(200).send(); } catch (err) { if (searchDeadline.isSearchClientDisconnectedError(err)) return; - if (typeof res.set === 'function') res.set('Retry-After', '5'); - return res.status(503).json(error.searchNotReadyError()); + return sendSearchNotReady(res); } finally { lifecycle.complete(); } @@ -446,7 +453,7 @@ router.get( /* #swagger.operationId = 'readinessCheck' #swagger.summary = "OpenSearch readiness check" -#swagger.description = "Sends a minimal zero-result search to the configured CVE index. Returns 200 when the index is available with search permission, or a sanitized 503 response when configuration, connectivity, timeout, permission, or index availability checks fail." +#swagger.description = "Sends a minimal zero-result search to the configured CVE index. Returns 200 only when the index is available with search permission and the process is still accepting traffic after the dependency check. Returns a sanitized 503 response when shutdown is already active, shutdown begins while the dependency check is pending, or configuration, connectivity, timeout, permission, or index availability checks fail." #swagger.responses[200] = { description: 'API is ready to serve search requests', headers: { diff --git a/routes/index.test.js b/routes/index.test.js index 9b7aff1..5aa1d56 100644 --- a/routes/index.test.js +++ b/routes/index.test.js @@ -41,6 +41,15 @@ const featureFlags = require('../utils/featureFlags'); const runtimeLifecycle = require('../utils/runtimeLifecycle'); const router = require('./index'); +function deferred() { + const state = {}; + state.promise = new Promise((resolve, reject) => { + state.resolve = resolve; + state.reject = reject; + }); + return state; +} + describe('routes/index router', () => { let app; @@ -107,6 +116,32 @@ describe('routes/index router', () => { expect(mockCheckOpenSearchReadiness).not.toHaveBeenCalled(); }); + test('GET /api/ready-check returns 503 when shutdown begins during the OpenSearch check', async () => { + const readinessStarted = deferred(); + const readinessResult = deferred(); + mockCheckOpenSearchReadiness.mockImplementation(() => { + readinessStarted.resolve(); + return readinessResult.promise; + }); + + const responsePromise = request(app, { + path: '/api/ready-check' + }); + await readinessStarted.promise; + runtimeLifecycle.beginShutdown(); + readinessResult.resolve(true); + + const response = await responsePromise; + + expect(response.status).toBe(503); + expect(response.headers['retry-after']).toBe('5'); + expect(response.body).toStrictEqual({ + error: 'SEARCH_NOT_READY', + message: 'Search service is not ready. Please try again later.' + }); + expect(mockCheckOpenSearchReadiness).toHaveBeenCalledTimes(1); + }); + test('POST /search reaches middleware and controller for valid JSON body fields', async () => { const response = await request(app, { method: 'POST', From 9bae61325c5431cb31d48efaf4f9e5037887a66d Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 15:51:20 -0400 Subject: [PATCH 11/33] Remove legacy search fallback paths --- controllers/middleware.js | 2 +- controllers/middleware.test.js | 1 + controllers/searchController.js | 12 +++------- controllers/searchController.test.js | 15 ++++++++++++ utils/searchQueryBuilder.js | 36 +++++++--------------------- utils/searchQueryBuilder.test.js | 11 ++++++++- 6 files changed, 39 insertions(+), 38 deletions(-) diff --git a/controllers/middleware.js b/controllers/middleware.js index cdbba5a..eea9080 100644 --- a/controllers/middleware.js +++ b/controllers/middleware.js @@ -160,7 +160,7 @@ function validateParams(req, res, next) { } } - if (hasParam(params, 'noRejected') && ![true, false, ''].includes(params.noRejected)) { + if (hasParam(params, 'noRejected') && ![true, false].includes(params.noRejected)) { return res.status(400).json(error.noRejectedValueError()); } diff --git a/controllers/middleware.test.js b/controllers/middleware.test.js index c793a8f..dd819e9 100644 --- a/controllers/middleware.test.js +++ b/controllers/middleware.test.js @@ -560,6 +560,7 @@ describe('middleware validation function', () => { test.each([ ['true string', 'true'], ['false string', 'false'], + ['empty string', ''], ['repeated query parameter', ['', '']] ])('ValidateParams returns an error when noRejected has a non-boolean %s value.', (_caseName, noRejected) => { validate({ diff --git a/controllers/searchController.js b/controllers/searchController.js index 249ca2b..e9d98bb 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -49,21 +49,15 @@ function hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext) { return true; } -function buildEmptyFilteredResult(hitsContainer) { - return { - hitsContainer, - candidateCount: 0, - filteredHits: [] - }; -} - function yieldToEventLoop() { return new Promise((resolve) => setImmediate(resolve)); } async function filterSearchHitsInChunks(resp, postFilters, signal, chunkSize) { const hitsContainer = getHitsContainer(resp); - if (!Array.isArray(hitsContainer?.hits)) return buildEmptyFilteredResult(hitsContainer); + if (!Array.isArray(hitsContainer?.hits)) { + throw new TypeError('Semantic search candidate response must contain a hits array.'); + } const effectiveChunkSize = Math.max(1, chunkSize ?? searchProvider.getPostFilterEvaluationChunkSize()); const cpeEvaluationContext = cpe.createEvaluationContext(); diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 5fdaf3c..6d10afb 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -615,6 +615,21 @@ describe('Search controller SEARCH handler', () => { }); }); + test('chunked filtering rejects malformed candidate hit collections', async () => { + await expect( + controller.filterSearchHitsInChunks( + { + data: { + hits: {} + } + }, + {}, + undefined, + 100 + ) + ).rejects.toThrow('Semantic search candidate response must contain a hits array.'); + }); + test('chunked filtering does not combine platform and version matches from different CPE entries', async () => { const candidate = cpeHit('CVE-2026-0250', { criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:macos:*:*:*:*:*', diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index f5c59ec..8519062 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -256,35 +256,17 @@ function escapeQueryStringTerm(term, { escapeWildcards = false } = {}) { } function buildQueryStrings(params) { - const queryStringObjects = []; + if (!Object.prototype.hasOwnProperty.call(params, 'keywordSearch')) return []; - getOrderedSearchParamNames(params).forEach((param) => { - if (param === 'virtualMatchString') return; - - const fieldSpec = constants.FIELDS[param]; - const fieldList = Array.isArray(fieldSpec) ? fieldSpec : [fieldSpec]; - - if (param === 'keywordSearch') { - queryStringObjects.push({ - simple_query_string: { - query: params[param], - default_operator: 'AND', - fields: fieldList - } - }); - return; - } - - queryStringObjects.push({ - query_string: { - query: params[param], + return [ + { + simple_query_string: { + query: params.keywordSearch, default_operator: 'AND', - fields: fieldList + fields: [constants.FIELDS.keywordSearch] } - }); - }); - - return queryStringObjects; + } + ]; } function getFilterFields(params) { @@ -308,7 +290,7 @@ function getFilterFields(params) { if (!fieldSpec) return; if (param === 'noRejected') { - if (params[param] === true || params[param] === '') { + if (params[param] === true) { fieldsToReturn.push(buildNoRejectedFilter(fieldSpec)); } return; diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index e8f4b88..f930f38 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -393,8 +393,9 @@ describe('searchQueryBuilder buildQueryStrings() function', () => { expect(output).toStrictEqual(testQueryString); }); - test('buildQueryStrings() ignores virtualMatchString because it is handled as a structured filter.', () => { + test('buildQueryStrings() ignores non-keyword fields instead of generating query_string clauses.', () => { const output = queryBuilder.buildQueryStrings({ + cweId: 'CWE-79', virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809' }); @@ -496,6 +497,14 @@ describe('searchQueryBuilder getFilterFields() function', () => { expect(output).toStrictEqual([]); }); + test('getFilterFields() does not retain legacy empty-string noRejected behavior.', () => { + const output = queryBuilder.getFilterFields({ + noRejected: '' + }); + + expect(output).toStrictEqual([]); + }); + test('getFilterFields() builds a cvssV2 severity range filter on baseScore.', () => { const output = queryBuilder.getFilterFields({ cvssV2Severity: 'MEDIUM' From 4cbb92f0c85b6008b1ed07f43b9bd6ef05caf020 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 16:42:58 -0400 Subject: [PATCH 12/33] Validate semantic search candidates incrementally --- controllers/searchController.js | 81 ++---------- controllers/searchController.test.js | 55 +++++++-- utils/cpe.js | 36 ++++++ utils/searchCandidateProcessor.js | 106 ++++++++++++++++ utils/searchCandidateProcessor.test.js | 165 +++++++++++++++++++++++++ utils/searchProvider.js | 122 ++---------------- utils/searchProvider.test.js | 73 ++++++----- utils/searchResponseValidator.js | 75 +++++++++++ 8 files changed, 484 insertions(+), 229 deletions(-) create mode 100644 utils/searchCandidateProcessor.js create mode 100644 utils/searchCandidateProcessor.test.js create mode 100644 utils/searchResponseValidator.js diff --git a/controllers/searchController.js b/controllers/searchController.js index e9d98bb..f36ea98 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -1,8 +1,8 @@ const PackageInfo = require('cve-core').packageInfo; const errors = require('../utils/errors'); -const cpe = require('../utils/cpe'); const searchPerformance = require('../utils/searchPerformance'); const searchDeadline = require('../utils/searchDeadline'); +const searchCandidateProcessor = require('../utils/searchCandidateProcessor'); const searchPostFilterCache = require('../utils/searchPostFilterCache'); const searchInFlightRegistry = require('../utils/searchInFlightRegistry'); const searchPostFilterWorkQueue = require('../utils/searchPostFilterWorkQueue'); @@ -18,75 +18,6 @@ function hasSearchPostFilters(postFilters) { return Boolean(postFilters?.cpeName || postFilters?.virtualMatchString || postFilters?.virtualMatchStringRange); } -function getHitsContainer(resp) { - return resp?.data?.hits; -} - -function hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext) { - if ( - postFilters?.cpeName && - !cpe.hitAppliesToCpeName(hit, postFilters.cpeName, cpeEvaluationContext, { - requireVulnerable: postFilters.isVulnerable === true - }) - ) { - return false; - } - - if ( - postFilters?.virtualMatchString && - !cpe.hitMatchesVirtualMatchString(hit, postFilters.virtualMatchString, cpeEvaluationContext) - ) { - return false; - } - - if ( - postFilters?.virtualMatchStringRange && - !cpe.hitOverlapsVirtualMatchStringRange(hit, postFilters.virtualMatchStringRange, cpeEvaluationContext) - ) { - return false; - } - - return true; -} - -function yieldToEventLoop() { - return new Promise((resolve) => setImmediate(resolve)); -} - -async function filterSearchHitsInChunks(resp, postFilters, signal, chunkSize) { - const hitsContainer = getHitsContainer(resp); - if (!Array.isArray(hitsContainer?.hits)) { - throw new TypeError('Semantic search candidate response must contain a hits array.'); - } - - const effectiveChunkSize = Math.max(1, chunkSize ?? searchProvider.getPostFilterEvaluationChunkSize()); - const cpeEvaluationContext = cpe.createEvaluationContext(); - const filteredHits = []; - - for (let start = 0; start < hitsContainer.hits.length; start += effectiveChunkSize) { - searchDeadline.throwIfAborted(signal); - const end = Math.min(start + effectiveChunkSize, hitsContainer.hits.length); - - for (let index = start; index < end; index += 1) { - const hit = hitsContainer.hits[index]; - if (hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext)) { - filteredHits.push(hit); - } - } - - if (end < hitsContainer.hits.length) { - await yieldToEventLoop(); - } - } - - searchDeadline.throwIfAborted(signal); - return { - hitsContainer, - candidateCount: hitsContainer.hits.length, - filteredHits - }; -} - function paginateHits(hits, responseOptions) { const from = responseOptions?.from ?? 0; const resultsPerPage = responseOptions?.resultsPerPage ?? hits.length; @@ -95,7 +26,7 @@ function paginateHits(hits, responseOptions) { } function buildPagedSearchResponse(resp, totalValue, pagedHits) { - const hitsContainer = getHitsContainer(resp); + const hitsContainer = resp?.data?.hits; return { ...resp, @@ -291,7 +222,12 @@ async function buildPostFilterCandidateResult(candidateBody, postFilters, metric data: candidateData }; const postFilterStart = searchPerformance.markTime(); - const { candidateCount, filteredHits } = await filterSearchHitsInChunks(candidateResp, postFilters, signal); + const { candidateCount, filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( + candidateResp, + postFilters, + signal, + searchProvider.getPostFilterEvaluationChunkSize() + ); metrics.postFilterElapsedMs = searchPerformance.elapsedMs(postFilterStart); return { @@ -598,7 +534,6 @@ async function webSearchHandler(req, res, next) { module.exports = { buildWebSearchSort, - filterSearchHitsInChunks, SEARCH: searchHandler, WEB_SEARCH: webSearchHandler }; diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 6d10afb..2ba9c83 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -57,6 +57,7 @@ jest.mock('../utils/searchProvider', () => ({ })); const controller = require('./searchController.js'); +const searchCandidateProcessor = require('../utils/searchCandidateProcessor'); const searchInFlightRegistry = require('../utils/searchInFlightRegistry'); const searchPostFilterCache = require('../utils/searchPostFilterCache'); @@ -617,7 +618,7 @@ describe('Search controller SEARCH handler', () => { test('chunked filtering rejects malformed candidate hit collections', async () => { await expect( - controller.filterSearchHitsInChunks( + searchCandidateProcessor.filterSearchHitsInChunks( { data: { hits: {} @@ -627,7 +628,10 @@ describe('Search controller SEARCH handler', () => { undefined, 100 ) - ).rejects.toThrow('Semantic search candidate response must contain a hits array.'); + ).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_response' + }); }); test('chunked filtering does not combine platform and version matches from different CPE entries', async () => { @@ -658,7 +662,7 @@ describe('Search controller SEARCH handler', () => { } ); - const { filteredHits } = await controller.filterSearchHitsInChunks( + const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(1, [candidate]), { cpeName: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:6.1.1:*:macos:*:*:*:*:*', @@ -679,7 +683,7 @@ describe('Search controller SEARCH handler', () => { versionEndExcluding: '6.1' }); - const { filteredHits } = await controller.filterSearchHitsInChunks( + const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(1, [candidate]), { cpeName: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:6.0.1:*:*:*:*:macos:*:*', @@ -703,7 +707,7 @@ describe('Search controller SEARCH handler', () => { versionEndExcluding: '6.02' }); - const exactResult = await controller.filterSearchHitsInChunks( + const exactResult = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(1, [exactCandidate]), { cpeName: 'cpe:2.3:o:linksys:re6250_firmware:1.0.4.1:*:*:*:*:*:*:*', @@ -712,7 +716,7 @@ describe('Search controller SEARCH handler', () => { undefined, 100 ); - const includedRangeResult = await controller.filterSearchHitsInChunks( + const includedRangeResult = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(1, [rangeCandidate]), { cpeName: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:6.1:*:*:*:*:*:*:*', @@ -721,7 +725,7 @@ describe('Search controller SEARCH handler', () => { undefined, 100 ); - const excludedRangeResult = await controller.filterSearchHitsInChunks( + const excludedRangeResult = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(1, [rangeCandidate]), { cpeName: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:6.2:*:*:*:*:*:*:*', @@ -758,7 +762,7 @@ describe('Search controller SEARCH handler', () => { }) ]; - const { filteredHits } = await controller.filterSearchHitsInChunks( + const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(candidates.length, candidates), { cpeName: 'cpe:2.3:a:microsoft:edge:-:*:*:*:*:android:*:*', @@ -1095,7 +1099,7 @@ describe('Search controller SEARCH handler', () => { }) ]; const abortController = new AbortController(); - const filtering = controller.filterSearchHitsInChunks( + const filtering = searchCandidateProcessor.filterSearchHitsInChunks( advancedResponse(2, hits), { virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809' @@ -1338,6 +1342,39 @@ describe('Search controller SEARCH handler', () => { } }); + test('SEARCH returns a sanitized 502 and does not cache candidates rejected during post-filter processing', async () => { + mockSearchRaw.mockResolvedValue( + rawResponse(1, [ + cpeHit('CVE-2024-0001', { + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + versionStartIncluding: '2.0', + versionEndExcluding: '1.0' + }) + ]) + ); + const req = { + searchParams: { + cpeName: 'cpe:2.3:a:vendor:product:1.5:*:*:*:*:*:*:*', + resultsPerPage: '1', + pageNumber: '1' + } + }; + + await controller.SEARCH(req, res, next); + const secondRes = createResponse(); + await controller.SEARCH(req, secondRes, next); + + expect(mockSearchRaw).toHaveBeenCalledTimes(2); + for (const response of [res, secondRes]) { + expect(response.status).toHaveBeenCalledWith(502); + expect(response.json).toHaveBeenCalledWith({ + error: 'UPSTREAM_SEARCH_ERROR', + message: 'Search backend returned an error. Please try again later.' + }); + } + expect(next).not.toHaveBeenCalled(); + }); + test('SEARCH logs redacted performance timing when enabled', async () => { mockShouldLogSearchPerformance.mockReturnValue(true); mockApiSearch.mockResolvedValue(advancedResponse(1, [{ _id: 'CVE-2024-0001' }])); diff --git a/utils/cpe.js b/utils/cpe.js index 97f0620..2a3dad9 100644 --- a/utils/cpe.js +++ b/utils/cpe.js @@ -3,6 +3,12 @@ const CPE_COMPONENT_COUNT = 11; const CPE_NOT_APPLICABLE = '-'; const REQUIRED_CPE_COMPONENT_INDEXES = [0, 1, 2, 3]; const VALID_CPE_PARTS = new Set(['a', 'h', 'o']); +const CPE_RANGE_FIELDS = [ + 'versionStartIncluding', + 'versionStartExcluding', + 'versionEndIncluding', + 'versionEndExcluding' +]; const VERSION_QUALIFIERS = ['alpha', 'beta', 'milestone', 'rc', 'snapshot', '', 'sp']; const VERSION_QUALIFIER_ALIASES = { a: 'alpha', @@ -391,6 +397,35 @@ function lowerBoundDoesNotExceedUpperBound(lower, upper) { return lower.inclusive && upper.inclusive; } +function isValidCpeMatch(cpeMatch, context) { + if (!cpeMatch || typeof cpeMatch !== 'object' || Array.isArray(cpeMatch)) return false; + if (typeof cpeMatch.criteria !== 'string' || cpeMatch.criteria.trim() === '') return false; + if (!getParsedCpeMatchCriteria(cpeMatch.criteria, context).valid) return false; + if (typeof cpeMatch.vulnerable !== 'boolean') return false; + if ( + !CPE_RANGE_FIELDS.every( + (field) => + cpeMatch[field] === undefined || (typeof cpeMatch[field] === 'string' && cpeMatch[field].trim() !== '') + ) + ) { + return false; + } + + if (hasBoundary(cpeMatch.versionStartIncluding) && hasBoundary(cpeMatch.versionStartExcluding)) return false; + if (hasBoundary(cpeMatch.versionEndIncluding) && hasBoundary(cpeMatch.versionEndExcluding)) return false; + + const range = buildVersionRange({ + start: hasBoundary(cpeMatch.versionStartIncluding) + ? cpeMatch.versionStartIncluding + : cpeMatch.versionStartExcluding, + startType: hasBoundary(cpeMatch.versionStartIncluding) ? 'including' : 'excluding', + end: hasBoundary(cpeMatch.versionEndIncluding) ? cpeMatch.versionEndIncluding : cpeMatch.versionEndExcluding, + endType: hasBoundary(cpeMatch.versionEndIncluding) ? 'including' : 'excluding' + }); + + return lowerBoundDoesNotExceedUpperBound(range.start, range.end); +} + function versionRangesOverlap(a, b) { return lowerBoundDoesNotExceedUpperBound(a.start, b.end) && lowerBoundDoesNotExceedUpperBound(b.start, a.end); } @@ -552,6 +587,7 @@ module.exports = { hitOverlapsVirtualMatchStringRange, hasConcreteComponentsAfter, hasConcreteLeadingComponents, + isValidCpeMatch, isValidCpeName, parseCpeMatchCriteria, parseCpeName, diff --git a/utils/searchCandidateProcessor.js b/utils/searchCandidateProcessor.js new file mode 100644 index 0000000..c34bba9 --- /dev/null +++ b/utils/searchCandidateProcessor.js @@ -0,0 +1,106 @@ +const cpe = require('./cpe'); +const constants = require('./constants'); +const searchDeadline = require('./searchDeadline'); +const { OPEN_SEARCH_RESPONSE_FAILURE_TYPES, OpenSearchResponseError } = require('./searchResponseValidator'); + +const CVE_ID_PATTERN = /^CVE-\d{4}-\d{4,19}$/; + +function isObject(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function isValidCpeCandidateHit(hit, cpeEvaluationContext) { + if (!isObject(hit) || !isObject(hit._source)) return false; + + const cveId = hit._source.cveMetadata?.cveId; + if (typeof cveId !== 'string' || !CVE_ID_PATTERN.test(cveId)) return false; + + const cpeApplicability = hit._source.containers?.cna?.cpeApplicability; + const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; + if (!applicabilityItems.length || applicabilityItems.some((item) => !isObject(item))) return false; + + let matchCount = 0; + for (const applicability of applicabilityItems) { + if (!Array.isArray(applicability.nodes)) return false; + + for (const node of applicability.nodes) { + if (!isObject(node) || !Array.isArray(node.cpeMatch)) return false; + if (!node.cpeMatch.every((match) => cpe.isValidCpeMatch(match, cpeEvaluationContext))) return false; + matchCount += node.cpeMatch.length; + } + } + + return matchCount > 0; +} + +function hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext) { + if ( + postFilters?.cpeName && + !cpe.hitAppliesToCpeName(hit, postFilters.cpeName, cpeEvaluationContext, { + requireVulnerable: postFilters.isVulnerable === true + }) + ) { + return false; + } + + if ( + postFilters?.virtualMatchString && + !cpe.hitMatchesVirtualMatchString(hit, postFilters.virtualMatchString, cpeEvaluationContext) + ) { + return false; + } + + if ( + postFilters?.virtualMatchStringRange && + !cpe.hitOverlapsVirtualMatchStringRange(hit, postFilters.virtualMatchStringRange, cpeEvaluationContext) + ) { + return false; + } + + return true; +} + +function yieldToEventLoop() { + return new Promise((resolve) => setImmediate(resolve)); +} + +async function filterSearchHitsInChunks(resp, postFilters, signal, chunkSize) { + const hitsContainer = resp?.data?.hits; + if (!Array.isArray(hitsContainer?.hits)) { + throw new OpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + const effectiveChunkSize = Math.max(1, chunkSize ?? constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE); + const cpeEvaluationContext = cpe.createEvaluationContext(); + const filteredHits = []; + + for (let start = 0; start < hitsContainer.hits.length; start += effectiveChunkSize) { + searchDeadline.throwIfAborted(signal); + const end = Math.min(start + effectiveChunkSize, hitsContainer.hits.length); + + for (let index = start; index < end; index += 1) { + const hit = hitsContainer.hits[index]; + if (!isValidCpeCandidateHit(hit, cpeEvaluationContext)) { + throw new OpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_CANDIDATE); + } + if (hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext)) { + filteredHits.push(hit); + } + } + + if (end < hitsContainer.hits.length) { + await yieldToEventLoop(); + } + } + + searchDeadline.throwIfAborted(signal); + return { + hitsContainer, + candidateCount: hitsContainer.hits.length, + filteredHits + }; +} + +module.exports = { + filterSearchHitsInChunks +}; diff --git a/utils/searchCandidateProcessor.test.js b/utils/searchCandidateProcessor.test.js new file mode 100644 index 0000000..fe205fa --- /dev/null +++ b/utils/searchCandidateProcessor.test.js @@ -0,0 +1,165 @@ +const searchCandidateProcessor = require('./searchCandidateProcessor'); +const searchDeadline = require('./searchDeadline'); + +function cpeCandidateHit(cpeMatch = {}) { + return { + _id: 'CVE-2024-0001', + _source: { + cveMetadata: { + cveId: 'CVE-2024-0001' + }, + containers: { + cna: { + cpeApplicability: [ + { + nodes: [ + { + cpeMatch: [ + { + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true, + ...cpeMatch + } + ] + } + ] + } + ] + } + } + } + }; +} + +function candidateResponse(hits) { + return { + data: { + hits: { + hits + } + } + }; +} + +describe('searchCandidateProcessor', () => { + test('validates and filters candidates in the same processing pass', async () => { + const matching = cpeCandidateHit({ + versionStartIncluding: '2.9', + versionEndExcluding: '2.11' + }); + const notMatching = cpeCandidateHit({ + versionStartIncluding: '3.0' + }); + notMatching._id = 'CVE-2024-0002'; + notMatching._source.cveMetadata.cveId = 'CVE-2024-0002'; + + await expect( + searchCandidateProcessor.filterSearchHitsInChunks( + candidateResponse([matching, notMatching]), + { + cpeName: 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*', + isVulnerable: true + }, + undefined, + 1 + ) + ).resolves.toMatchObject({ + candidateCount: 2, + filteredHits: [matching] + }); + }); + + test.each([ + [ + 'missing projected CVE ID', + { + ...cpeCandidateHit(), + _source: { + ...cpeCandidateHit()._source, + cveMetadata: {} + } + } + ], + [ + 'missing projected CPE applicability', + { + ...cpeCandidateHit(), + _source: { + ...cpeCandidateHit()._source, + containers: { + cna: {} + } + } + } + ], + ['missing CPE criteria', cpeCandidateHit({ criteria: undefined })], + ['invalid CPE criteria', cpeCandidateHit({ criteria: 'not-a-cpe-name' })], + ['missing CPE vulnerability flag', cpeCandidateHit({ vulnerable: undefined })], + ['invalid CPE range boundary', cpeCandidateHit({ versionEndExcluding: 2 })], + [ + 'conflicting inclusive and exclusive lower bounds', + cpeCandidateHit({ + versionStartIncluding: '1.0', + versionStartExcluding: '1.1' + }) + ], + [ + 'conflicting inclusive and exclusive upper bounds', + cpeCandidateHit({ + versionEndIncluding: '2.0', + versionEndExcluding: '2.1' + }) + ], + [ + 'reversed version range', + cpeCandidateHit({ + versionStartIncluding: '2.0', + versionEndIncluding: '1.0' + }) + ], + [ + 'empty exclusive version range', + cpeCandidateHit({ + versionStartIncluding: '1.0', + versionEndExcluding: '1.0' + }) + ] + ])('rejects CPE candidate hits with %s', async (_caseName, hit) => { + await expect( + searchCandidateProcessor.filterSearchHitsInChunks(candidateResponse([hit]), {}, undefined, 1) + ).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_candidate' + }); + }); + + test('accepts a one-version range when both equal boundaries are inclusive', async () => { + const candidate = cpeCandidateHit({ + versionStartIncluding: '1.0', + versionEndIncluding: '1.0' + }); + + await expect( + searchCandidateProcessor.filterSearchHitsInChunks(candidateResponse([candidate]), {}, undefined, 1) + ).resolves.toMatchObject({ + candidateCount: 1, + filteredHits: [candidate] + }); + }); + + test('honors cancellation before candidate validation begins', async () => { + const abortController = new AbortController(); + abortController.abort(new searchDeadline.SearchClientDisconnectedError()); + + await expect( + searchCandidateProcessor.filterSearchHitsInChunks( + candidateResponse([cpeCandidateHit({ criteria: undefined })]), + {}, + abortController.signal, + 1 + ) + ).rejects.toMatchObject({ + code: 'SEARCH_CLIENT_DISCONNECTED' + }); + }); +}); diff --git a/utils/searchProvider.js b/utils/searchProvider.js index 4d8b9eb..75502ae 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -1,7 +1,7 @@ const { AppConfig, BasicSearchManager, SearchProviderSpec, SearchReader } = require('cve-core'); const constants = require('./constants'); -const cpe = require('./cpe'); const searchDeadline = require('./searchDeadline'); +const searchResponseValidator = require('./searchResponseValidator'); class SearchProviderContractError extends Error { constructor() { @@ -19,30 +19,6 @@ class SearchProviderConfigurationError extends Error { } } -const OPEN_SEARCH_RESPONSE_FAILURE_TYPES = Object.freeze({ - MALFORMED_CANDIDATE: 'malformed_candidate', - MALFORMED_RESPONSE: 'malformed_response', - PARTIAL_SHARDS: 'partial_shards', - TIMEOUT: 'timeout' -}); -const CPE_CANDIDATE_OPERATION = 'cpe-candidate'; -const CVE_ID_PATTERN = /^CVE-\d{4}-\d{4,19}$/; -const CPE_RANGE_FIELDS = [ - 'versionStartIncluding', - 'versionStartExcluding', - 'versionEndIncluding', - 'versionEndExcluding' -]; - -class OpenSearchResponseError extends Error { - constructor(failureType) { - super(`OpenSearch returned an invalid ${failureType} response.`); - this.name = 'OpenSearchResponseError'; - this.code = 'OPEN_SEARCH_RESPONSE_INVALID'; - this.failureType = failureType; - } -} - let searchProviderConfig; let basicSearchManager; let searchReader; @@ -137,90 +113,6 @@ function validateSearchReader(reader) { return reader; } -function throwOpenSearchResponseError(failureType) { - throw new OpenSearchResponseError(failureType); -} - -function isObject(value) { - return Boolean(value) && typeof value === 'object' && !Array.isArray(value); -} - -function isValidCpeMatch(match) { - if (!isObject(match) || typeof match.criteria !== 'string' || match.criteria.trim() === '') return false; - if (!cpe.parseCpeMatchCriteria(match.criteria).valid) return false; - if (typeof match.vulnerable !== 'boolean') return false; - - return CPE_RANGE_FIELDS.every( - (field) => match[field] === undefined || (typeof match[field] === 'string' && match[field].trim() !== '') - ); -} - -function isValidCpeCandidateHit(hit) { - if (!isObject(hit) || !isObject(hit._source)) return false; - - const cveId = hit._source.cveMetadata?.cveId; - if (typeof cveId !== 'string' || !CVE_ID_PATTERN.test(cveId)) return false; - - const cpeApplicability = hit._source.containers?.cna?.cpeApplicability; - const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; - if (!applicabilityItems.length || applicabilityItems.some((item) => !isObject(item))) return false; - - let matchCount = 0; - for (const applicability of applicabilityItems) { - if (!Array.isArray(applicability.nodes)) return false; - - for (const node of applicability.nodes) { - if (!isObject(node) || !Array.isArray(node.cpeMatch)) return false; - if (!node.cpeMatch.every(isValidCpeMatch)) return false; - matchCount += node.cpeMatch.length; - } - } - - return matchCount > 0; -} - -function validateCpeCandidateHits(hits) { - if (hits.every(isValidCpeCandidateHit)) return; - - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_CANDIDATE); -} - -function validateOpenSearchResponse(body, { operation } = {}) { - if (!body || typeof body !== 'object' || Array.isArray(body)) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); - } - - if (body.timed_out === true) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.TIMEOUT); - } - - if (body._shards !== undefined) { - if (!body._shards || typeof body._shards !== 'object' || Array.isArray(body._shards)) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); - } - - if (body._shards.failed !== undefined) { - const failedShards = Number(body._shards.failed); - if (!Number.isInteger(failedShards) || failedShards < 0) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); - } - if (failedShards > 0) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.PARTIAL_SHARDS); - } - } - } - - if (!Array.isArray(body.hits?.hits)) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); - } - - if (operation === CPE_CANDIDATE_OPERATION) { - validateCpeCandidateHits(body.hits.hits); - } - - return body; -} - function getSearchReader() { if (!searchReader) { const config = getSearchProviderConfig(); @@ -356,7 +248,7 @@ async function searchOpenSearch(body, { signal, operation = 'search', onStart, r } ); - return validateOpenSearchResponse(response.body, { operation }); + return searchResponseValidator.validateOpenSearchResponse(response.body, { operation }); } async function checkOpenSearchReadiness({ signal } = {}) { @@ -387,7 +279,7 @@ async function checkOpenSearchReadiness({ signal } = {}) { operation: 'readiness' } ); - validateOpenSearchResponse(response.body); + searchResponseValidator.validateOpenSearchResponse(response.body); return true; } @@ -401,7 +293,7 @@ function isSearchProviderConfigurationError(err) { } function isOpenSearchResponseError(err) { - return err instanceof OpenSearchResponseError || err?.code === 'OPEN_SEARCH_RESPONSE_INVALID'; + return searchResponseValidator.isOpenSearchResponseError(err); } function getOpenSearchReadinessTimeoutMs() { @@ -508,10 +400,10 @@ module.exports = { resetSearchProviderForTests, searchOpenSearch, shouldLogSearchPerformance, - validateOpenSearchResponse, + validateOpenSearchResponse: searchResponseValidator.validateOpenSearchResponse, validateSearchReader, - OPEN_SEARCH_RESPONSE_FAILURE_TYPES, - OpenSearchResponseError, + OPEN_SEARCH_RESPONSE_FAILURE_TYPES: searchResponseValidator.OPEN_SEARCH_RESPONSE_FAILURE_TYPES, + OpenSearchResponseError: searchResponseValidator.OpenSearchResponseError, SearchProviderConfigurationError, SearchProviderContractError }; diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index 8abc85d..4fd4a79 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -327,6 +327,40 @@ describe('searchProvider', () => { }); }); + test.each([ + ['missing', undefined], + ['non-object', 1], + ['missing value', {}], + ['negative', { value: -1 }], + ['non-integer', { value: 1.5 }], + ['string', { value: '1' }] + ])('rejects a standard OpenSearch response with a %s total', async (_caseName, total) => { + mockClientSearch.mockResolvedValue({ + body: { + hits: { + total, + hits: [] + } + } + }); + + await expect( + searchProvider.searchOpenSearch( + { + query: { + match_all: {} + } + }, + { + operation: 'standard' + } + ) + ).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_response' + }); + }); + test('accepts schema-shaped CPE candidate hits for candidate operations', async () => { const body = { hits: { @@ -371,38 +405,12 @@ describe('searchProvider', () => { ).resolves.toBe(body); }); - test.each([ - [ - 'missing projected CVE ID', - { - ...cpeCandidateHit(), - _source: { - ...cpeCandidateHit()._source, - cveMetadata: {} - } - } - ], - [ - 'missing projected CPE applicability', - { - ...cpeCandidateHit(), - _source: { - ...cpeCandidateHit()._source, - containers: { - cna: {} - } - } - } - ], - ['missing CPE criteria', cpeCandidateHit({ criteria: undefined })], - ['invalid CPE criteria', cpeCandidateHit({ criteria: 'not-a-cpe-name' })], - ['missing CPE vulnerability flag', cpeCandidateHit({ vulnerable: undefined })], - ['invalid CPE range boundary', cpeCandidateHit({ versionEndExcluding: 2 })] - ])('rejects CPE candidate hits with %s', async (_caseName, hit) => { + test('does not synchronously traverse CPE candidate elements during response validation', async () => { + const malformedHit = cpeCandidateHit({ criteria: undefined }); mockClientSearch.mockResolvedValue({ body: { hits: { - hits: [hit] + hits: [malformedHit] } } }); @@ -418,9 +426,10 @@ describe('searchProvider', () => { operation: 'cpe-candidate' } ) - ).rejects.toMatchObject({ - code: 'OPEN_SEARCH_RESPONSE_INVALID', - failureType: 'malformed_candidate' + ).resolves.toStrictEqual({ + hits: { + hits: [malformedHit] + } }); }); diff --git a/utils/searchResponseValidator.js b/utils/searchResponseValidator.js new file mode 100644 index 0000000..47a4acc --- /dev/null +++ b/utils/searchResponseValidator.js @@ -0,0 +1,75 @@ +const OPEN_SEARCH_RESPONSE_FAILURE_TYPES = Object.freeze({ + MALFORMED_CANDIDATE: 'malformed_candidate', + MALFORMED_RESPONSE: 'malformed_response', + PARTIAL_SHARDS: 'partial_shards', + TIMEOUT: 'timeout' +}); +const TOTAL_REQUIRED_OPERATIONS = new Set(['standard']); + +class OpenSearchResponseError extends Error { + constructor(failureType) { + super(`OpenSearch returned an invalid ${failureType} response.`); + this.name = 'OpenSearchResponseError'; + this.code = 'OPEN_SEARCH_RESPONSE_INVALID'; + this.failureType = failureType; + } +} + +function throwOpenSearchResponseError(failureType) { + throw new OpenSearchResponseError(failureType); +} + +function isObject(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function hasValidTotal(hits) { + return isObject(hits.total) && Number.isInteger(hits.total.value) && hits.total.value >= 0; +} + +function validateOpenSearchResponse(body, { operation } = {}) { + if (!isObject(body)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + if (body.timed_out === true) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.TIMEOUT); + } + + if (body._shards !== undefined) { + if (!isObject(body._shards)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + if (body._shards.failed !== undefined) { + const failedShards = Number(body._shards.failed); + if (!Number.isInteger(failedShards) || failedShards < 0) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + if (failedShards > 0) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.PARTIAL_SHARDS); + } + } + } + + if (!isObject(body.hits) || !Array.isArray(body.hits.hits)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + if (TOTAL_REQUIRED_OPERATIONS.has(operation) && !hasValidTotal(body.hits)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + + return body; +} + +function isOpenSearchResponseError(err) { + return err instanceof OpenSearchResponseError || err?.code === 'OPEN_SEARCH_RESPONSE_INVALID'; +} + +module.exports = { + isOpenSearchResponseError, + validateOpenSearchResponse, + OPEN_SEARCH_RESPONSE_FAILURE_TYPES, + OpenSearchResponseError +}; From 26e3c543781ac82befc48130759e393093514577 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 16:54:26 -0400 Subject: [PATCH 13/33] Tighten CVE and virtual match validation --- README.md | 6 +-- api-docs/openapi.json | 4 +- api-docs/openapi.test.js | 7 ++- controllers/middleware.js | 9 ++-- controllers/middleware.test.js | 66 ++++++++++++++++---------- routes/swagger.js | 4 +- utils/constants.js | 8 ++-- utils/cpe.js | 3 ++ utils/cpe.test.js | 10 ++++ utils/cveId.js | 18 +++++++ utils/cveId.test.js | 28 +++++++++++ utils/errors.js | 12 ++--- utils/errors.test.js | 2 +- utils/searchCandidateProcessor.js | 7 ++- utils/searchCandidateProcessor.test.js | 25 ++++++++++ utils/searchQueryBuilder.js | 3 +- 16 files changed, 155 insertions(+), 57 deletions(-) create mode 100644 utils/cveId.js create mode 100644 utils/cveId.test.js diff --git a/README.md b/README.md index fb9c40d..8cb5545 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ String inputs are bounded before query construction: | Field | Limit | | ---------------------------- | ---------------------------------------------------------------------: | | `keywordSearch` | 512 characters and 64 whitespace-delimited terms | -| `cveIds` | 4,096 characters overall, 100 IDs, and 64 characters per ID | +| `cveIds` | 4,096 characters overall, 100 IDs, and 4-19 sequence digits per CVE ID | | `cweId` | 32 characters | | `cpeName` | 2,048 characters | | `virtualMatchString` | 2,048 characters | @@ -134,7 +134,7 @@ Publication and last-modified date filters support one-sided ranges. When both v Pagination is limited to complete pages inside the first 10,000 results: `from + resultsPerPage` must not exceed 10,000. `totalPages` reports the smaller of the pages implied by the match count and the complete pages accessible inside that window. For example, a page size of 30 allows at most 333 pages. The match total itself is not reduced by this pagination calculation. Point-in-time search and deep pagination remain deferred. -Use `cveIds` to request up to 100 CVE records by comma-separated CVE ID. +Use `cveIds` to request up to 100 CVE records by comma-separated CVE ID. Each ID must contain the `CVE` prefix, a four-digit year, and a sequence number containing 4 to 19 digits. The prefix is accepted without regard to case and normalized to uppercase for the OpenSearch query. ```bash curl -X POST "http://localhost:3000/search" \ @@ -180,7 +180,7 @@ Set `isVulnerable` to `true` with `cpeName` to require the matching CPE Match Cr ### Virtual Match String Search -Use `virtualMatchString` when a user needs CPE Match Criteria compatibility matching instead of concrete asset matching. String literal components are compared without regard to lexical case. When a part is supplied, it must be `a`, `h`, `o`, `*`, or `-`. Omitted trailing components and `*` components act as broad matches. +Use `virtualMatchString` when a user needs CPE Match Criteria compatibility matching instead of concrete asset matching. String literal components are compared without regard to lexical case. When a part is supplied, it must be `a`, `h`, `o`, `*`, or `-`. Omitted trailing components and `*` components act as broad matches. Explicit empty colon-delimited components are invalid; omit trailing components instead. Partial product example: diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 6b1f60e..66fc284 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -403,7 +403,7 @@ "type": "string", "maxLength": 4096, "example": "CVE-2024-1234,CVE-2024-5678", - "description": "Returns one or more CVE records identified by a comma-separated list of CVE IDs. Limited to 100 CVE IDs, 64 characters per ID, and 4096 characters overall." + "description": "Returns one or more CVE records identified by a comma-separated list of CVE IDs. Each ID must contain the CVE prefix, a four-digit year, and a 4-to-19-digit sequence number. The prefix is accepted without regard to case. Limited to 100 CVE IDs and 4096 characters overall." }, "noRejected": { "type": "boolean", @@ -577,7 +577,7 @@ "type": "string", "maxLength": 2048, "example": "cpe:2.3:o:microsoft:windows_10_1809", - "description": "Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component." + "description": "Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component." } } }, diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 4f7e846..33cdda6 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -78,7 +78,9 @@ describe('OpenAPI contract', () => { expect(schema.properties.cveIds.type).toBe('string'); expect(schema.properties.cveIds.description).toContain('comma-separated list of CVE IDs'); expect(schema.properties.cveIds.description).toContain(`${constants.MAX_CVE_IDS} CVE IDs`); - expect(schema.properties.cveIds.description).toContain(`${constants.MAX_CVE_ID_LENGTH} characters per ID`); + expect(schema.properties.cveIds.description).toContain( + `${constants.MIN_CVE_ID_SEQUENCE_DIGITS}-to-${constants.MAX_CVE_ID_SEQUENCE_DIGITS}-digit sequence number` + ); expect(schema.properties.keywordSearch.description).toContain( `${constants.MAX_KEYWORD_SEARCH_TOKENS} whitespace-delimited terms` ); @@ -116,6 +118,9 @@ describe('OpenAPI contract', () => { expect(schema.properties.virtualMatchString.description).toContain('insensitive to lexical case'); expect(schema.properties.virtualMatchString.description).toContain('without a version component'); expect(schema.properties.virtualMatchString.description).toContain('part must be a, h, o, *, or -'); + expect(schema.properties.virtualMatchString.description).toContain( + 'explicit empty colon-delimited components are invalid' + ); expect(schema.properties.virtualMatchString.example).toBe('cpe:2.3:o:microsoft:windows_10_1809'); expect(schema.properties.cweId.description).toContain('CNA data'); expect(schema.properties.cweId.description).toContain('ADP-provided CWE data is not searched'); diff --git a/controllers/middleware.js b/controllers/middleware.js index eea9080..f210169 100644 --- a/controllers/middleware.js +++ b/controllers/middleware.js @@ -1,6 +1,7 @@ const errors = require('../utils/errors'); const constants = require('../utils/constants'); const cpe = require('../utils/cpe'); +const cveId = require('../utils/cveId'); const cvss = require('../utils/cvss'); const error = new errors.errors(); @@ -58,7 +59,6 @@ const nonStringSearchParams = [ const nonEmptyStringParams = constants.SEARCH_QUERY_PARAMS.filter((param) => !nonStringSearchParams.includes(param)); const dateTimeFormat = /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])T([01]\d|2[0-3]):[0-5]\d:[0-5]\d\.\d{3}Z$/; -const cveIdFormat = /^CVE-\d{4}-\d{4,}$/i; function isTrueParam(value) { return value === true || String(value).toLowerCase() === 'true'; @@ -83,8 +83,8 @@ function countWhitespaceDelimitedTokens(value) { function parseCveIds(value) { if (typeof value !== 'string') return null; - const cveIds = value.split(',').map((id) => id.trim()); - if (!cveIds.length || cveIds.some((id) => !id || !cveIdFormat.test(id))) return null; + const cveIds = value.split(',').map(cveId.normalizeCveId); + if (!cveIds.length || cveIds.some((id) => !cveId.isValidCveId(id))) return null; return cveIds; } @@ -152,9 +152,6 @@ function validateParams(req, res, next) { if (!cveIds) { return res.status(400).json(error.cveIdsFormatError()); } - if (cveIds.some((cveId) => cveId.length > constants.MAX_CVE_ID_LENGTH)) { - return res.status(400).json(error.cveIdLengthError()); - } if (cveIds.length > constants.MAX_CVE_IDS) { return res.status(400).json(error.cveIdsLimitError()); } diff --git a/controllers/middleware.test.js b/controllers/middleware.test.js index dd819e9..6f82844 100644 --- a/controllers/middleware.test.js +++ b/controllers/middleware.test.js @@ -158,6 +158,17 @@ describe('middleware validation function', () => { ); }); + test('ValidateParams rejects an explicit empty virtualMatchString component', () => { + validate({ + virtualMatchString: 'cpe:2.3:o::windows_10_1809' + }); + + expectValidationError( + 'VIRTUAL_MATCH_STRING_FORMAT', + 'The virtualMatchString parameter must be a CPE 2.3 match string.' + ); + }); + test('ValidateParams accepts broad virtualMatchString values', () => { validate({ virtualMatchString: 'cpe:2.3:o:*:windows_10_1809' @@ -344,6 +355,19 @@ describe('middleware validation function', () => { ); }); + test('ValidateParams rejects an explicit empty virtualMatchString component for version ranges', () => { + validate({ + virtualMatchString: 'cpe:2.3:o::linux_kernel', + versionStart: '1.0', + versionStartType: 'including' + }); + + expectValidationError( + 'VIRTUAL_MATCH_STRING_RANGE_FORMAT', + 'The virtualMatchString parameter must be a partial CPE 2.3 match string when used with version range parameters.' + ); + }); + test('ValidateParams returns an error when version range virtualMatchString includes a version component', () => { validate({ virtualMatchString: 'cpe:2.3:o:linux:linux_kernel:*', @@ -435,15 +459,20 @@ describe('middleware validation function', () => { }); test.each([ - ['empty entry', 'CVE-2024-0001,,CVE-2024-0002'], - ['invalid CVE ID', 'CVE-2024-0001,not-a-cve'], - ['repeated query parameter', ['CVE-2024-0001', 'CVE-2024-0002']] - ])('ValidateParams returns an error when cveIds has an %s.', (_caseName, cveIds) => { + ['an empty entry', 'CVE-2024-0001,,CVE-2024-0002'], + ['an invalid CVE ID', 'CVE-2024-0001,not-a-cve'], + ['a short sequence number', `CVE-2024-${'1'.repeat(constants.MIN_CVE_ID_SEQUENCE_DIGITS - 1)}`], + ['a long sequence number', `CVE-2024-${'1'.repeat(constants.MAX_CVE_ID_SEQUENCE_DIGITS + 1)}`], + ['a repeated query parameter', ['CVE-2024-0001', 'CVE-2024-0002']] + ])('ValidateParams returns an error when cveIds has %s.', (_caseName, cveIds) => { validate({ cveIds }); - expectValidationError('CVE_IDS_FORMAT', 'The cveIds parameter must be a comma-separated list of CVE IDs.'); + expectValidationError( + 'CVE_IDS_FORMAT', + `The cveIds parameter must be a comma-separated list of CVE IDs with a four-digit year and a ${constants.MIN_CVE_ID_SEQUENCE_DIGITS}-to-${constants.MAX_CVE_ID_SEQUENCE_DIGITS}-digit sequence number.` + ); }); test('ValidateParams returns an error when cveIds contains more than 100 IDs.', () => { @@ -481,30 +510,15 @@ describe('middleware validation function', () => { expectValidationSuccess(); }); - test('ValidateParams accepts cveIds and each CVE ID at their maximum lengths.', () => { - const idPrefix = 'CVE-2024-'; - const longId = `${idPrefix}${'1'.repeat(constants.MAX_CVE_ID_LENGTH - idPrefix.length)}`; - const shortId = `${idPrefix}${'2'.repeat(32 - idPrefix.length)}`; - const mediumId = `${idPrefix}${'3'.repeat(33 - idPrefix.length)}`; - const cveIds = [...Array(62).fill(longId), shortId, mediumId].join(','); - - expect(cveIds).toHaveLength(constants.SEARCH_STRING_LENGTH_LIMITS.cveIds); - validate({ cveIds }); - - expectValidationSuccess(); - }); - - test('ValidateParams returns an error when an individual CVE ID exceeds its maximum length.', () => { - const idPrefix = 'CVE-2024-'; - + test('ValidateParams accepts case-insensitive cveIds at both sequence-length boundaries.', () => { validate({ - cveIds: `${idPrefix}${'1'.repeat(constants.MAX_CVE_ID_LENGTH - idPrefix.length + 1)}` + cveIds: [ + `cve-2024-${'1'.repeat(constants.MIN_CVE_ID_SEQUENCE_DIGITS)}`, + `CVE-2024-${'2'.repeat(constants.MAX_CVE_ID_SEQUENCE_DIGITS)}` + ].join(',') }); - expectValidationError( - 'CVE_ID_LENGTH', - `Each CVE ID in the cveIds field must be ${constants.MAX_CVE_ID_LENGTH} characters or fewer.` - ); + expectValidationSuccess(); }); test.each([ diff --git a/routes/swagger.js b/routes/swagger.js index ba704cc..6d9f9c1 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -31,7 +31,7 @@ const searchRequestProperties = { } ), cveIds: searchRequestProperty( - `Returns one or more CVE records identified by a comma-separated list of CVE IDs. Limited to ${constants.MAX_CVE_IDS} CVE IDs, ${constants.MAX_CVE_ID_LENGTH} characters per ID, and ${constants.SEARCH_STRING_LENGTH_LIMITS.cveIds} characters overall.`, + `Returns one or more CVE records identified by a comma-separated list of CVE IDs. Each ID must contain the CVE prefix, a four-digit year, and a ${constants.MIN_CVE_ID_SEQUENCE_DIGITS}-to-${constants.MAX_CVE_ID_SEQUENCE_DIGITS}-digit sequence number. The prefix is accepted without regard to case. Limited to ${constants.MAX_CVE_IDS} CVE IDs and ${constants.SEARCH_STRING_LENGTH_LIMITS.cveIds} characters overall.`, { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cveIds, @@ -222,7 +222,7 @@ const searchRequestProperties = { } ), virtualMatchString: searchRequestProperty( - 'Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component.', + 'Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component.', { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.virtualMatchString, diff --git a/utils/constants.js b/utils/constants.js index 0093369..85edb5a 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -5,7 +5,8 @@ const MAX_CVSS_METRICS_LENGTH = 256; const MAX_CVSS_METRIC_TOKENS = 32; const MAX_CVSS_METRIC_TOKEN_LENGTH = 32; const MAX_CVE_IDS = 100; -const MAX_CVE_ID_LENGTH = 64; +const MIN_CVE_ID_SEQUENCE_DIGITS = 4; +const MAX_CVE_ID_SEQUENCE_DIGITS = 19; const MAX_KEYWORD_SEARCH_TOKENS = 64; const JSON_REQUEST_BODY_LIMIT_KB = 100; const JSON_REQUEST_BODY_LIMIT_BYTES = JSON_REQUEST_BODY_LIMIT_KB * 1024; @@ -54,7 +55,7 @@ const cpeMatchFields = { /** * keywordSearch: list of terms * keywordExactMatch: list of terms, must be an exact match to terms in a description - * cveIds: comma-separated list of CVE-year-XXXX identifiers + * cveIds: comma-separated list of CVE-year-sequence identifiers * noRejected: no-value flag that excludes rejected CVE records * cweId: CWE-XXX * cvssV2Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL @@ -132,7 +133,8 @@ module.exports = { MAX_CVSS_METRIC_TOKENS, MAX_CVSS_METRIC_TOKEN_LENGTH, MAX_CVE_IDS, - MAX_CVE_ID_LENGTH, + MIN_CVE_ID_SEQUENCE_DIGITS, + MAX_CVE_ID_SEQUENCE_DIGITS, MAX_KEYWORD_SEARCH_TOKENS, JSON_REQUEST_BODY_LIMIT_KB, JSON_REQUEST_BODY_LIMIT_BYTES, diff --git a/utils/cpe.js b/utils/cpe.js index 2a3dad9..ca00341 100644 --- a/utils/cpe.js +++ b/utils/cpe.js @@ -131,6 +131,9 @@ function parseVirtualMatchString(input) { if (rawComponents.length > CPE_COMPONENT_COUNT) { return { valid: false, reason: 'component_count' }; } + if (rawComponents.some((component) => component.trim() === '')) { + return { valid: false, reason: 'empty_component' }; + } if (rawComponents.length && !isValidCpePart(rawComponents[0], { allowLogicalValues: true })) { return { valid: false, reason: 'part' }; } diff --git a/utils/cpe.test.js b/utils/cpe.test.js index caa429e..995dccb 100644 --- a/utils/cpe.test.js +++ b/utils/cpe.test.js @@ -84,6 +84,16 @@ describe('CPE helpers', () => { expect(parsed.components).toStrictEqual(['*', '*', '*', '*', '*', '*', '*', '*', '*', '*', '*']); }); + test.each(['cpe:2.3:', 'cpe:2.3:o:', 'cpe:2.3:o::product', 'cpe:2.3:o:vendor::1.0', 'cpe:2.3:o:vendor:product: '])( + 'parseVirtualMatchString rejects explicit empty component in %s', + (value) => { + expect(cpe.parseVirtualMatchString(value)).toStrictEqual({ + valid: false, + reason: 'empty_component' + }); + } + ); + test('parseVirtualMatchString rejects an unsupported concrete part value', () => { expect(cpe.parseVirtualMatchString('cpe:2.3:x:vendor:product')).toStrictEqual({ valid: false, diff --git a/utils/cveId.js b/utils/cveId.js new file mode 100644 index 0000000..759d0ef --- /dev/null +++ b/utils/cveId.js @@ -0,0 +1,18 @@ +const constants = require('./constants'); + +const CVE_ID_PATTERN = new RegExp( + `^CVE-\\d{4}-\\d{${constants.MIN_CVE_ID_SEQUENCE_DIGITS},${constants.MAX_CVE_ID_SEQUENCE_DIGITS}}$` +); + +function normalizeCveId(value) { + return typeof value === 'string' ? value.trim().toUpperCase() : ''; +} + +function isValidCveId(value) { + return typeof value === 'string' && CVE_ID_PATTERN.test(value); +} + +module.exports = { + isValidCveId, + normalizeCveId +}; diff --git a/utils/cveId.test.js b/utils/cveId.test.js new file mode 100644 index 0000000..2710819 --- /dev/null +++ b/utils/cveId.test.js @@ -0,0 +1,28 @@ +const constants = require('./constants'); +const cveId = require('./cveId'); + +describe('CVE ID helpers', () => { + test.each([ + 'CVE-2024-0001', + `CVE-2024-${'1'.repeat(constants.MIN_CVE_ID_SEQUENCE_DIGITS)}`, + `CVE-2024-${'1'.repeat(constants.MAX_CVE_ID_SEQUENCE_DIGITS)}` + ])('isValidCveId accepts a valid identifier %s', (value) => { + expect(cveId.isValidCveId(value)).toBe(true); + }); + + test.each([ + 'cve-2024-0001', + 'CVE-24-0001', + `CVE-2024-${'1'.repeat(constants.MIN_CVE_ID_SEQUENCE_DIGITS - 1)}`, + `CVE-2024-${'1'.repeat(constants.MAX_CVE_ID_SEQUENCE_DIGITS + 1)}`, + 'CVE-2024-12A4', + ' CVE-2024-0001 ' + ])('isValidCveId rejects a malformed identifier %s', (value) => { + expect(cveId.isValidCveId(value)).toBe(false); + }); + + test('normalizeCveId trims and uppercases request input', () => { + expect(cveId.normalizeCveId(' cve-2024-0001 ')).toBe('CVE-2024-0001'); + expect(cveId.normalizeCveId(undefined)).toBe(''); + }); +}); diff --git a/utils/errors.js b/utils/errors.js index d001e55..9b442d8 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -110,7 +110,10 @@ class errors { } cveIdsFormatError() { - return makeError('CVE_IDS_FORMAT', 'The cveIds parameter must be a comma-separated list of CVE IDs.'); + return makeError( + 'CVE_IDS_FORMAT', + `The cveIds parameter must be a comma-separated list of CVE IDs with a four-digit year and a ${constants.MIN_CVE_ID_SEQUENCE_DIGITS}-to-${constants.MAX_CVE_ID_SEQUENCE_DIGITS}-digit sequence number.` + ); } cveIdsLimitError() { @@ -165,13 +168,6 @@ class errors { return makeError('SEARCH_FIELD_TOKEN_LIMIT', `The ${field} field must contain ${maxTokens} or fewer terms.`); } - cveIdLengthError() { - return makeError( - 'CVE_ID_LENGTH', - `Each CVE ID in the cveIds field must be ${constants.MAX_CVE_ID_LENGTH} characters or fewer.` - ); - } - cvssV2SeverityError(severity) { return makeError( 'INVALID_CVSSV2_SEVERITY', diff --git a/utils/errors.test.js b/utils/errors.test.js index b40efc8..582098e 100644 --- a/utils/errors.test.js +++ b/utils/errors.test.js @@ -156,7 +156,7 @@ describe('error response helpers', () => { [], { error: 'CVE_IDS_FORMAT', - message: 'The cveIds parameter must be a comma-separated list of CVE IDs.' + message: `The cveIds parameter must be a comma-separated list of CVE IDs with a four-digit year and a ${constants.MIN_CVE_ID_SEQUENCE_DIGITS}-to-${constants.MAX_CVE_ID_SEQUENCE_DIGITS}-digit sequence number.` } ], [ diff --git a/utils/searchCandidateProcessor.js b/utils/searchCandidateProcessor.js index c34bba9..5e408c1 100644 --- a/utils/searchCandidateProcessor.js +++ b/utils/searchCandidateProcessor.js @@ -1,10 +1,9 @@ const cpe = require('./cpe'); const constants = require('./constants'); +const cveId = require('./cveId'); const searchDeadline = require('./searchDeadline'); const { OPEN_SEARCH_RESPONSE_FAILURE_TYPES, OpenSearchResponseError } = require('./searchResponseValidator'); -const CVE_ID_PATTERN = /^CVE-\d{4}-\d{4,19}$/; - function isObject(value) { return Boolean(value) && typeof value === 'object' && !Array.isArray(value); } @@ -12,8 +11,8 @@ function isObject(value) { function isValidCpeCandidateHit(hit, cpeEvaluationContext) { if (!isObject(hit) || !isObject(hit._source)) return false; - const cveId = hit._source.cveMetadata?.cveId; - if (typeof cveId !== 'string' || !CVE_ID_PATTERN.test(cveId)) return false; + const candidateCveId = hit._source.cveMetadata?.cveId; + if (!cveId.isValidCveId(candidateCveId)) return false; const cpeApplicability = hit._source.containers?.cna?.cpeApplicability; const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; diff --git a/utils/searchCandidateProcessor.test.js b/utils/searchCandidateProcessor.test.js index fe205fa..8c39b81 100644 --- a/utils/searchCandidateProcessor.test.js +++ b/utils/searchCandidateProcessor.test.js @@ -1,3 +1,4 @@ +const constants = require('./constants'); const searchCandidateProcessor = require('./searchCandidateProcessor'); const searchDeadline = require('./searchDeadline'); @@ -80,6 +81,30 @@ describe('searchCandidateProcessor', () => { } } ], + [ + 'lowercase projected CVE ID', + { + ...cpeCandidateHit(), + _source: { + ...cpeCandidateHit()._source, + cveMetadata: { + cveId: 'cve-2024-0001' + } + } + } + ], + [ + 'oversized projected CVE ID sequence', + { + ...cpeCandidateHit(), + _source: { + ...cpeCandidateHit()._source, + cveMetadata: { + cveId: `CVE-2024-${'1'.repeat(constants.MAX_CVE_ID_SEQUENCE_DIGITS + 1)}` + } + } + } + ], [ 'missing projected CPE applicability', { diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index 8519062..dda5ca9 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -1,5 +1,6 @@ const constants = require('./constants'); const cpe = require('./cpe'); +const cveId = require('./cveId'); const cvss = require('./cvss'); const { MAX_RESULTS_LIMIT, RESULTS_PER_PAGE } = constants; @@ -361,7 +362,7 @@ function buildCvssTokenRegexp(token) { function parseCveIds(value) { return String(value ?? '') .split(',') - .map((id) => id.trim().toUpperCase()) + .map(cveId.normalizeCveId) .filter(Boolean); } From 3444581d98758b8262e50ff445a6c906c8ccf1b4 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 17:20:26 -0400 Subject: [PATCH 14/33] Clarify CPE applicability entry semantics --- README.md | 2 + api-docs/openapi.json | 4 +- api-docs/openapi.test.js | 6 ++ routes/swagger.js | 4 +- utils/cpe.js | 16 ++--- utils/cpe.test.js | 81 ++++++++++++++++++++------ utils/searchCandidateProcessor.test.js | 22 +++++++ 7 files changed, 101 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 8cb5545..47d6537 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,8 @@ curl -X POST "http://localhost:3000/search" \ When filtering with `cpeName`, provide a CPE 2.3 name with part `a`, `h`, or `o` and non-wildcard `vendor`, `product`, and `version` components. Invalid `cpeName` values return HTTP 404. The API compares the provided CPE Name against CPE Match Criteria, including exact match strings and version range match strings when the indexed CVE data includes range fields. Use `virtualMatchString` when you intentionally need to search for partial CPE Match String values without those required components. +Both `cpeName` and `virtualMatchString` evaluate individual CPE Match Criteria entries in the direct `cpeApplicability.nodes[].cpeMatch` arrays. They do not evaluate a node's `operator` or `negate` value as a complete system applicability expression. Evaluating those Boolean expressions requires a set of installed assets, while each search request supplies only one CPE Name or Match String. + ### CPE Name Search Use `cpeName` when a user has a concrete asset CPE Name and needs CVE records whose CPE Match Criteria apply to that asset version. The part must be `a`, `h`, or `o`; the `vendor`, `product`, and `version` components must all be concrete values, not `*`. diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 66fc284..b889eed 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -499,7 +499,7 @@ "type": "string", "maxLength": 2048, "example": "cpe:2.3:a:freetype:freetype:2.8.1:*:*:*:*:*:*:*", - "description": "Returns CVE records whose CPE Match Criteria apply to the provided CPE 2.3 asset name. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches." + "description": "Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches." }, "isVulnerable": { "type": "boolean", @@ -577,7 +577,7 @@ "type": "string", "maxLength": 2048, "example": "cpe:2.3:o:microsoft:windows_10_1809", - "description": "Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component." + "description": "Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component." } } }, diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 33cdda6..5162c29 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -99,6 +99,8 @@ describe('OpenAPI contract', () => { expect(schema.properties.cpeName.description).toContain('optional * components remain broad matches'); expect(schema.properties.cpeName.description).toContain('insensitive to lexical case'); expect(schema.properties.cpeName.description).toContain('exact CPE Match Criteria preserve leading zeros'); + expect(schema.properties.cpeName.description).toContain('entries independently'); + expect(schema.properties.cpeName.description).toContain("does not evaluate a node's operator or negate value"); expect(schema.properties.versionStartType.enum).toStrictEqual(constants.CPE_VERSION_RANGE_TYPES); expect(schema.properties.versionEndType.enum).toStrictEqual(constants.CPE_VERSION_RANGE_TYPES); expect(schema.properties.resultsPerPage.maximum).toBe(constants.MAX_RESULTS_LIMIT); @@ -121,6 +123,10 @@ describe('OpenAPI contract', () => { expect(schema.properties.virtualMatchString.description).toContain( 'explicit empty colon-delimited components are invalid' ); + expect(schema.properties.virtualMatchString.description).toContain('entries independently'); + expect(schema.properties.virtualMatchString.description).toContain( + "does not evaluate a node's operator or negate value" + ); expect(schema.properties.virtualMatchString.example).toBe('cpe:2.3:o:microsoft:windows_10_1809'); expect(schema.properties.cweId.description).toContain('CNA data'); expect(schema.properties.cweId.description).toContain('ADP-provided CWE data is not searched'); diff --git a/routes/swagger.js b/routes/swagger.js index 6d9f9c1..674ae6b 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -126,7 +126,7 @@ const searchRequestProperties = { } ), cpeName: searchRequestProperty( - 'Returns CVE records whose CPE Match Criteria apply to the provided CPE 2.3 asset name. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches.', + "Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches.", { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cpeName, @@ -222,7 +222,7 @@ const searchRequestProperties = { } ), virtualMatchString: searchRequestProperty( - 'Returns CVE records whose CPE Match Criteria are compatible with the provided CPE 2.3 match string. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component.', + "Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component.", { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.virtualMatchString, diff --git a/utils/cpe.js b/utils/cpe.js index ca00341..58e6741 100644 --- a/utils/cpe.js +++ b/utils/cpe.js @@ -535,16 +535,6 @@ function cpeMatchMatchesVirtualMatchString(cpeMatch, virtualMatchString, context return true; } -function collectCpeMatchesFromNode(node, matches) { - if (!node || typeof node !== 'object') return; - - const cpeMatches = Array.isArray(node.cpeMatch) ? node.cpeMatch : []; - matches.push(...cpeMatches); - - const childNodes = Array.isArray(node.nodes) ? node.nodes : []; - childNodes.forEach((child) => collectCpeMatchesFromNode(child, matches)); -} - function getCpeMatchesFromHit(hit) { const cpeApplicability = hit?._source?.containers?.cna?.cpeApplicability; const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; @@ -552,7 +542,11 @@ function getCpeMatchesFromHit(hit) { applicabilityItems.forEach((applicability) => { const rootNodes = Array.isArray(applicability?.nodes) ? applicability.nodes : []; - rootNodes.forEach((node) => collectCpeMatchesFromNode(node, matches)); + rootNodes.forEach((node) => { + if (Array.isArray(node?.cpeMatch)) { + matches.push(...node.cpeMatch); + } + }); }); return matches; diff --git a/utils/cpe.test.js b/utils/cpe.test.js index 995dccb..6068cac 100644 --- a/utils/cpe.test.js +++ b/utils/cpe.test.js @@ -634,7 +634,7 @@ describe('CPE helpers', () => { ).toBe(true); }); - test('hitAppliesToCpeName searches nested cpeApplicability nodes', () => { + test('hitAppliesToCpeName evaluates direct entries independently of node operator and negate values', () => { const hit = { _source: { containers: { @@ -642,15 +642,18 @@ describe('CPE helpers', () => { cpeApplicability: { nodes: [ { - nodes: [ + operator: 'AND', + negate: true, + cpeMatch: [ { - cpeMatch: [ - { - criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', - versionStartIncluding: '2.9', - versionEndIncluding: '2.10' - } - ] + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true, + versionStartIncluding: '2.9', + versionEndIncluding: '2.10' + }, + { + criteria: 'cpe:2.3:a:other:component:*:*:*:*:*:*:*:*', + vulnerable: true } ] } @@ -661,11 +664,15 @@ describe('CPE helpers', () => { } }; - expect(cpe.hitAppliesToCpeName(hit, 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*')).toBe(true); + expect( + cpe.hitAppliesToCpeName(hit, 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*', undefined, { + requireVulnerable: true + }) + ).toBe(true); expect(cpe.hitAppliesToCpeName(hit, 'cpe:2.3:a:vendor:product:2.11:*:*:*:*:*:*:*')).toBe(false); }); - test('hitOverlapsVirtualMatchStringRange searches nested cpeApplicability nodes', () => { + test('hitOverlapsVirtualMatchStringRange evaluates direct entries independently of node metadata', () => { const hit = { _source: { containers: { @@ -673,15 +680,18 @@ describe('CPE helpers', () => { cpeApplicability: { nodes: [ { - nodes: [ + operator: 'OR', + negate: true, + cpeMatch: [ { - cpeMatch: [ - { - criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', - versionStartIncluding: '2.9', - versionEndIncluding: '2.10' - } - ] + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true, + versionStartIncluding: '2.9', + versionEndIncluding: '2.10' + }, + { + criteria: 'cpe:2.3:a:other:component:*:*:*:*:*:*:*:*', + vulnerable: true } ] } @@ -711,4 +721,37 @@ describe('CPE helpers', () => { }) ).toBe(false); }); + + test('getCpeMatchesFromHit ignores nested-only applicability entries', () => { + const hit = { + _source: { + containers: { + cna: { + cpeApplicability: { + nodes: [ + { + operator: 'OR', + negate: false, + nodes: [ + { + cpeMatch: [ + { + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true + } + ] + } + ] + } + ] + } + } + } + } + }; + + expect(cpe.getCpeMatchesFromHit(hit)).toStrictEqual([]); + expect(cpe.hitAppliesToCpeName(hit, 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*')).toBe(false); + expect(cpe.hitMatchesVirtualMatchString(hit, 'cpe:2.3:a:vendor:product')).toBe(false); + }); }); diff --git a/utils/searchCandidateProcessor.test.js b/utils/searchCandidateProcessor.test.js index 8c39b81..6048b35 100644 --- a/utils/searchCandidateProcessor.test.js +++ b/utils/searchCandidateProcessor.test.js @@ -42,6 +42,27 @@ function candidateResponse(hits) { }; } +function nestedOnlyCpeCandidateHit() { + const candidate = cpeCandidateHit(); + candidate._source.containers.cna.cpeApplicability[0].nodes = [ + { + operator: 'OR', + negate: false, + nodes: [ + { + cpeMatch: [ + { + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true + } + ] + } + ] + } + ]; + return candidate; +} + describe('searchCandidateProcessor', () => { test('validates and filters candidates in the same processing pass', async () => { const matching = cpeCandidateHit({ @@ -117,6 +138,7 @@ describe('searchCandidateProcessor', () => { } } ], + ['nested-only CPE applicability entry', nestedOnlyCpeCandidateHit()], ['missing CPE criteria', cpeCandidateHit({ criteria: undefined })], ['invalid CPE criteria', cpeCandidateHit({ criteria: 'not-a-cpe-name' })], ['missing CPE vulnerability flag', cpeCandidateHit({ vulnerable: undefined })], From 6e2b477cd0e3884e71ec9c1c7d81cf351016b746 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 17:45:35 -0400 Subject: [PATCH 15/33] Centralize runtime configuration validation --- .env.example | 2 + Dockerfile | 6 +- Dockerfile.local | 6 +- README.md | 37 +++ config/default.jsonc | 2 + config/runtime-config.test.js | 35 ++- config/test.jsonc | 10 +- server.js | 81 ++++++- server.test.js | 38 ++- utils/featureFlags.js | 25 +- utils/featureFlags.test.js | 13 +- utils/requestLogging.js | 45 +--- utils/requestLogging.test.js | 13 +- utils/runtimeConfig.js | 404 ++++++++++++++++++++++++++++++++ utils/runtimeConfig.test.js | 250 ++++++++++++++++++++ utils/runtimeLifecycle.js | 13 +- utils/runtimeLifecycle.test.js | 15 +- utils/searchProvider.js | 113 ++------- utils/searchProvider.test.js | 31 +-- utils/searchRateLimiter.js | 59 +---- utils/searchRateLimiter.test.js | 7 +- 21 files changed, 914 insertions(+), 291 deletions(-) create mode 100644 utils/runtimeConfig.js create mode 100644 utils/runtimeConfig.test.js diff --git a/.env.example b/.env.example index 795db04..2067d05 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,6 @@ +NODE_CONFIG_ENV=devel PORT=3000 +BASE_URL= OpenSearchCveIndex=cve-index-local OpenSearchAllowUnknownSslCerts=false OpenSearchDomainEndpoint= diff --git a/Dockerfile b/Dockerfile index 8914334..10d044f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot FROM node:${NODE_VERSION} AS deps -ENV NODE_ENV=production +ENV NODE_ENV=production \ + NODE_CONFIG_ENV=prod WORKDIR /usr/src/app @@ -18,7 +19,8 @@ RUN npm ci --omit=dev \ FROM ${DISTROLESS_NODE_IMAGE} AS runtime -ENV NODE_ENV=production +ENV NODE_ENV=production \ + NODE_CONFIG_ENV=prod WORKDIR /usr/src/app diff --git a/Dockerfile.local b/Dockerfile.local index 65345fb..a0977de 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -5,7 +5,8 @@ ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot FROM node:${NODE_VERSION} AS deps -ENV NODE_ENV=production +ENV NODE_ENV=production \ + NODE_CONFIG_ENV=prod WORKDIR /usr/src/app @@ -23,7 +24,8 @@ RUN --mount=type=secret,id=npm_ca,required=false \ FROM ${DISTROLESS_NODE_IMAGE} AS runtime -ENV NODE_ENV=production +ENV NODE_ENV=production \ + NODE_CONFIG_ENV=prod WORKDIR /usr/src/app diff --git a/README.md b/README.md index 47d6537..fd109f1 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ Rename `.env.example` to `.env` and replace or check the following values. +- `NODE_CONFIG_ENV` - node-config profile name; use `devel` locally and `prod` in production images - `PORT` - port used for the search API, defaulted to 3000 for development +- `BASE_URL` - optional absolute public URL used by local runtime helpers; credentials are not permitted - `OpenSearchCveIndex` - existing CVE index name in your OpenSearch deployment - `OpenSearchAllowUnknownSslCerts` - set to `true` only for local development against self-signed certificates - `OpenSearchDomainEndpoint` - absolute URL to an existing OpenSearch deployment. This API does not create or manage the OpenSearch index or mapping. @@ -32,6 +34,41 @@ Rename `.env.example` to `.env` and replace or check the following values. mv .env.example .env ``` +### Runtime Configuration Validation + +The API loads `.env` before `cve-core`, resolves runtime settings once, and uses one immutable configuration snapshot for the life of the process. Explicitly configured Boolean values must be `true` or `false`, without regard to case. Malformed or out-of-range values stop startup instead of silently falling back to defaults. + +The following numeric bounds are enforced: + +| Setting | Accepted range | +| ----------------------------------------- | -------------------------------------------------------------------: | +| `PORT` | 1 through 65,535 | +| `SearchPostFilterCacheTtlMs` | 0 through 86,400,000 | +| `SearchPostFilterCacheMaxEntries` | 0 through 10,000 | +| `SearchPostFilterColdSearchConcurrency` | 1 through 100 | +| `SearchPostFilterColdSearchMaxQueueDepth` | 0 through 10,000 | +| `SearchPostFilterColdSearchMaxWaitMs` | 1 through 300,000 | +| `SearchInFlightMaxWaitersPerOperation` | 1 through 10,000 | +| `SearchPostFilterEvaluationChunkSize` | 1 through 10,000 | +| `SearchOpenSearchRequestTimeoutMs` | 1 through 300,000 | +| `SearchRequestDeadlineMs` | 1 through 600,000 | +| `SearchReadinessTimeoutMs` | 1 through 60,000 | +| `RequestLogSuccessSampleRate` | 0 through 1 | +| `ShutdownTimeoutMs` | 1 through 900,000 | +| `SearchRateLimitWindowMs` | 1 through 86,400,000 | +| `SearchRateLimitMaxRequests` | 0 through 10,000,000; must be positive when rate limiting is enabled | + +Cache TTL, cache entry count, and cold-search queue depth accept `0` because those settings have defined disable or no-wait semantics. The timeout settings must also satisfy: + +```text +SearchReadinessTimeoutMs <= SearchOpenSearchRequestTimeoutMs < SearchRequestDeadlineMs < ShutdownTimeoutMs +SearchPostFilterColdSearchMaxWaitMs < SearchRequestDeadlineMs +``` + +`OpenSearchDomainEndpoint` must be an absolute HTTP or HTTPS URL, and `OpenSearchCveIndex` must be non-empty. `BASE_URL`, when supplied, must also be an absolute HTTP or HTTPS URL and cannot contain credentials. OpenSearch connectivity is not required at startup; `/api/ready-check` remains responsible for dependency readiness. + +Production images set both `NODE_ENV=production` and `NODE_CONFIG_ENV=prod`, matching `config/prod.jsonc`. Successful startup emits a sanitized `runtime.lifecycle` JSON event with effective safety settings. Startup failures identify only the invalid configuration path and never log configured values, URLs, index names, credentials, or exception messages. + ## Docker API Only Use `docker-compose.yaml` when OpenSearch 3.5 is already running on the external Docker network `opensearch-35_opensearch-net`. diff --git a/config/default.jsonc b/config/default.jsonc index 33561f2..bd81068 100644 --- a/config/default.jsonc +++ b/config/default.jsonc @@ -5,6 +5,8 @@ // - Each configuration is mapped using custom-environment-variables.jsonc to enable environment varialbe overrides. // - For more information, see cve-core/src/adapters/config/AppConfig.ts // NOTE for consistency, all values need to be strings for proper type when using .env overrides + // Explicit invalid values fail application startup. See README.md for accepted ranges and + // required timeout relationships. "appConfig": { // constants for search capability "search": { diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index 4c969ec..ea646dc 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -12,6 +12,7 @@ describe('runtime container and compose configuration', () => { expect(dockerfile).toContain('ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot'); expect(dockerfile).toContain('FROM node:${NODE_VERSION} AS deps'); expect(dockerfile).toContain('FROM ${DISTROLESS_NODE_IMAGE} AS runtime'); + expect(dockerfile).toContain('NODE_CONFIG_ENV=prod'); expect(dockerfile).toMatch(/^RUN npm ci --omit=dev \\/m); expect(dockerfile).toMatch(/^COPY --chown=65532:65532 config \.\/config\//m); expect(dockerfile).toMatch(/^COPY --chown=65532:65532 controllers \.\/controllers\//m); @@ -36,6 +37,7 @@ describe('runtime container and compose configuration', () => { expect(dockerfile).toContain('ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot'); expect(dockerfile).toContain('FROM node:${NODE_VERSION} AS deps'); expect(dockerfile).toContain('FROM ${DISTROLESS_NODE_IMAGE} AS runtime'); + expect(dockerfile).toContain('NODE_CONFIG_ENV=prod'); expect(dockerfile).toMatch(/^RUN --mount=type=secret,id=npm_ca,required=false/m); expect(dockerfile).toContain('if [ -s /run/secrets/npm_ca ]; then'); expect(dockerfile).toContain('NPM_CONFIG_CAFILE=/run/secrets/npm_ca npm ci --omit=dev'); @@ -61,6 +63,12 @@ describe('runtime container and compose configuration', () => { expect(localDockerfile).not.toContain('CMD ["./docker-entrypoint.sh"]'); }); + test('local environment example selects the development node-config profile', () => { + const environmentExample = readProjectFile('.env.example'); + + expect(environmentExample).toContain('NODE_CONFIG_ENV=devel'); + }); + test('Docker build context excludes tests and local report artifacts from runtime directory copies', () => { const dockerignore = readProjectFile('.dockerignore'); @@ -140,16 +148,19 @@ describe('runtime container and compose configuration', () => { } ); - test.each([['README.md'], ['config/default.jsonc'], ['config/devel.jsonc'], ['config/prod.jsonc']])( - '%s does not include credential-bearing OpenSearch endpoint examples', - (filePath) => { - const content = readProjectFile(filePath); - const defaultCredentialPair = ['admin', 'admin'].join(':'); - const placeholderCredentialPair = ['', ''].join(':'); - - expect(content).not.toContain(defaultCredentialPair); - expect(content).not.toContain(placeholderCredentialPair); - expect(content).not.toMatch(/https?:\/\/[^/\s]+:[^@\s]+@/); - } - ); + test.each([ + ['README.md'], + ['config/default.jsonc'], + ['config/devel.jsonc'], + ['config/prod.jsonc'], + ['config/test.jsonc'] + ])('%s does not include credential-bearing OpenSearch endpoint examples', (filePath) => { + const content = readProjectFile(filePath); + const defaultCredentialPair = ['admin', 'admin'].join(':'); + const placeholderCredentialPair = ['', ''].join(':'); + + expect(content).not.toContain(defaultCredentialPair); + expect(content).not.toContain(placeholderCredentialPair); + expect(content).not.toMatch(/https?:\/\/[^/\s]+:[^@\s]+@/); + }); }); diff --git a/config/test.jsonc b/config/test.jsonc index 3cde5d6..d522127 100644 --- a/config/test.jsonc +++ b/config/test.jsonc @@ -1,3 +1,11 @@ { - // Tests inherit default configuration. This file exists so node-config recognizes NODE_ENV=test. + // Tests inherit default configuration. These non-routable values satisfy startup validation + // without requiring a live OpenSearch dependency. + "appConfig": { + "search": { + "providerEndpoint": "https://search.example.com", + "index": "test-index", + "allowUnknownSslCerts": "FALSE" + } + } } diff --git a/server.js b/server.js index 1340374..b3ae874 100644 --- a/server.js +++ b/server.js @@ -1,21 +1,23 @@ const dotenv = require('dotenv'); + +dotenv.config(); + const cors = require('cors'); const express = require('express'); const swaggerUi = require('swagger-ui-express'); const openApiSpecification = require('./api-docs/openapi.json'); const http = require('http'); const logger = require('morgan'); -const router = require('./routes/index'); const constants = require('./utils/constants'); const errors = require('./utils/errors'); const requestLogging = require('./utils/requestLogging'); +const runtimeConfig = require('./utils/runtimeConfig'); const runtimeLifecycle = require('./utils/runtimeLifecycle'); const searchProvider = require('./utils/searchProvider'); -dotenv.load(); - const error = new errors.errors(); const SHUTDOWN_PROBE_PATHS = new Set(['/api/health-check', '/api/ready-check']); +let router; const swaggerSetupOptions = { swaggerOptions: { @@ -30,7 +32,7 @@ const swaggerSetupOptions = { }; function createRequestLogger() { - if (process.env.NODE_ENV === 'test') { + if (runtimeConfig.getRuntimeConfig().server.nodeEnvironment === 'test') { return (_req, _res, next) => next(); } @@ -94,7 +96,16 @@ function handleUnexpectedError(err, req, res, next) { return res.status(500).json(error.internalServerError()); } +function getRouter() { + if (!router) { + router = require('./routes/index'); + } + + return router; +} + function createApp() { + runtimeConfig.validateRuntimeConfig(); const app = express(); app.use(requestLogging.assignRequestId); @@ -107,7 +118,7 @@ function createApp() { app.get('/api-docs/openapi.json', (req, res) => res.json(openApiSpecification)); app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(openApiSpecification, swaggerSetupOptions)); - app.use('/', router); + app.use('/', getRouter()); app.use(handleRequestError); app.use(handleUnexpectedError); @@ -115,20 +126,60 @@ function createApp() { } function getRuntimeConfig(port) { + const config = runtimeConfig.validateRuntimeConfig(); + const effectivePort = resolveListenPort(port, config.server.port); + return { - baseURL: process.env.BASE_URL || `http://127.0.0.1:${port}` + baseURL: config.server.baseURL || `http://127.0.0.1:${effectivePort}` }; } -function startServer(port = process.env.PORT || 3000) { +function resolveListenPort(port, defaultPort) { + if (port === undefined) return defaultPort; + + const parsed = Number(port); + if (!Number.isInteger(parsed) || parsed < 0 || parsed > runtimeConfig.RUNTIME_CONFIG_LIMITS.port.max) { + throw new runtimeConfig.RuntimeConfigurationError( + 'PORT', + `must be between 0 and ${runtimeConfig.RUNTIME_CONFIG_LIMITS.port.max}` + ); + } + + return parsed; +} + +function startServer(port) { + const config = runtimeConfig.validateRuntimeConfig(); + const effectivePort = resolveListenPort(port, config.server.port); const app = createApp(); - const config = getRuntimeConfig(port); - return http.createServer(app).listen(port, () => { - console.log(`Listening on ${config.baseURL}`); + return http.createServer(app).listen(effectivePort, () => { + const sanitizedConfig = runtimeConfig.sanitizeRuntimeConfig(config); + logRuntimeEvent(console.log, { + event: 'runtime.lifecycle', + outcome: 'started', + ...sanitizedConfig, + server: { + ...sanitizedConfig.server, + port: effectivePort + } + }); }); } +function handleStartupError(err, log = console.error) { + const configurationError = runtimeConfig.isRuntimeConfigurationError(err); + const event = { + event: 'runtime.lifecycle', + outcome: 'startup_failed', + errorCode: configurationError ? err.code : 'STARTUP_ERROR' + }; + if (configurationError) event.configPath = err.configPath; + + logRuntimeEvent(log, event); + return event; +} + function logRuntimeEvent(log, event) { log(JSON.stringify(event)); } @@ -257,8 +308,13 @@ function installShutdownHandlers(server, { processRef = process, shutdownOptions } if (require.main === module) { - const server = startServer(); - installShutdownHandlers(server); + try { + const server = startServer(); + installShutdownHandlers(server); + } catch (err) { + handleStartupError(err); + process.exitCode = 1; + } } module.exports = { @@ -268,6 +324,7 @@ module.exports = { installShutdownHandlers, getRedactedRequestPath: requestLogging.getRedactedRequestPath, getRuntimeConfig, + handleStartupError, rejectRequestsDuringShutdown, shutdownServer, startServer, diff --git a/server.test.js b/server.test.js index 61248a8..d2aaf78 100644 --- a/server.test.js +++ b/server.test.js @@ -36,12 +36,14 @@ const { createRequestLogger, getRedactedRequestPath, getRuntimeConfig, + handleStartupError, handleUnexpectedError, installShutdownHandlers, shutdownServer, startServer, swaggerSetupOptions } = require('./server'); +const runtimeConfig = require('./utils/runtimeConfig'); const runtimeLifecycle = require('./utils/runtimeLifecycle'); function restoreEnv(name, value) { @@ -63,6 +65,7 @@ describe('server application bootstrap', () => { restoreEnv('NODE_ENV', originalNodeEnv); restoreEnv('PORT', originalPort); runtimeLifecycle.resetForTests(); + runtimeConfig.resetForTests(); jest.useRealTimers(); jest.restoreAllMocks(); jest.clearAllMocks(); @@ -324,7 +327,40 @@ describe('server application bootstrap', () => { jest.spyOn(http, 'createServer').mockReturnValue(fakeServer); expect(startServer('4567')).toBe(fakeServer); - expect(fakeServer.listen).toHaveBeenCalledWith('4567', expect.any(Function)); + expect(fakeServer.listen).toHaveBeenCalledWith(4567, expect.any(Function)); + }); + + test('startServer rejects invalid configuration before creating an HTTP server', () => { + process.env.PORT = 'invalid'; + runtimeConfig.resetForTests(); + const createServer = jest.spyOn(http, 'createServer'); + + expect(() => startServer()).toThrow('PORT must be an integer'); + expect(createServer).not.toHaveBeenCalled(); + }); + + test('handleStartupError logs only a sanitized configuration failure', () => { + const log = jest.fn(); + const thrownError = new runtimeConfig.RuntimeConfigurationError( + 'search.providerEndpoint', + 'must be configured' + ); + + expect(handleStartupError(thrownError, log)).toStrictEqual({ + event: 'runtime.lifecycle', + outcome: 'startup_failed', + errorCode: 'RUNTIME_CONFIGURATION', + configPath: 'search.providerEndpoint' + }); + expect(log).toHaveBeenCalledWith( + JSON.stringify({ + event: 'runtime.lifecycle', + outcome: 'startup_failed', + errorCode: 'RUNTIME_CONFIGURATION', + configPath: 'search.providerEndpoint' + }) + ); + expect(JSON.stringify(log.mock.calls)).not.toContain('must be configured'); }); test('shutdownServer marks readiness unavailable and drains the HTTP server', async () => { diff --git a/utils/featureFlags.js b/utils/featureFlags.js index 61ff67f..29626de 100644 --- a/utils/featureFlags.js +++ b/utils/featureFlags.js @@ -1,25 +1,14 @@ -const { AppConfig } = require('cve-core'); - -const DEFAULT_WEB_SEARCH_ENABLED = true; - -function normalizeBoolean(value, defaultValue) { - if (value === undefined || value === null || value === '') return defaultValue; - if (typeof value === 'boolean') return value; - if (typeof value === 'number') return value !== 0; - - const normalized = String(value).trim().toLowerCase(); - if (['true', '1', 'yes', 'y', 'on'].includes(normalized)) return true; - if (['false', '0', 'no', 'n', 'off'].includes(normalized)) return false; - - return defaultValue; -} +const runtimeConfig = require('./runtimeConfig'); function isWebSearchEnabled() { - if (!AppConfig.has('features.webSearchEnabled')) return DEFAULT_WEB_SEARCH_ENABLED; + return runtimeConfig.getRuntimeConfig().features.webSearchEnabled; +} - return normalizeBoolean(AppConfig.get('features.webSearchEnabled'), false); +function resetForTests() { + runtimeConfig.resetForTests(); } module.exports = { - isWebSearchEnabled + isWebSearchEnabled, + resetForTests }; diff --git a/utils/featureFlags.test.js b/utils/featureFlags.test.js index 3d47d75..8114b87 100644 --- a/utils/featureFlags.test.js +++ b/utils/featureFlags.test.js @@ -12,6 +12,7 @@ const featureFlags = require('./featureFlags'); describe('featureFlags', () => { beforeEach(() => { + featureFlags.resetForTests(); mockAppConfigHas.mockReset(); mockAppConfigGet.mockReset(); mockAppConfigHas.mockReturnValue(false); @@ -24,21 +25,19 @@ describe('featureFlags', () => { test.each([ ['TRUE', true], ['false', false], - ['1', true], - ['0', false], [true, true], [false, false] ])('normalizes a configured webSearch value of %p', (configuredValue, expected) => { - mockAppConfigHas.mockReturnValue(true); + mockAppConfigHas.mockImplementation((path) => path === 'features.webSearchEnabled'); mockAppConfigGet.mockReturnValue(configuredValue); expect(featureFlags.isWebSearchEnabled()).toBe(expected); }); - test('fails closed when an explicitly configured webSearch value is invalid', () => { - mockAppConfigHas.mockReturnValue(true); - mockAppConfigGet.mockReturnValue('invalid'); + test.each(['invalid', '1', '0'])('rejects an explicitly configured webSearch value of %p', (value) => { + mockAppConfigHas.mockImplementation((path) => path === 'features.webSearchEnabled'); + mockAppConfigGet.mockReturnValue(value); - expect(featureFlags.isWebSearchEnabled()).toBe(false); + expect(() => featureFlags.isWebSearchEnabled()).toThrow('features.webSearchEnabled'); }); }); diff --git a/utils/requestLogging.js b/utils/requestLogging.js index 57e7248..e47f60c 100644 --- a/utils/requestLogging.js +++ b/utils/requestLogging.js @@ -1,49 +1,11 @@ const { randomUUID } = require('crypto'); -const { AppConfig } = require('cve-core'); +const runtimeConfig = require('./runtimeConfig'); -const DEFAULT_SUCCESS_SAMPLE_RATE = 0.1; -const DEFAULT_LOG_HEALTH_CHECKS = false; const REQUEST_ID_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/; const HEALTH_CHECK_PATHS = new Set(['/api/health-check', '/api/ready-check']); -let requestLoggingConfig; - -function normalizeBoolean(value, defaultValue = false) { - if (value === undefined || value === null || value === '') return defaultValue; - if (typeof value === 'boolean') return value; - if (typeof value === 'number') return value !== 0; - - const normalized = String(value).trim().toLowerCase(); - if (['true', '1', 'yes', 'y', 'on'].includes(normalized)) return true; - if (['false', '0', 'no', 'n', 'off'].includes(normalized)) return false; - - return defaultValue; -} - -function readSuccessSampleRate() { - if (!AppConfig.has('requestLogging.successSampleRate')) return DEFAULT_SUCCESS_SAMPLE_RATE; - - const value = Number(AppConfig.get('requestLogging.successSampleRate')); - if (Number.isFinite(value) && value >= 0 && value <= 1) return value; - - return DEFAULT_SUCCESS_SAMPLE_RATE; -} - -function readLogHealthChecks() { - if (!AppConfig.has('requestLogging.healthChecks')) return DEFAULT_LOG_HEALTH_CHECKS; - - return normalizeBoolean(AppConfig.get('requestLogging.healthChecks'), DEFAULT_LOG_HEALTH_CHECKS); -} - function getRequestLoggingConfig() { - if (!requestLoggingConfig) { - requestLoggingConfig = Object.freeze({ - successSampleRate: readSuccessSampleRate(), - healthChecks: readLogHealthChecks() - }); - } - - return requestLoggingConfig; + return runtimeConfig.getRuntimeConfig().requestLogging; } function getSuccessSampleRate() { @@ -83,7 +45,7 @@ function shouldSkipAccessLog(req, res, random = Math.random) { } function resetForTests() { - requestLoggingConfig = undefined; + runtimeConfig.resetForTests(); } module.exports = { @@ -91,7 +53,6 @@ module.exports = { getRedactedRequestPath, getRequestLoggingConfig, getSuccessSampleRate, - normalizeBoolean, resetForTests, resolveRequestId, shouldLogHealthChecks, diff --git a/utils/requestLogging.test.js b/utils/requestLogging.test.js index 1cae11e..77badb8 100644 --- a/utils/requestLogging.test.js +++ b/utils/requestLogging.test.js @@ -120,12 +120,14 @@ describe('requestLogging', () => { expect(requestLogging.shouldSkipAccessLog(req, { statusCode: 500 }, () => 1)).toBe(false); }); - test('request logging settings fall back when config values are invalid', () => { - mockAppConfigHas.mockReturnValue(true); + test.each([ + ['requestLogging.successSampleRate', () => requestLogging.getSuccessSampleRate()], + ['requestLogging.healthChecks', () => requestLogging.shouldLogHealthChecks()] + ])('rejects an invalid explicit %s value', (path, readValue) => { + mockAppConfigHas.mockImplementation((candidatePath) => candidatePath === path); mockAppConfigGet.mockReturnValue('invalid'); - expect(requestLogging.getSuccessSampleRate()).toBe(0.1); - expect(requestLogging.shouldLogHealthChecks()).toBe(false); + expect(readValue).toThrow(path); }); test('successful health checks can be enabled through config', () => { @@ -151,6 +153,7 @@ describe('requestLogging', () => { expect(first).toBe(second); expect(Object.isFrozen(first)).toBe(true); - expect(mockAppConfigHas).toHaveBeenCalledTimes(2); + expect(mockAppConfigHas).toHaveBeenCalledWith('requestLogging.successSampleRate'); + expect(mockAppConfigHas).toHaveBeenCalledWith('requestLogging.healthChecks'); }); }); diff --git a/utils/runtimeConfig.js b/utils/runtimeConfig.js new file mode 100644 index 0000000..987aca2 --- /dev/null +++ b/utils/runtimeConfig.js @@ -0,0 +1,404 @@ +const { AppConfig, SearchProviderSpec } = require('cve-core'); +const constants = require('./constants'); + +const DEFAULT_PORT = 3000; +const DEFAULT_REQUEST_LOG_SUCCESS_SAMPLE_RATE = 0.1; +const DEFAULT_REQUEST_LOG_HEALTH_CHECKS = false; +const DEFAULT_WEB_SEARCH_ENABLED = true; +const DEFAULT_SEARCH_RATE_LIMIT_ENABLED = false; +const DEFAULT_SEARCH_RATE_LIMIT_WINDOW_MS = 60 * 1000; +const DEFAULT_SEARCH_RATE_LIMIT_MAX_REQUESTS = 0; + +const RUNTIME_CONFIG_LIMITS = Object.freeze({ + port: Object.freeze({ min: 1, max: 65535 }), + postFilterCacheTtlMs: Object.freeze({ min: 0, max: 24 * 60 * 60 * 1000 }), + postFilterCacheMaxEntries: Object.freeze({ min: 0, max: 10000 }), + postFilterColdSearchConcurrency: Object.freeze({ min: 1, max: 100 }), + postFilterColdSearchMaxQueueDepth: Object.freeze({ min: 0, max: 10000 }), + postFilterColdSearchMaxWaitMs: Object.freeze({ min: 1, max: 5 * 60 * 1000 }), + inFlightMaxWaitersPerOperation: Object.freeze({ min: 1, max: 10000 }), + postFilterEvaluationChunkSize: Object.freeze({ min: 1, max: 10000 }), + openSearchRequestTimeoutMs: Object.freeze({ min: 1, max: 5 * 60 * 1000 }), + requestDeadlineMs: Object.freeze({ min: 1, max: 10 * 60 * 1000 }), + readinessTimeoutMs: Object.freeze({ min: 1, max: 60 * 1000 }), + shutdownTimeoutMs: Object.freeze({ min: 1, max: 15 * 60 * 1000 }), + searchRateLimitWindowMs: Object.freeze({ min: 1, max: 24 * 60 * 60 * 1000 }), + searchRateLimitMaxRequests: Object.freeze({ min: 0, max: 10000000 }) +}); + +class RuntimeConfigurationError extends Error { + constructor(configPath, requirement) { + super(`${configPath} ${requirement}.`); + this.name = 'RuntimeConfigurationError'; + this.code = 'RUNTIME_CONFIGURATION'; + this.configPath = configPath; + } +} + +let resolvedRuntimeConfig; + +function deepFreeze(value) { + if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value; + + Object.values(value).forEach(deepFreeze); + return Object.freeze(value); +} + +function hasConfigValue(appConfig, path) { + return typeof appConfig?.has === 'function' && appConfig.has(path); +} + +function readConfigValue(appConfig, path, defaultValue) { + return hasConfigValue(appConfig, path) ? appConfig.get(path) : defaultValue; +} + +function parseBoolean(path, value) { + if (typeof value === 'boolean') return value; + + const normalized = typeof value === 'string' ? value.trim().toLowerCase() : ''; + if (normalized === 'true') return true; + if (normalized === 'false') return false; + + throw new RuntimeConfigurationError(path, 'must be true or false'); +} + +function parseInteger(path, value, limits) { + const parsed = Number(value); + if (!Number.isInteger(parsed)) { + throw new RuntimeConfigurationError(path, 'must be an integer'); + } + if (parsed < limits.min || parsed > limits.max) { + throw new RuntimeConfigurationError(path, `must be between ${limits.min} and ${limits.max}`); + } + + return parsed; +} + +function parseNumber(path, value, { min, max }) { + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + throw new RuntimeConfigurationError(path, 'must be a finite number'); + } + if (parsed < min || parsed > max) { + throw new RuntimeConfigurationError(path, `must be between ${min} and ${max}`); + } + + return parsed; +} + +function parseOptionalString(path, value, { maxLength } = {}) { + if (value === undefined || value === null || value === '') return ''; + if (typeof value !== 'string' || value.trim() === '') { + throw new RuntimeConfigurationError(path, 'must be a non-empty string'); + } + + const normalized = value.trim(); + if (maxLength && normalized.length > maxLength) { + throw new RuntimeConfigurationError(path, `must not exceed ${maxLength} characters`); + } + + return normalized; +} + +function parseHttpUrl(path, value, { required = false, allowCredentials = true } = {}) { + if (!value) { + if (required) throw new RuntimeConfigurationError(path, 'must be configured'); + return ''; + } + + let parsed; + try { + parsed = new URL(value); + } catch (_err) { + throw new RuntimeConfigurationError(path, 'must be an absolute HTTP or HTTPS URL'); + } + + if (!['http:', 'https:'].includes(parsed.protocol)) { + throw new RuntimeConfigurationError(path, 'must be an absolute HTTP or HTTPS URL'); + } + if (!allowCredentials && (parsed.username || parsed.password)) { + throw new RuntimeConfigurationError(path, 'must not contain credentials'); + } + + return value; +} + +function getDefaultSearchProviderSpec(searchProviderSpec) { + if (typeof searchProviderSpec?.getDefaultSearchProviderSpec !== 'function') return {}; + + return searchProviderSpec.getDefaultSearchProviderSpec() ?? {}; +} + +function validateRelationships(config) { + const search = config.search; + + if (search.readinessTimeoutMs > search.openSearchRequestTimeoutMs) { + throw new RuntimeConfigurationError( + 'search.readinessTimeoutMs', + 'must be less than or equal to search.openSearchRequestTimeoutMs' + ); + } + if (search.openSearchRequestTimeoutMs >= search.requestDeadlineMs) { + throw new RuntimeConfigurationError( + 'search.openSearchRequestTimeoutMs', + 'must be less than search.requestDeadlineMs' + ); + } + if (search.postFilterColdSearchMaxWaitMs >= search.requestDeadlineMs) { + throw new RuntimeConfigurationError( + 'search.postFilterColdSearchMaxWaitMs', + 'must be less than search.requestDeadlineMs' + ); + } + if (config.runtime.shutdownTimeoutMs <= search.requestDeadlineMs) { + throw new RuntimeConfigurationError( + 'runtime.shutdownTimeoutMs', + 'must be greater than search.requestDeadlineMs' + ); + } + if (config.rateLimiting.search.enabled && config.rateLimiting.search.maxRequests < 1) { + throw new RuntimeConfigurationError( + 'rateLimiting.search.maxRequests', + 'must be a positive integer when rate limiting is enabled' + ); + } +} + +function resolveRuntimeConfig({ + appConfig = AppConfig, + env = process.env, + searchProviderSpec = SearchProviderSpec +} = {}) { + const defaultSearchProvider = getDefaultSearchProviderSpec(searchProviderSpec); + const providerEndpoint = parseOptionalString( + 'search.providerEndpoint', + readConfigValue(appConfig, 'search.providerEndpoint', defaultSearchProvider.providerEndpoint), + { maxLength: 4096 } + ); + const index = parseOptionalString( + 'search.index', + readConfigValue(appConfig, 'search.index', defaultSearchProvider.index), + { maxLength: 255 } + ); + const baseURL = parseOptionalString('BASE_URL', env.BASE_URL, { maxLength: 4096 }); + + const config = { + server: { + port: parseInteger('PORT', env.PORT ?? DEFAULT_PORT, RUNTIME_CONFIG_LIMITS.port), + baseURL: parseHttpUrl('BASE_URL', baseURL, { allowCredentials: false }), + nodeEnvironment: String(env.NODE_ENV || 'development'), + configEnvironment: String(env.NODE_CONFIG_ENV || env.NODE_ENV || 'default') + }, + search: { + provider: { + providerEndpoint, + index, + allowUnknownSslCerts: parseBoolean( + 'search.allowUnknownSslCerts', + readConfigValue( + appConfig, + 'search.allowUnknownSslCerts', + defaultSearchProvider.allowUnknownSslCerts ?? false + ) + ) + }, + logPerformanceTimings: parseBoolean( + 'search.logPerformanceTimings', + readConfigValue(appConfig, 'search.logPerformanceTimings', false) + ), + postFilterCacheTtlMs: parseInteger( + 'search.postFilterCacheTtlMs', + readConfigValue(appConfig, 'search.postFilterCacheTtlMs', constants.SEARCH_POST_FILTER_CACHE_TTL_MS), + RUNTIME_CONFIG_LIMITS.postFilterCacheTtlMs + ), + postFilterCacheMaxEntries: parseInteger( + 'search.postFilterCacheMaxEntries', + readConfigValue( + appConfig, + 'search.postFilterCacheMaxEntries', + constants.SEARCH_POST_FILTER_CACHE_MAX_ENTRIES + ), + RUNTIME_CONFIG_LIMITS.postFilterCacheMaxEntries + ), + postFilterColdSearchConcurrency: parseInteger( + 'search.postFilterColdSearchConcurrency', + readConfigValue( + appConfig, + 'search.postFilterColdSearchConcurrency', + constants.SEARCH_POST_FILTER_COLD_SEARCH_CONCURRENCY + ), + RUNTIME_CONFIG_LIMITS.postFilterColdSearchConcurrency + ), + postFilterColdSearchMaxQueueDepth: parseInteger( + 'search.postFilterColdSearchMaxQueueDepth', + readConfigValue( + appConfig, + 'search.postFilterColdSearchMaxQueueDepth', + constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_QUEUE_DEPTH + ), + RUNTIME_CONFIG_LIMITS.postFilterColdSearchMaxQueueDepth + ), + postFilterColdSearchMaxWaitMs: parseInteger( + 'search.postFilterColdSearchMaxWaitMs', + readConfigValue( + appConfig, + 'search.postFilterColdSearchMaxWaitMs', + constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_WAIT_MS + ), + RUNTIME_CONFIG_LIMITS.postFilterColdSearchMaxWaitMs + ), + inFlightMaxWaitersPerOperation: parseInteger( + 'search.inFlightMaxWaitersPerOperation', + readConfigValue( + appConfig, + 'search.inFlightMaxWaitersPerOperation', + constants.SEARCH_IN_FLIGHT_MAX_WAITERS_PER_OPERATION + ), + RUNTIME_CONFIG_LIMITS.inFlightMaxWaitersPerOperation + ), + postFilterEvaluationChunkSize: parseInteger( + 'search.postFilterEvaluationChunkSize', + readConfigValue( + appConfig, + 'search.postFilterEvaluationChunkSize', + constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE + ), + RUNTIME_CONFIG_LIMITS.postFilterEvaluationChunkSize + ), + openSearchRequestTimeoutMs: parseInteger( + 'search.openSearchRequestTimeoutMs', + readConfigValue( + appConfig, + 'search.openSearchRequestTimeoutMs', + constants.SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS + ), + RUNTIME_CONFIG_LIMITS.openSearchRequestTimeoutMs + ), + requestDeadlineMs: parseInteger( + 'search.requestDeadlineMs', + readConfigValue(appConfig, 'search.requestDeadlineMs', constants.SEARCH_REQUEST_DEADLINE_MS), + RUNTIME_CONFIG_LIMITS.requestDeadlineMs + ), + readinessTimeoutMs: parseInteger( + 'search.readinessTimeoutMs', + readConfigValue(appConfig, 'search.readinessTimeoutMs', constants.SEARCH_READINESS_TIMEOUT_MS), + RUNTIME_CONFIG_LIMITS.readinessTimeoutMs + ) + }, + requestLogging: { + successSampleRate: parseNumber( + 'requestLogging.successSampleRate', + readConfigValue(appConfig, 'requestLogging.successSampleRate', DEFAULT_REQUEST_LOG_SUCCESS_SAMPLE_RATE), + { min: 0, max: 1 } + ), + healthChecks: parseBoolean( + 'requestLogging.healthChecks', + readConfigValue(appConfig, 'requestLogging.healthChecks', DEFAULT_REQUEST_LOG_HEALTH_CHECKS) + ) + }, + runtime: { + shutdownTimeoutMs: parseInteger( + 'runtime.shutdownTimeoutMs', + readConfigValue(appConfig, 'runtime.shutdownTimeoutMs', constants.RUNTIME_SHUTDOWN_TIMEOUT_MS), + RUNTIME_CONFIG_LIMITS.shutdownTimeoutMs + ) + }, + features: { + webSearchEnabled: parseBoolean( + 'features.webSearchEnabled', + readConfigValue(appConfig, 'features.webSearchEnabled', DEFAULT_WEB_SEARCH_ENABLED) + ) + }, + rateLimiting: { + search: { + enabled: parseBoolean( + 'rateLimiting.search.enabled', + readConfigValue(appConfig, 'rateLimiting.search.enabled', DEFAULT_SEARCH_RATE_LIMIT_ENABLED) + ), + windowMs: parseInteger( + 'rateLimiting.search.windowMs', + readConfigValue(appConfig, 'rateLimiting.search.windowMs', DEFAULT_SEARCH_RATE_LIMIT_WINDOW_MS), + RUNTIME_CONFIG_LIMITS.searchRateLimitWindowMs + ), + maxRequests: parseInteger( + 'rateLimiting.search.maxRequests', + readConfigValue( + appConfig, + 'rateLimiting.search.maxRequests', + DEFAULT_SEARCH_RATE_LIMIT_MAX_REQUESTS + ), + RUNTIME_CONFIG_LIMITS.searchRateLimitMaxRequests + ) + } + } + }; + + validateRelationships(config); + return deepFreeze(config); +} + +function getRuntimeConfig() { + if (!resolvedRuntimeConfig) { + resolvedRuntimeConfig = resolveRuntimeConfig(); + } + + return resolvedRuntimeConfig; +} + +function validateRuntimeConfig(config = getRuntimeConfig()) { + const provider = config.search.provider; + parseHttpUrl('search.providerEndpoint', provider.providerEndpoint, { required: true }); + if (!provider.index) { + throw new RuntimeConfigurationError('search.index', 'must be configured'); + } + + validateRelationships(config); + return config; +} + +function sanitizeRuntimeConfig(config) { + return deepFreeze({ + server: { + port: config.server.port, + nodeEnvironment: config.server.nodeEnvironment, + configEnvironment: config.server.configEnvironment + }, + search: { + allowUnknownSslCerts: config.search.provider.allowUnknownSslCerts, + logPerformanceTimings: config.search.logPerformanceTimings, + postFilterCacheTtlMs: config.search.postFilterCacheTtlMs, + postFilterCacheMaxEntries: config.search.postFilterCacheMaxEntries, + postFilterColdSearchConcurrency: config.search.postFilterColdSearchConcurrency, + postFilterColdSearchMaxQueueDepth: config.search.postFilterColdSearchMaxQueueDepth, + postFilterColdSearchMaxWaitMs: config.search.postFilterColdSearchMaxWaitMs, + inFlightMaxWaitersPerOperation: config.search.inFlightMaxWaitersPerOperation, + postFilterEvaluationChunkSize: config.search.postFilterEvaluationChunkSize, + openSearchRequestTimeoutMs: config.search.openSearchRequestTimeoutMs, + requestDeadlineMs: config.search.requestDeadlineMs, + readinessTimeoutMs: config.search.readinessTimeoutMs + }, + requestLogging: config.requestLogging, + runtime: config.runtime, + features: config.features, + rateLimiting: config.rateLimiting + }); +} + +function isRuntimeConfigurationError(err) { + return err instanceof RuntimeConfigurationError || err?.code === 'RUNTIME_CONFIGURATION'; +} + +function resetForTests() { + resolvedRuntimeConfig = undefined; +} + +module.exports = { + getRuntimeConfig, + isRuntimeConfigurationError, + resetForTests, + resolveRuntimeConfig, + sanitizeRuntimeConfig, + validateRuntimeConfig, + RUNTIME_CONFIG_LIMITS, + RuntimeConfigurationError +}; diff --git a/utils/runtimeConfig.test.js b/utils/runtimeConfig.test.js new file mode 100644 index 0000000..638642b --- /dev/null +++ b/utils/runtimeConfig.test.js @@ -0,0 +1,250 @@ +const constants = require('./constants'); +const runtimeConfig = require('./runtimeConfig'); + +function createAppConfig(values = {}) { + return { + get: jest.fn((path) => values[path]), + has: jest.fn((path) => Object.hasOwn(values, path)) + }; +} + +function createSearchProviderSpec(overrides = {}) { + return { + getDefaultSearchProviderSpec: jest.fn(() => ({ + providerEndpoint: 'https://search.example', + index: 'test-index', + allowUnknownSslCerts: false, + ...overrides + })) + }; +} + +function resolve(values = {}, env = {}, providerOverrides = {}) { + return runtimeConfig.resolveRuntimeConfig({ + appConfig: createAppConfig(values), + env: { + NODE_ENV: 'test', + ...env + }, + searchProviderSpec: createSearchProviderSpec(providerOverrides) + }); +} + +describe('runtimeConfig', () => { + test('resolves and deeply freezes documented defaults', () => { + const config = resolve(); + + expect(config).toMatchObject({ + server: { + port: 3000, + baseURL: '', + nodeEnvironment: 'test', + configEnvironment: 'test' + }, + search: { + provider: { + providerEndpoint: 'https://search.example', + index: 'test-index', + allowUnknownSslCerts: false + }, + logPerformanceTimings: false, + postFilterCacheTtlMs: constants.SEARCH_POST_FILTER_CACHE_TTL_MS, + postFilterCacheMaxEntries: constants.SEARCH_POST_FILTER_CACHE_MAX_ENTRIES, + postFilterColdSearchConcurrency: constants.SEARCH_POST_FILTER_COLD_SEARCH_CONCURRENCY, + postFilterColdSearchMaxQueueDepth: constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_QUEUE_DEPTH, + postFilterColdSearchMaxWaitMs: constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_WAIT_MS, + inFlightMaxWaitersPerOperation: constants.SEARCH_IN_FLIGHT_MAX_WAITERS_PER_OPERATION, + postFilterEvaluationChunkSize: constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE, + openSearchRequestTimeoutMs: constants.SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS, + requestDeadlineMs: constants.SEARCH_REQUEST_DEADLINE_MS, + readinessTimeoutMs: constants.SEARCH_READINESS_TIMEOUT_MS + }, + requestLogging: { + successSampleRate: 0.1, + healthChecks: false + }, + runtime: { + shutdownTimeoutMs: constants.RUNTIME_SHUTDOWN_TIMEOUT_MS + }, + features: { + webSearchEnabled: true + }, + rateLimiting: { + search: { + enabled: false, + windowMs: 60000, + maxRequests: 0 + } + } + }); + expect(Object.isFrozen(config)).toBe(true); + expect(Object.isFrozen(config.search)).toBe(true); + expect(Object.isFrozen(config.search.provider)).toBe(true); + }); + + test('resolves typed environment and AppConfig overrides', () => { + const config = resolve( + { + 'search.allowUnknownSslCerts': 'TRUE', + 'search.logPerformanceTimings': 'true', + 'search.postFilterCacheTtlMs': '120000', + 'search.postFilterCacheMaxEntries': '250', + 'search.postFilterColdSearchConcurrency': '6', + 'search.postFilterColdSearchMaxQueueDepth': '50', + 'search.postFilterColdSearchMaxWaitMs': '15000', + 'search.inFlightMaxWaitersPerOperation': '125', + 'search.postFilterEvaluationChunkSize': '100', + 'search.openSearchRequestTimeoutMs': '25000', + 'search.requestDeadlineMs': '45000', + 'search.readinessTimeoutMs': '7000', + 'requestLogging.successSampleRate': '0.25', + 'requestLogging.healthChecks': 'TRUE', + 'runtime.shutdownTimeoutMs': '60000', + 'features.webSearchEnabled': 'FALSE', + 'rateLimiting.search.enabled': 'TRUE', + 'rateLimiting.search.windowMs': '30000', + 'rateLimiting.search.maxRequests': '25' + }, + { + PORT: '4000', + BASE_URL: 'https://api.example', + NODE_CONFIG_ENV: 'prod' + } + ); + + expect(config).toMatchObject({ + server: { + port: 4000, + baseURL: 'https://api.example', + nodeEnvironment: 'test', + configEnvironment: 'prod' + }, + search: { + provider: { + allowUnknownSslCerts: true + }, + logPerformanceTimings: true, + postFilterCacheTtlMs: 120000, + postFilterCacheMaxEntries: 250, + postFilterColdSearchConcurrency: 6, + postFilterColdSearchMaxQueueDepth: 50, + postFilterColdSearchMaxWaitMs: 15000, + inFlightMaxWaitersPerOperation: 125, + postFilterEvaluationChunkSize: 100, + openSearchRequestTimeoutMs: 25000, + requestDeadlineMs: 45000, + readinessTimeoutMs: 7000 + }, + requestLogging: { + successSampleRate: 0.25, + healthChecks: true + }, + runtime: { + shutdownTimeoutMs: 60000 + }, + features: { + webSearchEnabled: false + }, + rateLimiting: { + search: { + enabled: true, + windowMs: 30000, + maxRequests: 25 + } + } + }); + }); + + test.each([ + ['1', 'search.allowUnknownSslCerts'], + ['yes', 'search.logPerformanceTimings'], + ['invalid', 'requestLogging.healthChecks'], + [1, 'features.webSearchEnabled'], + ['on', 'rateLimiting.search.enabled'] + ])('rejects non-boolean value %p for %s', (value, path) => { + expect(() => resolve({ [path]: value })).toThrow(`${path} must be true or false.`); + }); + + test.each([ + ['PORT', { PORT: '0' }, {}], + ['search.postFilterCacheTtlMs', {}, { 'search.postFilterCacheTtlMs': '-1' }], + ['search.postFilterCacheMaxEntries', {}, { 'search.postFilterCacheMaxEntries': '10001' }], + ['search.postFilterColdSearchConcurrency', {}, { 'search.postFilterColdSearchConcurrency': '101' }], + ['search.postFilterColdSearchMaxQueueDepth', {}, { 'search.postFilterColdSearchMaxQueueDepth': '10001' }], + ['search.postFilterEvaluationChunkSize', {}, { 'search.postFilterEvaluationChunkSize': '1.5' }], + ['requestLogging.successSampleRate', {}, { 'requestLogging.successSampleRate': '1.1' }] + ])('rejects an out-of-range or malformed %s value', (path, env, values) => { + expect(() => resolve(values, env)).toThrow(path); + }); + + test.each([ + [ + 'search.readinessTimeoutMs', + { + 'search.readinessTimeoutMs': '21000' + } + ], + [ + 'search.openSearchRequestTimeoutMs', + { + 'search.openSearchRequestTimeoutMs': '30000' + } + ], + [ + 'search.postFilterColdSearchMaxWaitMs', + { + 'search.postFilterColdSearchMaxWaitMs': '30000' + } + ], + [ + 'runtime.shutdownTimeoutMs', + { + 'runtime.shutdownTimeoutMs': '30000' + } + ], + [ + 'rateLimiting.search.maxRequests', + { + 'rateLimiting.search.enabled': 'true' + } + ] + ])('rejects invalid cross-setting relationship for %s', (path, values) => { + expect(() => resolve(values)).toThrow(path); + }); + + test.each([ + [{ providerEndpoint: '' }, 'search.providerEndpoint'], + [{ index: '' }, 'search.index'], + [{ providerEndpoint: 'not-a-url' }, 'search.providerEndpoint'] + ])('startup validation rejects an invalid provider configuration', (providerOverrides, path) => { + const config = resolve({}, {}, providerOverrides); + + expect(() => runtimeConfig.validateRuntimeConfig(config)).toThrow(path); + }); + + test('rejects credential-bearing BASE_URL values', () => { + expect(() => + resolve( + {}, + { + BASE_URL: 'https://admin:secret@api.example' + } + ) + ).toThrow('BASE_URL must not contain credentials.'); + }); + + test('sanitized startup metadata does not expose provider or base URLs', () => { + const config = resolve( + {}, + { + BASE_URL: 'https://api.example' + } + ); + + const serialized = JSON.stringify(runtimeConfig.sanitizeRuntimeConfig(config)); + + expect(serialized).not.toContain('search.example'); + expect(serialized).not.toContain('test-index'); + expect(serialized).not.toContain('api.example'); + }); +}); diff --git a/utils/runtimeLifecycle.js b/utils/runtimeLifecycle.js index cc2c86f..09bc3a8 100644 --- a/utils/runtimeLifecycle.js +++ b/utils/runtimeLifecycle.js @@ -1,18 +1,10 @@ -const { AppConfig } = require('cve-core'); -const constants = require('./constants'); +const runtimeConfig = require('./runtimeConfig'); let shuttingDown = false; let activeRequestCount = 0; function getShutdownTimeoutMs() { - if (!AppConfig.has('runtime.shutdownTimeoutMs')) { - return constants.RUNTIME_SHUTDOWN_TIMEOUT_MS; - } - - const value = Number(AppConfig.get('runtime.shutdownTimeoutMs')); - if (Number.isInteger(value) && value >= 1) return value; - - return constants.RUNTIME_SHUTDOWN_TIMEOUT_MS; + return runtimeConfig.getRuntimeConfig().runtime.shutdownTimeoutMs; } function getState() { @@ -52,6 +44,7 @@ function trackRequest(_req, res, next) { function resetForTests() { shuttingDown = false; activeRequestCount = 0; + runtimeConfig.resetForTests(); } module.exports = { diff --git a/utils/runtimeLifecycle.test.js b/utils/runtimeLifecycle.test.js index 47b2571..a01edf4 100644 --- a/utils/runtimeLifecycle.test.js +++ b/utils/runtimeLifecycle.test.js @@ -65,13 +65,10 @@ describe('runtimeLifecycle', () => { expect(mockAppConfigGet).not.toHaveBeenCalled(); }); - test.each(['not-a-number', '0', '-1', '1.5'])( - 'falls back to the shutdown timeout default for invalid value %s', - (value) => { - mockAppConfigHas.mockReturnValue(true); - mockAppConfigGet.mockReturnValue(value); - - expect(runtimeLifecycle.getShutdownTimeoutMs()).toBe(35000); - } - ); + test.each(['not-a-number', '0', '-1', '1.5'])('rejects an invalid shutdown timeout value %s', (value) => { + mockAppConfigHas.mockImplementation((path) => path === 'runtime.shutdownTimeoutMs'); + mockAppConfigGet.mockReturnValue(value); + + expect(() => runtimeLifecycle.getShutdownTimeoutMs()).toThrow('runtime.shutdownTimeoutMs'); + }); }); diff --git a/utils/searchProvider.js b/utils/searchProvider.js index 75502ae..eb4393b 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -1,5 +1,5 @@ -const { AppConfig, BasicSearchManager, SearchProviderSpec, SearchReader } = require('cve-core'); -const constants = require('./constants'); +const { BasicSearchManager, SearchReader } = require('cve-core'); +const runtimeConfig = require('./runtimeConfig'); const searchDeadline = require('./searchDeadline'); const searchResponseValidator = require('./searchResponseValidator'); @@ -19,7 +19,6 @@ class SearchProviderConfigurationError extends Error { } } -let searchProviderConfig; let basicSearchManager; let searchReader; let activeOpenSearchRequestCount = 0; @@ -39,59 +38,14 @@ function configureManagerRequestTimeout(manager) { return manager; } -function normalizeBoolean(value, defaultValue = false) { - if (value === undefined || value === null || value === '') return defaultValue; - if (typeof value === 'boolean') return value; - if (typeof value === 'number') return value !== 0; - - const normalized = String(value).trim().toLowerCase(); - if (['true', '1', 'yes', 'y', 'on'].includes(normalized)) return true; - if (['false', '0', 'no', 'n', 'off'].includes(normalized)) return false; - - return defaultValue; -} - -function getIntegerConfig(path, defaultValue, { min = 0 } = {}) { - if (!AppConfig.has(path)) return defaultValue; - - const value = Number(AppConfig.get(path)); - if (Number.isInteger(value) && value >= min) return value; - - return defaultValue; -} - -function getAllowUnknownSslCerts(spec) { - if (AppConfig.has('search.allowUnknownSslCerts')) { - return normalizeBoolean(AppConfig.get('search.allowUnknownSslCerts')); - } - - return normalizeBoolean(spec?.allowUnknownSslCerts); -} - -function normalizeSearchProviderConfig(spec) { - const providerEndpoint = spec?.providerEndpoint; - const index = spec?.index; - - if (!providerEndpoint || !index) { - throw new SearchProviderConfigurationError(); - } - - return Object.freeze({ - providerEndpoint, - index, - allowUnknownSslCerts: getAllowUnknownSslCerts(spec) - }); -} - function getSearchProviderConfig() { - if (!searchProviderConfig) { - const defaultSpec = SearchProviderSpec.getDefaultSearchProviderSpec(); - searchProviderConfig = normalizeSearchProviderConfig(defaultSpec); - defaultSpec.allowUnknownSslCerts = searchProviderConfig.allowUnknownSslCerts; - SearchProviderSpec._sDefaultSearchProviderSpec = defaultSpec; + const config = runtimeConfig.getRuntimeConfig().search.provider; + if (!config.providerEndpoint || !config.index) { + throw new SearchProviderConfigurationError(); } - return searchProviderConfig; + runtimeConfig.validateRuntimeConfig(); + return config; } function getBasicSearchManager() { @@ -297,89 +251,61 @@ function isOpenSearchResponseError(err) { } function getOpenSearchReadinessTimeoutMs() { - return getIntegerConfig('search.readinessTimeoutMs', constants.SEARCH_READINESS_TIMEOUT_MS, { - min: 1 - }); + return runtimeConfig.getRuntimeConfig().search.readinessTimeoutMs; } function shouldLogSearchPerformance() { - if (!AppConfig.has('search.logPerformanceTimings')) return false; - - return normalizeBoolean(AppConfig.get('search.logPerformanceTimings')); + return runtimeConfig.getRuntimeConfig().search.logPerformanceTimings; } function getPostFilterCacheTtlMs() { - return getIntegerConfig('search.postFilterCacheTtlMs', constants.SEARCH_POST_FILTER_CACHE_TTL_MS); + return runtimeConfig.getRuntimeConfig().search.postFilterCacheTtlMs; } function getPostFilterCacheMaxEntries() { - return getIntegerConfig('search.postFilterCacheMaxEntries', constants.SEARCH_POST_FILTER_CACHE_MAX_ENTRIES); + return runtimeConfig.getRuntimeConfig().search.postFilterCacheMaxEntries; } function getPostFilterColdSearchConcurrency() { - return getIntegerConfig( - 'search.postFilterColdSearchConcurrency', - constants.SEARCH_POST_FILTER_COLD_SEARCH_CONCURRENCY, - { min: 1 } - ); + return runtimeConfig.getRuntimeConfig().search.postFilterColdSearchConcurrency; } function getPostFilterColdSearchMaxQueueDepth() { - return getIntegerConfig( - 'search.postFilterColdSearchMaxQueueDepth', - constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_QUEUE_DEPTH, - { min: 0 } - ); + return runtimeConfig.getRuntimeConfig().search.postFilterColdSearchMaxQueueDepth; } function getPostFilterColdSearchMaxWaitMs() { - return getIntegerConfig( - 'search.postFilterColdSearchMaxWaitMs', - constants.SEARCH_POST_FILTER_COLD_SEARCH_MAX_WAIT_MS, - { min: 1 } - ); + return runtimeConfig.getRuntimeConfig().search.postFilterColdSearchMaxWaitMs; } function getInFlightMaxWaitersPerOperation() { - return getIntegerConfig( - 'search.inFlightMaxWaitersPerOperation', - constants.SEARCH_IN_FLIGHT_MAX_WAITERS_PER_OPERATION, - { min: 1 } - ); + return runtimeConfig.getRuntimeConfig().search.inFlightMaxWaitersPerOperation; } function getPostFilterEvaluationChunkSize() { - return getIntegerConfig( - 'search.postFilterEvaluationChunkSize', - constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE, - { min: 1 } - ); + return runtimeConfig.getRuntimeConfig().search.postFilterEvaluationChunkSize; } function getOpenSearchRequestTimeoutMs() { - return getIntegerConfig('search.openSearchRequestTimeoutMs', constants.SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS, { - min: 1 - }); + return runtimeConfig.getRuntimeConfig().search.openSearchRequestTimeoutMs; } function getSearchRequestDeadlineMs() { - return getIntegerConfig('search.requestDeadlineMs', constants.SEARCH_REQUEST_DEADLINE_MS, { min: 1 }); + return runtimeConfig.getRuntimeConfig().search.requestDeadlineMs; } function resetSearchProviderForTests() { - searchProviderConfig = undefined; basicSearchManager = undefined; searchReader = undefined; activeOpenSearchRequestCount = 0; activeOpenSearchRequestsByOperation.clear(); + runtimeConfig.resetForTests(); } module.exports = { checkOpenSearchReadiness, closeSearchProviderClients, getBasicSearchManager, - getAllowUnknownSslCerts, - getIntegerConfig, getOpenSearchRequestState, getOpenSearchReadinessTimeoutMs, getInFlightMaxWaitersPerOperation, @@ -396,7 +322,6 @@ module.exports = { isOpenSearchResponseError, isSearchProviderConfigurationError, isSearchProviderContractError, - normalizeBoolean, resetSearchProviderForTests, searchOpenSearch, shouldLogSearchPerformance, diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index 4fd4a79..ed13c55 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -115,11 +115,6 @@ describe('searchProvider', () => { allowUnknownSslCerts: false }); expect(Object.isFrozen(first)).toBe(true); - expect(mockSearchProviderSpec._sDefaultSearchProviderSpec).toStrictEqual({ - providerEndpoint: 'https://search.example', - index: 'test-index', - allowUnknownSslCerts: false - }); expect(mockGetDefaultSearchProviderSpec).toHaveBeenCalledTimes(1); }); @@ -658,7 +653,7 @@ describe('searchProvider', () => { index: 'test-index', allowUnknownSslCerts: true }); - mockAppConfigHas.mockReturnValue(true); + mockAppConfigHas.mockImplementation((path) => path === 'search.allowUnknownSslCerts'); mockAppConfigGet.mockReturnValue('FALSE'); expect(searchProvider.getSearchProviderConfig()).toStrictEqual({ @@ -666,11 +661,10 @@ describe('searchProvider', () => { index: 'test-index', allowUnknownSslCerts: false }); - expect(mockSearchProviderSpec._sDefaultSearchProviderSpec.allowUnknownSslCerts).toBe(false); }); test('normalizes string TRUE from AppConfig for local certificate bypasses', () => { - mockAppConfigHas.mockReturnValue(true); + mockAppConfigHas.mockImplementation((path) => path === 'search.allowUnknownSslCerts'); mockAppConfigGet.mockReturnValue('TRUE'); expect(searchProvider.getSearchProviderConfig().allowUnknownSslCerts).toBe(true); @@ -695,7 +689,8 @@ describe('searchProvider', () => { 'search.postFilterEvaluationChunkSize', 'search.openSearchRequestTimeoutMs', 'search.requestDeadlineMs', - 'search.readinessTimeoutMs' + 'search.readinessTimeoutMs', + 'runtime.shutdownTimeoutMs' ].includes(key) ); mockAppConfigGet.mockImplementation((key) => { @@ -709,7 +704,8 @@ describe('searchProvider', () => { 'search.postFilterEvaluationChunkSize': '100', 'search.openSearchRequestTimeoutMs': '25000', 'search.requestDeadlineMs': '45000', - 'search.readinessTimeoutMs': '7000' + 'search.readinessTimeoutMs': '7000', + 'runtime.shutdownTimeoutMs': '60000' }; return values[key]; }); @@ -726,19 +722,10 @@ describe('searchProvider', () => { expect(searchProvider.getOpenSearchReadinessTimeoutMs()).toBe(7000); }); - test('falls back to CPE post-filter runtime defaults when config values are invalid', () => { - mockAppConfigHas.mockReturnValue(true); + test('rejects an invalid explicit CPE post-filter runtime value', () => { + mockAppConfigHas.mockImplementation((key) => key === 'search.postFilterCacheTtlMs'); mockAppConfigGet.mockReturnValue('not-a-number'); - expect(searchProvider.getPostFilterCacheTtlMs()).toBe(60000); - expect(searchProvider.getPostFilterCacheMaxEntries()).toBe(100); - expect(searchProvider.getPostFilterColdSearchConcurrency()).toBe(10); - expect(searchProvider.getPostFilterColdSearchMaxQueueDepth()).toBe(500); - expect(searchProvider.getPostFilterColdSearchMaxWaitMs()).toBe(10000); - expect(searchProvider.getInFlightMaxWaitersPerOperation()).toBe(500); - expect(searchProvider.getPostFilterEvaluationChunkSize()).toBe(250); - expect(searchProvider.getOpenSearchRequestTimeoutMs()).toBe(20000); - expect(searchProvider.getSearchRequestDeadlineMs()).toBe(30000); - expect(searchProvider.getOpenSearchReadinessTimeoutMs()).toBe(5000); + expect(() => searchProvider.getPostFilterCacheTtlMs()).toThrow('search.postFilterCacheTtlMs'); }); }); diff --git a/utils/searchRateLimiter.js b/utils/searchRateLimiter.js index 00026e1..eee12ad 100644 --- a/utils/searchRateLimiter.js +++ b/utils/searchRateLimiter.js @@ -1,62 +1,14 @@ -const { AppConfig } = require('cve-core'); const { rateLimit } = require('express-rate-limit'); const errors = require('./errors'); +const runtimeConfig = require('./runtimeConfig'); const searchPerformance = require('./searchPerformance'); -const DEFAULT_SEARCH_RATE_LIMIT_ENABLED = false; -const DEFAULT_SEARCH_RATE_LIMIT_WINDOW_MS = 60000; const GLOBAL_SEARCH_RATE_LIMIT_KEY = 'search-api-process'; const error = new errors.errors(); -function parseBoolean(value) { - if (typeof value === 'boolean') return value; - - const normalized = String(value).trim().toLowerCase(); - if (['true', '1', 'yes', 'y', 'on'].includes(normalized)) return true; - if (['false', '0', 'no', 'n', 'off'].includes(normalized)) return false; - - return undefined; -} - -function readBooleanConfig(path, defaultValue) { - if (!AppConfig.has(path)) return defaultValue; - - const value = parseBoolean(AppConfig.get(path)); - if (value === undefined) throw new Error(`${path} must be a boolean.`); - - return value; -} - -function readPositiveIntegerConfig(path, defaultValue) { - if (!AppConfig.has(path)) return defaultValue; - - const value = Number(AppConfig.get(path)); - if (!Number.isInteger(value) || value < 1) return undefined; - - return value; -} - function getSearchRateLimitConfig() { - const enabled = readBooleanConfig('rateLimiting.search.enabled', DEFAULT_SEARCH_RATE_LIMIT_ENABLED); - if (!enabled) { - return Object.freeze({ - enabled, - windowMs: DEFAULT_SEARCH_RATE_LIMIT_WINDOW_MS - }); - } - - const windowMs = readPositiveIntegerConfig('rateLimiting.search.windowMs', DEFAULT_SEARCH_RATE_LIMIT_WINDOW_MS); - const maxRequests = readPositiveIntegerConfig('rateLimiting.search.maxRequests'); - - if (!windowMs) throw new Error('rateLimiting.search.windowMs must be a positive integer when enabled.'); - if (!maxRequests) throw new Error('rateLimiting.search.maxRequests must be a positive integer when enabled.'); - - return Object.freeze({ - enabled, - windowMs, - maxRequests - }); + return runtimeConfig.getRuntimeConfig().rateLimiting.search; } function disabledRateLimiter(_req, _res, next) { @@ -93,8 +45,13 @@ function createSearchRateLimiter(config = getSearchRateLimitConfig()) { }); } +function resetForTests() { + runtimeConfig.resetForTests(); +} + module.exports = { createSearchRateLimiter, getSearchRateLimitConfig, - handleRateLimit + handleRateLimit, + resetForTests }; diff --git a/utils/searchRateLimiter.test.js b/utils/searchRateLimiter.test.js index da00ea6..170d5ef 100644 --- a/utils/searchRateLimiter.test.js +++ b/utils/searchRateLimiter.test.js @@ -20,6 +20,7 @@ const searchRateLimiter = require('./searchRateLimiter'); describe('searchRateLimiter', () => { beforeEach(() => { + searchRateLimiter.resetForTests(); mockAppConfigHas.mockReset(); mockAppConfigGet.mockReset(); mockLogSearchPerformance.mockReset(); @@ -56,7 +57,7 @@ describe('searchRateLimiter', () => { { 'rateLimiting.search.enabled': 'invalid' }, - 'rateLimiting.search.enabled must be a boolean.' + 'rateLimiting.search.enabled' ], [ { @@ -64,7 +65,7 @@ describe('searchRateLimiter', () => { 'rateLimiting.search.windowMs': '0', 'rateLimiting.search.maxRequests': '25' }, - 'rateLimiting.search.windowMs must be a positive integer when enabled.' + 'rateLimiting.search.windowMs' ], [ { @@ -72,7 +73,7 @@ describe('searchRateLimiter', () => { 'rateLimiting.search.windowMs': '60000', 'rateLimiting.search.maxRequests': '0' }, - 'rateLimiting.search.maxRequests must be a positive integer when enabled.' + 'rateLimiting.search.maxRequests' ] ])('rejects invalid explicit configuration', (values, expectedMessage) => { mockAppConfigHas.mockImplementation((path) => Object.hasOwn(values, path)); From 113f858c895bfa82fc1956a1fb29ee209785cfde Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 18:02:34 -0400 Subject: [PATCH 16/33] Bound search resource usage --- .env.example | 3 + README.md | 19 ++- config/custom-environment-variables.jsonc | 7 +- config/default.jsonc | 10 +- config/runtime-config.test.js | 3 + controllers/searchController.js | 9 +- controllers/searchController.test.js | 21 ++- routes/index.js | 5 +- routes/index.test.js | 34 +--- server.js | 14 +- server.test.js | 56 +++++++ utils/constants.js | 6 + utils/runtimeConfig.js | 28 +++- utils/runtimeConfig.test.js | 26 ++- utils/searchPerformance.js | 77 +++++++++ utils/searchPerformance.test.js | 59 +++++++ utils/searchPostFilterCache.js | 15 ++ utils/searchPostFilterCache.test.js | 26 +++ utils/searchQueryBuilder.js | 64 +++----- utils/searchQueryBuilder.test.js | 185 +++++++--------------- 20 files changed, 438 insertions(+), 229 deletions(-) diff --git a/.env.example b/.env.example index 2067d05..ef0af0e 100644 --- a/.env.example +++ b/.env.example @@ -18,6 +18,9 @@ SearchReadinessTimeoutMs=5000 RequestLogSuccessSampleRate=0.1 RequestLogHealthChecks=false ShutdownTimeoutMs=35000 +HttpHeadersTimeoutMs=10000 +HttpRequestTimeoutMs=15000 +HttpKeepAliveTimeoutMs=5000 WebSearchEnabled=true SearchRateLimitEnabled=false SearchRateLimitWindowMs=60000 diff --git a/README.md b/README.md index fd109f1..b3d9e6b 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,9 @@ Rename `.env.example` to `.env` and replace or check the following values. - `RequestLogSuccessSampleRate` - optional fraction of successful non-health-check access logs to retain, default `0.1` - `RequestLogHealthChecks` - optional flag to retain successful health-check access logs, default `false` - `ShutdownTimeoutMs` - optional graceful HTTP shutdown deadline, default `35000` +- `HttpHeadersTimeoutMs` - optional deadline for receiving HTTP request headers, default `10000` +- `HttpRequestTimeoutMs` - optional deadline for receiving a complete HTTP request, including its body, default `15000` +- `HttpKeepAliveTimeoutMs` - optional idle keep-alive connection timeout, default `5000` - `WebSearchEnabled` - optional deprecated `/webSearch` feature flag, default `true` - `SearchRateLimitEnabled` - optional per-process `/search` rate-limit flag, default `false` - `SearchRateLimitWindowMs` - optional rate-limit window in milliseconds, default `60000` @@ -55,6 +58,9 @@ The following numeric bounds are enforced: | `SearchReadinessTimeoutMs` | 1 through 60,000 | | `RequestLogSuccessSampleRate` | 0 through 1 | | `ShutdownTimeoutMs` | 1 through 900,000 | +| `HttpHeadersTimeoutMs` | 1 through 60,000 | +| `HttpRequestTimeoutMs` | 1 through 300,000 | +| `HttpKeepAliveTimeoutMs` | 1 through 60,000 | | `SearchRateLimitWindowMs` | 1 through 86,400,000 | | `SearchRateLimitMaxRequests` | 0 through 10,000,000; must be positive when rate limiting is enabled | @@ -63,6 +69,7 @@ Cache TTL, cache entry count, and cold-search queue depth accept `0` because tho ```text SearchReadinessTimeoutMs <= SearchOpenSearchRequestTimeoutMs < SearchRequestDeadlineMs < ShutdownTimeoutMs SearchPostFilterColdSearchMaxWaitMs < SearchRequestDeadlineMs +HttpHeadersTimeoutMs <= HttpRequestTimeoutMs ``` `OpenSearchDomainEndpoint` must be an absolute HTTP or HTTPS URL, and `OpenSearchCveIndex` must be non-empty. `BASE_URL`, when supplied, must also be an absolute HTTP or HTTPS URL and cannot contain credentials. OpenSearch connectivity is not required at startup; `/api/ready-check` remains responsible for dependency readiness. @@ -330,13 +337,15 @@ Access logs omit query strings and request bodies. By default, the API retains 1 The API always emits a sanitized JSON `search.performance` event for `/search` queue saturation, shared-operation waiter saturation, queue wait timeout, search timeout, client disconnect, OpenSearch connection errors, and upstream search errors. These events include the request ID, outcome, HTTP status, strategy, cache state, total elapsed time, cancellation reason, and bounded queue or waiter state when applicable. Search terms, CPE values, request bodies, credentials, and upstream error details are never included. -Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, shared waiter counts, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, and total handler elapsed time. Leave it disabled during normal operation unless the deployment logging volume is acceptable. +Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, shared waiter counts, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, response bytes, JSON serialization time, process heap usage and per-request heap delta, cache entry and cached CVE ID counts, process-wide event-loop delay, and total handler elapsed time. Leave it disabled during normal operation unless the deployment logging volume is acceptable. + +Heap deltas and event-loop delay are process-level diagnostics rather than isolated per-request measurements because concurrent requests share the same Node.js process. Use them to correlate latency with memory pressure or event-loop stalls across a test window, not to attribute resource use precisely to one request. Each `/search` request owns a lifecycle cancellation signal. When its overall deadline expires, the API aborts the active OpenSearch transport request before returning HTTP 504. When the caller disconnects, the API aborts work that is needed only by that caller and does not attempt to write a response. This prevents timed-out or abandoned standard searches from continuing until the transport timeout. ### Application Rate Limiting -Application rate limiting is disabled by default and applies only to `POST /search`. It does not apply to `/webSearch`, health checks, Swagger, OpenAPI, or static assets. +Application rate limiting is disabled by default and applies only to `POST /search`. It runs before JSON body parsing so a rejected request does not consume body-parsing work. It does not apply to `/webSearch`, health checks, Swagger, OpenAPI, or static assets. The fallback limiter uses one in-memory counter per Node.js process. Set `SearchRateLimitEnabled=true`, provide a positive `SearchRateLimitMaxRequests`, and optionally change `SearchRateLimitWindowMs` to enable it. Invalid enabled configuration prevents the application from starting. Rejected requests return HTTP 429 with `SEARCH_RATE_LIMITED`, `Retry-After`, `RateLimit`, and `RateLimit-Policy` headers, and emit a sanitized `search.performance` event with the `rate_limited` outcome. @@ -348,6 +357,8 @@ Searches that use `cpeName`, `virtualMatchString`, or `virtualMatchString` with The post-filter cache stores only the filtered CVE ID list and candidate count for that expensive candidate query. It does not cache full CVE records. On a cache hit, the API skips the 10,000-candidate OpenSearch query and semantic post-filtering, slices the cached CVE IDs for the requested page, and fetches that page of full CVE records by ID. +Candidate retrieval asks OpenSearch for exact-version or wildcard-version criteria and leaves range applicability decisions to the semantic post-filter. The full-record page fetch uses an exact CVE ID set without total-hit counting or OpenSearch sorting; the API restores the requested CVE ID order locally. Standard searches retain exact total counting and deterministic sorting. + The cache is intentionally small and short-lived: - TTL: 60 seconds @@ -389,6 +400,10 @@ The cache and cold-search queue are a temporary per-process proof of concept. Th The request lifecycle and in-flight waiter registry are separate from the result cache, so a later shared-cache replacement does not need to change caller cancellation behavior. The per-process TTL, entry limit, cold-search concurrency, queue depth, queue wait, shared waiter limit, evaluation chunk size, OpenSearch timeout, and overall request deadline can be tuned with `SearchPostFilterCacheTtlMs`, `SearchPostFilterCacheMaxEntries`, `SearchPostFilterColdSearchConcurrency`, `SearchPostFilterColdSearchMaxQueueDepth`, `SearchPostFilterColdSearchMaxWaitMs`, `SearchInFlightMaxWaitersPerOperation`, `SearchPostFilterEvaluationChunkSize`, `SearchOpenSearchRequestTimeoutMs`, and `SearchRequestDeadlineMs`. +### HTTP Transport Timeouts + +The Node.js HTTP server limits request-header receipt to 10 seconds, complete request receipt (including the body) to 15 seconds, and idle keep-alive connections to 5 seconds. These limits bound slow-client connection usage before application validation begins. They do not shorten the 30-second `/search` processing deadline, which starts after the request reaches the search handler. Override the defaults with `HttpHeadersTimeoutMs`, `HttpRequestTimeoutMs`, and `HttpKeepAliveTimeoutMs`; header receipt cannot be configured longer than complete request receipt. + Docker runtime smoke checks can be run after building the image. The default `Dockerfile` avoids BuildKit `RUN --mount` syntax for AWS builders that do not support it: ```bash diff --git a/config/custom-environment-variables.jsonc b/config/custom-environment-variables.jsonc index 205aa1f..e9a2744 100644 --- a/config/custom-environment-variables.jsonc +++ b/config/custom-environment-variables.jsonc @@ -28,7 +28,12 @@ "healthChecks": "RequestLogHealthChecks" }, "runtime": { - "shutdownTimeoutMs": "ShutdownTimeoutMs" + "shutdownTimeoutMs": "ShutdownTimeoutMs", + "http": { + "headersTimeoutMs": "HttpHeadersTimeoutMs", + "requestTimeoutMs": "HttpRequestTimeoutMs", + "keepAliveTimeoutMs": "HttpKeepAliveTimeoutMs" + } }, "features": { "webSearchEnabled": "WebSearchEnabled" diff --git a/config/default.jsonc b/config/default.jsonc index bd81068..0c4c0b8 100644 --- a/config/default.jsonc +++ b/config/default.jsonc @@ -47,7 +47,15 @@ }, "runtime": { // Must remain shorter than the container orchestrator's stop timeout. - "shutdownTimeoutMs": "35000" + "shutdownTimeoutMs": "35000", + "http": { + // Limits how long a client can take to send request headers. + "headersTimeoutMs": "10000", + // Limits how long a client can take to send the complete request, including its body. + "requestTimeoutMs": "15000", + // Limits how long an idle keep-alive connection remains open. + "keepAliveTimeoutMs": "5000" + } }, // Feature flags used for controlled endpoint migrations. "features": { diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index ea646dc..1dc7f66 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -132,6 +132,9 @@ describe('runtime container and compose configuration', () => { expect(customEnvironmentVariables).toContain('"successSampleRate": "RequestLogSuccessSampleRate"'); expect(customEnvironmentVariables).toContain('"healthChecks": "RequestLogHealthChecks"'); expect(customEnvironmentVariables).toContain('"shutdownTimeoutMs": "ShutdownTimeoutMs"'); + expect(customEnvironmentVariables).toContain('"headersTimeoutMs": "HttpHeadersTimeoutMs"'); + expect(customEnvironmentVariables).toContain('"requestTimeoutMs": "HttpRequestTimeoutMs"'); + expect(customEnvironmentVariables).toContain('"keepAliveTimeoutMs": "HttpKeepAliveTimeoutMs"'); expect(customEnvironmentVariables).toContain('"webSearchEnabled": "WebSearchEnabled"'); expect(customEnvironmentVariables).toContain('"enabled": "SearchRateLimitEnabled"'); expect(customEnvironmentVariables).toContain('"windowMs": "SearchRateLimitWindowMs"'); diff --git a/controllers/searchController.js b/controllers/searchController.js index f36ea98..64152cc 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -187,8 +187,7 @@ async function fetchFullHitsByCveIds(cveIds, metrics, signal) { const inFlight = searchInFlightRegistry.acquire( inFlightKey, async (operationSignal) => { - const { searchOptions, queryStringObjects } = searchQueryBuilder.buildCveIdsSearchRequest(cveIds); - const body = searchQueryBuilder.buildOpenSearchBody(searchOptions, queryStringObjects); + const body = searchQueryBuilder.buildCveIdsSearchBody(cveIds); const pageFetchStart = searchPerformance.markTime(); const pageData = await executeOpenSearch(body, 'page-fetch', operationSignal, metrics); metrics.pageFetchElapsedMs = searchPerformance.elapsedMs(pageFetchStart); @@ -296,6 +295,10 @@ async function executeOptimizedPostFilterSearch( const candidateResult = await getPostFilterCandidateResult(cacheKey, candidateBody, postFilters, metrics, signal); searchDeadline.throwIfAborted(signal); const pageCveIds = paginateHits(candidateResult.filteredCveIds, responseOptions); + if (metrics.collectDetailedMetrics) { + metrics.cachedResultCveIdCount = candidateResult.filteredCveIds.length; + metrics.cacheState = searchPostFilterCache.getState(); + } metrics.candidateCount = candidateResult.candidateCount; metrics.filteredCount = candidateResult.filteredCount ?? candidateResult.filteredCveIds.length; metrics.returnedCount = pageCveIds.length; @@ -337,6 +340,7 @@ async function searchHandler(req, res, next) { const metrics = { requestId: req.requestId }; + metrics.collectDetailedMetrics = searchPerformance.beginSearchRequest(metrics, res); const handlerStart = searchPerformance.markTime(); const lifecycle = searchDeadline.createSearchRequestLifecycle( req, @@ -466,6 +470,7 @@ async function searchHandler(req, res, next) { } finally { lifecycle.complete(); metrics.totalElapsedMs = searchPerformance.elapsedMs(handlerStart); + searchPerformance.completeSearchRequest(metrics); searchPerformance.logSearchPerformance(metrics, searchOptions, responseOptions, postFilters); } } diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 2ba9c83..cd3be86 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -59,6 +59,7 @@ jest.mock('../utils/searchProvider', () => ({ const controller = require('./searchController.js'); const searchCandidateProcessor = require('../utils/searchCandidateProcessor'); const searchInFlightRegistry = require('../utils/searchInFlightRegistry'); +const searchPerformance = require('../utils/searchPerformance'); const searchPostFilterCache = require('../utils/searchPostFilterCache'); function createResponse() { @@ -215,6 +216,7 @@ describe('Search controller SEARCH handler', () => { afterEach(() => { searchInFlightRegistry.clear(); + searchPerformance.resetForTests(); jest.restoreAllMocks(); }); @@ -583,14 +585,7 @@ describe('Search controller SEARCH handler', () => { ] } }, - sort: [ - { - 'cveMetadata.cveId.keyword': { - order: 'asc' - } - } - ], - track_total_hits: true, + track_total_hits: false, size: 1, from: 0 }, @@ -829,10 +824,11 @@ describe('Search controller SEARCH handler', () => { } ]); expect(mockApiSearch.mock.calls[0][0]).toMatchObject({ - track_total_hits: true, + track_total_hits: false, size: 1, from: 0 }); + expect(mockApiSearch.mock.calls[0][0].sort).toBeUndefined(); expect(mockApiSearch.mock.calls[0][1]).toEqual( expect.objectContaining({ operation: 'page-fetch', @@ -901,10 +897,11 @@ describe('Search controller SEARCH handler', () => { } ]); expect(mockApiSearch.mock.calls[1][0]).toMatchObject({ - track_total_hits: true, + track_total_hits: false, size: 1, from: 0 }); + expect(mockApiSearch.mock.calls[1][0].sort).toBeUndefined(); expect(secondRes.status).toHaveBeenCalledWith(200); expect(secondRes.json).toHaveBeenCalledWith({ page: 1, @@ -1408,7 +1405,9 @@ describe('Search controller SEARCH handler', () => { postFilters: [], requestedFrom: 0, requestedSize: 1, - openSearchSize: 1 + openSearchSize: 1, + heapUsedBytes: expect.any(Number), + heapUsedDeltaBytes: expect.any(Number) }); expect(console.log.mock.calls[0][0]).not.toContain('remote code'); }); diff --git a/routes/index.js b/routes/index.js index 2d30dbc..966044f 100644 --- a/routes/index.js +++ b/routes/index.js @@ -7,11 +7,9 @@ const constants = require('../utils/constants'); const featureFlags = require('../utils/featureFlags'); const searchDeadline = require('../utils/searchDeadline'); const searchProvider = require('../utils/searchProvider'); -const searchRateLimiter = require('../utils/searchRateLimiter'); const runtimeLifecycle = require('../utils/runtimeLifecycle'); const error = new errors.errors(); -const limitSearchRequests = searchRateLimiter.createSearchRateLimiter(); const SEARCH_BODY_FIELDS = new Set(constants.SEARCH_QUERY_PARAMS); const SEARCH_STRING_BODY_FIELDS = constants.SEARCH_QUERY_PARAMS.filter( (field) => @@ -391,9 +389,8 @@ content: { } } } -*/ + */ requireSearchJsonContentType, - limitSearchRequests, rejectSearchQueryParams, body().custom(isSearchBodyObject).withMessage(error.searchBodyError()), body().custom(hasOnlySearchBodyFields), diff --git a/routes/index.test.js b/routes/index.test.js index 5aa1d56..b0693a3 100644 --- a/routes/index.test.js +++ b/routes/index.test.js @@ -23,13 +23,8 @@ jest.mock('../utils/featureFlags', () => ({ isWebSearchEnabled: jest.fn(() => true) })); -const mockLimitSearchRequests = jest.fn((req, res, next) => next()); const mockCheckOpenSearchReadiness = jest.fn(); -jest.mock('../utils/searchRateLimiter', () => ({ - createSearchRateLimiter: jest.fn(() => mockLimitSearchRequests) -})); - jest.mock('../utils/searchProvider', () => ({ checkOpenSearchReadiness: mockCheckOpenSearchReadiness, getOpenSearchReadinessTimeoutMs: jest.fn(() => 5000) @@ -58,7 +53,6 @@ describe('routes/index router', () => { jest.clearAllMocks(); featureFlags.isWebSearchEnabled.mockReturnValue(true); runtimeLifecycle.resetForTests(); - mockLimitSearchRequests.mockImplementation((req, res, next) => next()); mockCheckOpenSearchReadiness.mockResolvedValue(true); }); @@ -153,7 +147,6 @@ describe('routes/index router', () => { }); expect(response.status).toBe(200); - expect(mockLimitSearchRequests).toHaveBeenCalledTimes(1); expect(mw.validateParams).toHaveBeenCalledTimes(1); expect(controller.SEARCH).toHaveBeenCalledTimes(1); expect(response.body).toStrictEqual({ @@ -172,7 +165,6 @@ describe('routes/index router', () => { }); expect(response.status).toBe(200); - expect(mockLimitSearchRequests).toHaveBeenCalledTimes(1); expect(mw.validateParams).toHaveBeenCalledTimes(1); expect(controller.SEARCH).toHaveBeenCalledTimes(1); expect(response.body).toStrictEqual({ @@ -181,7 +173,7 @@ describe('routes/index router', () => { }); }); - test('POST /search rejects a non-empty body with an unsupported media type before rate limiting', async () => { + test('POST /search rejects a non-empty body with an unsupported media type', async () => { const response = await request(app, { method: 'POST', path: '/search', @@ -196,29 +188,6 @@ describe('routes/index router', () => { error: 'SEARCH_CONTENT_TYPE', message: 'The /search request body must use the application/json media type with UTF-8 encoding.' }); - expect(mockLimitSearchRequests).not.toHaveBeenCalled(); - expect(mw.validateParams).not.toHaveBeenCalled(); - expect(controller.SEARCH).not.toHaveBeenCalled(); - }); - - test('POST /search returns a rate-limit response before validation and controller work', async () => { - mockLimitSearchRequests.mockImplementation((req, res) => - res.status(429).json({ - error: 'SEARCH_RATE_LIMITED', - message: 'Too many search requests. Please retry later.' - }) - ); - - const response = await request(app, { - method: 'POST', - path: '/search', - body: { - keywordSearch: 'test' - } - }); - - expect(response.status).toBe(429); - expect(response.body.error).toBe('SEARCH_RATE_LIMITED'); expect(mw.validateParams).not.toHaveBeenCalled(); expect(controller.SEARCH).not.toHaveBeenCalled(); }); @@ -385,7 +354,6 @@ describe('routes/index router', () => { }); expect(response.status).toBe(200); - expect(mockLimitSearchRequests).not.toHaveBeenCalled(); expect(controller.WEB_SEARCH).toHaveBeenCalledTimes(1); expect(response.body).toStrictEqual({ route: 'webSearch', diff --git a/server.js b/server.js index b3ae874..820510a 100644 --- a/server.js +++ b/server.js @@ -14,6 +14,7 @@ const requestLogging = require('./utils/requestLogging'); const runtimeConfig = require('./utils/runtimeConfig'); const runtimeLifecycle = require('./utils/runtimeLifecycle'); const searchProvider = require('./utils/searchProvider'); +const searchRateLimiter = require('./utils/searchRateLimiter'); const error = new errors.errors(); const SHUTDOWN_PROBE_PATHS = new Set(['/api/health-check', '/api/ready-check']); @@ -113,6 +114,7 @@ function createApp() { app.use(cors()); app.use(runtimeLifecycle.trackRequest); app.use(rejectRequestsDuringShutdown); + app.post('/search', searchRateLimiter.createSearchRateLimiter()); app.use(express.json({ limit: constants.JSON_REQUEST_BODY_LIMIT_BYTES })); app.get('/api-docs/openapi.json', (req, res) => res.json(openApiSpecification)); @@ -152,8 +154,9 @@ function startServer(port) { const config = runtimeConfig.validateRuntimeConfig(); const effectivePort = resolveListenPort(port, config.server.port); const app = createApp(); + const server = configureHttpServer(http.createServer(app), config.runtime.http); - return http.createServer(app).listen(effectivePort, () => { + return server.listen(effectivePort, () => { const sanitizedConfig = runtimeConfig.sanitizeRuntimeConfig(config); logRuntimeEvent(console.log, { event: 'runtime.lifecycle', @@ -167,6 +170,14 @@ function startServer(port) { }); } +function configureHttpServer(server, httpConfig) { + server.headersTimeout = httpConfig.headersTimeoutMs; + server.requestTimeout = httpConfig.requestTimeoutMs; + server.keepAliveTimeout = httpConfig.keepAliveTimeoutMs; + + return server; +} + function handleStartupError(err, log = console.error) { const configurationError = runtimeConfig.isRuntimeConfigurationError(err); const event = { @@ -318,6 +329,7 @@ if (require.main === module) { } module.exports = { + configureHttpServer, createApp, createRequestLogger, handleUnexpectedError, diff --git a/server.test.js b/server.test.js index d2aaf78..ba27bfe 100644 --- a/server.test.js +++ b/server.test.js @@ -18,9 +18,14 @@ const mockWebSearch = jest.fn((req, res) => ); const mockMorganMiddleware = jest.fn((req, res, next) => next()); const mockMorgan = jest.fn(() => mockMorganMiddleware); +const mockLimitSearchRequests = jest.fn((req, res, next) => next()); +const mockCreateSearchRateLimiter = jest.fn(() => mockLimitSearchRequests); mockMorgan.token = jest.fn(); jest.mock('morgan', () => mockMorgan); +jest.mock('./utils/searchRateLimiter', () => ({ + createSearchRateLimiter: mockCreateSearchRateLimiter +})); jest.mock('./controllers/middleware', () => ({ validateParams: mockValidateParams @@ -32,6 +37,7 @@ jest.mock('./controllers/searchController', () => ({ })); const { + configureHttpServer, createApp, createRequestLogger, getRedactedRequestPath, @@ -60,6 +66,10 @@ describe('server application bootstrap', () => { const originalNodeEnv = process.env.NODE_ENV; const originalPort = process.env.PORT; + beforeEach(() => { + mockLimitSearchRequests.mockImplementation((req, res, next) => next()); + }); + afterEach(() => { restoreEnv('BASE_URL', originalBaseUrl); restoreEnv('NODE_ENV', originalNodeEnv); @@ -127,6 +137,7 @@ describe('server application bootstrap', () => { expect(response.status).toBe(200); expect(mockWebSearch).toHaveBeenCalledTimes(1); + expect(mockLimitSearchRequests).not.toHaveBeenCalled(); expect(response.body).toStrictEqual({ route: 'webSearch', body @@ -151,6 +162,29 @@ describe('server application bootstrap', () => { route: 'search', params: body }); + expect(mockLimitSearchRequests).toHaveBeenCalledTimes(1); + }); + + test('createApp applies the /search limiter before parsing its JSON body', async () => { + mockLimitSearchRequests.mockImplementation((req, res) => + res.status(429).json({ + error: 'SEARCH_RATE_LIMITED', + message: 'Too many search requests. Please retry later.' + }) + ); + + const response = await request(createApp(), { + method: 'POST', + path: '/search', + rawBody: '{"invalid":', + headers: { + 'Content-Type': 'application/json' + } + }); + + expect(response.status).toBe(429); + expect(response.body.error).toBe('SEARCH_RATE_LIMITED'); + expect(mockSearch).not.toHaveBeenCalled(); }); test('createApp rejects non-probe requests after shutdown starts', async () => { @@ -319,6 +353,28 @@ describe('server application bootstrap', () => { expect(startServer()).toBe(fakeServer); expect(createServer).toHaveBeenCalledWith(expect.any(Function)); expect(fakeServer.listen).toHaveBeenCalledWith(3000, expect.any(Function)); + expect(fakeServer).toMatchObject({ + headersTimeout: constants.HTTP_HEADERS_TIMEOUT_MS, + requestTimeout: constants.HTTP_REQUEST_TIMEOUT_MS, + keepAliveTimeout: constants.HTTP_KEEP_ALIVE_TIMEOUT_MS + }); + }); + + test('configureHttpServer applies explicit transport deadlines', () => { + const fakeServer = {}; + + expect( + configureHttpServer(fakeServer, { + headersTimeoutMs: 9000, + requestTimeoutMs: 12000, + keepAliveTimeoutMs: 4000 + }) + ).toBe(fakeServer); + expect(fakeServer).toStrictEqual({ + headersTimeout: 9000, + requestTimeout: 12000, + keepAliveTimeout: 4000 + }); }); test('startServer passes through an explicit port', () => { diff --git a/utils/constants.js b/utils/constants.js index 85edb5a..b422d8a 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -30,6 +30,9 @@ const SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS = 20 * 1000; const SEARCH_REQUEST_DEADLINE_MS = 30 * 1000; const SEARCH_READINESS_TIMEOUT_MS = 5 * 1000; const RUNTIME_SHUTDOWN_TIMEOUT_MS = 35 * 1000; +const HTTP_HEADERS_TIMEOUT_MS = 10 * 1000; +const HTTP_REQUEST_TIMEOUT_MS = 15 * 1000; +const HTTP_KEEP_ALIVE_TIMEOUT_MS = 5 * 1000; const rejectedCveStates = ['REJECT', 'REJECTED', 'Rejected']; const rejectedCveStateFields = ['cveMetadata.state', 'containers.cna.x_legacyV4Record.CVE_data_meta.STATE']; @@ -150,6 +153,9 @@ module.exports = { SEARCH_REQUEST_DEADLINE_MS, SEARCH_READINESS_TIMEOUT_MS, RUNTIME_SHUTDOWN_TIMEOUT_MS, + HTTP_HEADERS_TIMEOUT_MS, + HTTP_REQUEST_TIMEOUT_MS, + HTTP_KEEP_ALIVE_TIMEOUT_MS, RESULTS_PER_PAGE, REJECTED_CVE_STATES: rejectedCveStates, CPE_VERSION_RANGE_TYPES: cpeVersionRangeTypes, diff --git a/utils/runtimeConfig.js b/utils/runtimeConfig.js index 987aca2..9fc709f 100644 --- a/utils/runtimeConfig.js +++ b/utils/runtimeConfig.js @@ -22,6 +22,9 @@ const RUNTIME_CONFIG_LIMITS = Object.freeze({ requestDeadlineMs: Object.freeze({ min: 1, max: 10 * 60 * 1000 }), readinessTimeoutMs: Object.freeze({ min: 1, max: 60 * 1000 }), shutdownTimeoutMs: Object.freeze({ min: 1, max: 15 * 60 * 1000 }), + httpHeadersTimeoutMs: Object.freeze({ min: 1, max: 60 * 1000 }), + httpRequestTimeoutMs: Object.freeze({ min: 1, max: 5 * 60 * 1000 }), + httpKeepAliveTimeoutMs: Object.freeze({ min: 1, max: 60 * 1000 }), searchRateLimitWindowMs: Object.freeze({ min: 1, max: 24 * 60 * 60 * 1000 }), searchRateLimitMaxRequests: Object.freeze({ min: 0, max: 10000000 }) }); @@ -156,6 +159,12 @@ function validateRelationships(config) { 'must be greater than search.requestDeadlineMs' ); } + if (config.runtime.http.headersTimeoutMs > config.runtime.http.requestTimeoutMs) { + throw new RuntimeConfigurationError( + 'runtime.http.headersTimeoutMs', + 'must be less than or equal to runtime.http.requestTimeoutMs' + ); + } if (config.rateLimiting.search.enabled && config.rateLimiting.search.maxRequests < 1) { throw new RuntimeConfigurationError( 'rateLimiting.search.maxRequests', @@ -301,7 +310,24 @@ function resolveRuntimeConfig({ 'runtime.shutdownTimeoutMs', readConfigValue(appConfig, 'runtime.shutdownTimeoutMs', constants.RUNTIME_SHUTDOWN_TIMEOUT_MS), RUNTIME_CONFIG_LIMITS.shutdownTimeoutMs - ) + ), + http: { + headersTimeoutMs: parseInteger( + 'runtime.http.headersTimeoutMs', + readConfigValue(appConfig, 'runtime.http.headersTimeoutMs', constants.HTTP_HEADERS_TIMEOUT_MS), + RUNTIME_CONFIG_LIMITS.httpHeadersTimeoutMs + ), + requestTimeoutMs: parseInteger( + 'runtime.http.requestTimeoutMs', + readConfigValue(appConfig, 'runtime.http.requestTimeoutMs', constants.HTTP_REQUEST_TIMEOUT_MS), + RUNTIME_CONFIG_LIMITS.httpRequestTimeoutMs + ), + keepAliveTimeoutMs: parseInteger( + 'runtime.http.keepAliveTimeoutMs', + readConfigValue(appConfig, 'runtime.http.keepAliveTimeoutMs', constants.HTTP_KEEP_ALIVE_TIMEOUT_MS), + RUNTIME_CONFIG_LIMITS.httpKeepAliveTimeoutMs + ) + } }, features: { webSearchEnabled: parseBoolean( diff --git a/utils/runtimeConfig.test.js b/utils/runtimeConfig.test.js index 638642b..850ead3 100644 --- a/utils/runtimeConfig.test.js +++ b/utils/runtimeConfig.test.js @@ -64,7 +64,12 @@ describe('runtimeConfig', () => { healthChecks: false }, runtime: { - shutdownTimeoutMs: constants.RUNTIME_SHUTDOWN_TIMEOUT_MS + shutdownTimeoutMs: constants.RUNTIME_SHUTDOWN_TIMEOUT_MS, + http: { + headersTimeoutMs: constants.HTTP_HEADERS_TIMEOUT_MS, + requestTimeoutMs: constants.HTTP_REQUEST_TIMEOUT_MS, + keepAliveTimeoutMs: constants.HTTP_KEEP_ALIVE_TIMEOUT_MS + } }, features: { webSearchEnabled: true @@ -100,6 +105,9 @@ describe('runtimeConfig', () => { 'requestLogging.successSampleRate': '0.25', 'requestLogging.healthChecks': 'TRUE', 'runtime.shutdownTimeoutMs': '60000', + 'runtime.http.headersTimeoutMs': '12000', + 'runtime.http.requestTimeoutMs': '20000', + 'runtime.http.keepAliveTimeoutMs': '6000', 'features.webSearchEnabled': 'FALSE', 'rateLimiting.search.enabled': 'TRUE', 'rateLimiting.search.windowMs': '30000', @@ -140,7 +148,12 @@ describe('runtimeConfig', () => { healthChecks: true }, runtime: { - shutdownTimeoutMs: 60000 + shutdownTimeoutMs: 60000, + http: { + headersTimeoutMs: 12000, + requestTimeoutMs: 20000, + keepAliveTimeoutMs: 6000 + } }, features: { webSearchEnabled: false @@ -172,6 +185,9 @@ describe('runtimeConfig', () => { ['search.postFilterColdSearchConcurrency', {}, { 'search.postFilterColdSearchConcurrency': '101' }], ['search.postFilterColdSearchMaxQueueDepth', {}, { 'search.postFilterColdSearchMaxQueueDepth': '10001' }], ['search.postFilterEvaluationChunkSize', {}, { 'search.postFilterEvaluationChunkSize': '1.5' }], + ['runtime.http.headersTimeoutMs', {}, { 'runtime.http.headersTimeoutMs': '0' }], + ['runtime.http.requestTimeoutMs', {}, { 'runtime.http.requestTimeoutMs': 'invalid' }], + ['runtime.http.keepAliveTimeoutMs', {}, { 'runtime.http.keepAliveTimeoutMs': '60001' }], ['requestLogging.successSampleRate', {}, { 'requestLogging.successSampleRate': '1.1' }] ])('rejects an out-of-range or malformed %s value', (path, env, values) => { expect(() => resolve(values, env)).toThrow(path); @@ -202,6 +218,12 @@ describe('runtimeConfig', () => { 'runtime.shutdownTimeoutMs': '30000' } ], + [ + 'runtime.http.headersTimeoutMs', + { + 'runtime.http.headersTimeoutMs': '16000' + } + ], [ 'rateLimiting.search.maxRequests', { diff --git a/utils/searchPerformance.js b/utils/searchPerformance.js index 7dfa764..95caf0b 100644 --- a/utils/searchPerformance.js +++ b/utils/searchPerformance.js @@ -1,5 +1,10 @@ +const { monitorEventLoopDelay } = require('perf_hooks'); const searchProvider = require('./searchProvider'); +const EVENT_LOOP_DELAY_RESOLUTION_MS = 20; +const RESPONSE_JSON_INSTRUMENTED = Symbol('responseJsonInstrumented'); +let eventLoopDelayMonitor; + function markTime() { return process.hrtime.bigint(); } @@ -20,6 +25,61 @@ function getCacheStatus(metrics, postFilters) { return 'miss'; } +function nanosecondsToMs(value) { + return Number.isFinite(value) ? value / 1e6 : undefined; +} + +function getEventLoopDelayMonitor() { + if (!eventLoopDelayMonitor) { + eventLoopDelayMonitor = monitorEventLoopDelay({ + resolution: EVENT_LOOP_DELAY_RESOLUTION_MS + }); + eventLoopDelayMonitor.enable(); + } + + return eventLoopDelayMonitor; +} + +function instrumentJsonResponse(res, metrics) { + if (typeof res?.json !== 'function' || res[RESPONSE_JSON_INSTRUMENTED]) return; + + const originalJson = res.json; + res[RESPONSE_JSON_INSTRUMENTED] = true; + res.json = function instrumentedJson(body) { + const serializationStart = markTime(); + const result = originalJson.call(this, body); + metrics.responseSerializationElapsedMs = elapsedMs(serializationStart); + + const contentLength = Number(this.getHeader?.('content-length')); + if (Number.isFinite(contentLength) && contentLength >= 0) { + metrics.responseBytes = contentLength; + } + + return result; + }; +} + +function beginSearchRequest(metrics, res) { + if (!searchProvider.shouldLogSearchPerformance()) return false; + + metrics.heapUsedStartBytes = process.memoryUsage().heapUsed; + getEventLoopDelayMonitor(); + instrumentJsonResponse(res, metrics); + return true; +} + +function completeSearchRequest(metrics) { + if (metrics.heapUsedStartBytes === undefined) return; + + const heapUsedBytes = process.memoryUsage().heapUsed; + const monitor = getEventLoopDelayMonitor(); + metrics.heapUsedBytes = heapUsedBytes; + metrics.heapUsedDeltaBytes = heapUsedBytes - metrics.heapUsedStartBytes; + metrics.eventLoopDelayMeanMs = nanosecondsToMs(monitor.mean); + metrics.eventLoopDelayP99Ms = nanosecondsToMs(monitor.percentile(99)); + metrics.eventLoopDelayMaxMs = nanosecondsToMs(monitor.max); +} + function logSearchPerformance(metrics, searchOptions, responseOptions, postFilters) { const outcome = metrics.outcome ?? 'success'; const includeDetailedTimings = searchProvider.shouldLogSearchPerformance(); @@ -57,6 +117,15 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte candidateCount: metrics.candidateCount, filteredCount: metrics.filteredCount, returnedCount: metrics.returnedCount, + cachedResultCveIdCount: metrics.cachedResultCveIdCount, + cacheState: metrics.cacheState, + responseBytes: metrics.responseBytes, + responseSerializationElapsedMs: roundMs(metrics.responseSerializationElapsedMs), + heapUsedBytes: metrics.heapUsedBytes, + heapUsedDeltaBytes: metrics.heapUsedDeltaBytes, + eventLoopDelayMeanMs: roundMs(metrics.eventLoopDelayMeanMs), + eventLoopDelayP99Ms: roundMs(metrics.eventLoopDelayP99Ms), + eventLoopDelayMaxMs: roundMs(metrics.eventLoopDelayMaxMs), openSearchTookMs: metrics.openSearchTookMs, pageFetchTookMs: metrics.pageFetchTookMs, openSearchElapsedMs: roundMs(metrics.openSearchElapsedMs), @@ -72,10 +141,18 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte console.log(JSON.stringify(event)); } +function resetForTests() { + eventLoopDelayMonitor?.disable(); + eventLoopDelayMonitor = undefined; +} + module.exports = { + beginSearchRequest, + completeSearchRequest, elapsedMs, getCacheStatus, logSearchPerformance, markTime, + resetForTests, roundMs }; diff --git a/utils/searchPerformance.test.js b/utils/searchPerformance.test.js index 4d93959..52a7036 100644 --- a/utils/searchPerformance.test.js +++ b/utils/searchPerformance.test.js @@ -13,6 +13,7 @@ describe('searchPerformance', () => { }); afterEach(() => { + searchPerformance.resetForTests(); jest.restoreAllMocks(); }); @@ -33,6 +34,18 @@ describe('searchPerformance', () => { candidateCount: 10000, filteredCount: 4, returnedCount: 1, + cachedResultCveIdCount: 4, + cacheState: { + entryCount: 2, + cachedCveIdCount: 8 + }, + responseBytes: 2048, + responseSerializationElapsedMs: 1.234, + heapUsedBytes: 2000000, + heapUsedDeltaBytes: 50000, + eventLoopDelayMeanMs: 2.345, + eventLoopDelayP99Ms: 8.765, + eventLoopDelayMaxMs: 12.345, openSearchTookMs: 97, pageFetchTookMs: 8, openSearchElapsedMs: 123.456, @@ -84,6 +97,18 @@ describe('searchPerformance', () => { candidateCount: 10000, filteredCount: 4, returnedCount: 1, + cachedResultCveIdCount: 4, + cacheState: { + entryCount: 2, + cachedCveIdCount: 8 + }, + responseBytes: 2048, + responseSerializationElapsedMs: 1.23, + heapUsedBytes: 2000000, + heapUsedDeltaBytes: 50000, + eventLoopDelayMeanMs: 2.35, + eventLoopDelayP99Ms: 8.77, + eventLoopDelayMaxMs: 12.35, openSearchTookMs: 97, pageFetchTookMs: 8, openSearchElapsedMs: 123.46, @@ -118,6 +143,40 @@ describe('searchPerformance', () => { expect(console.log).not.toHaveBeenCalled(); }); + test('does not instrument response or process resources when detailed logging is disabled', () => { + mockShouldLogSearchPerformance.mockReturnValue(false); + const metrics = {}; + const json = jest.fn(); + const res = { json }; + + expect(searchPerformance.beginSearchRequest(metrics, res)).toBe(false); + searchPerformance.completeSearchRequest(metrics); + + expect(res.json).toBe(json); + expect(metrics).toStrictEqual({}); + }); + + test('captures response size and process resource metrics only when detailed logging is enabled', () => { + mockShouldLogSearchPerformance.mockReturnValue(true); + const metrics = {}; + const res = { + getHeader: jest.fn(() => '42'), + json: jest.fn(() => res) + }; + + searchPerformance.beginSearchRequest(metrics, res); + res.json({ status: 'ok' }); + searchPerformance.completeSearchRequest(metrics); + + expect(metrics).toMatchObject({ + responseBytes: 42, + responseSerializationElapsedMs: expect.any(Number), + heapUsedBytes: expect.any(Number), + heapUsedDeltaBytes: expect.any(Number) + }); + expect(metrics.responseSerializationElapsedMs).toBeGreaterThanOrEqual(0); + }); + test('logSearchPerformance always emits sanitized failure outcomes', () => { mockShouldLogSearchPerformance.mockReturnValue(false); diff --git a/utils/searchPostFilterCache.js b/utils/searchPostFilterCache.js index adfd906..ee90e58 100644 --- a/utils/searchPostFilterCache.js +++ b/utils/searchPostFilterCache.js @@ -77,6 +77,20 @@ function set(cacheKey, value, now = Date.now()) { } } +function getState(now = Date.now()) { + for (const [key, entry] of entries) { + if (entry.expiresAt <= now) entries.delete(key); + } + + return { + entryCount: entries.size, + cachedCveIdCount: [...entries.values()].reduce( + (total, entry) => total + (entry.value?.filteredCveIds?.length ?? 0), + 0 + ) + }; +} + function clear() { entries.clear(); } @@ -85,6 +99,7 @@ module.exports = { buildCacheKey, clear, get, + getState, normalizeForKey, set }; diff --git a/utils/searchPostFilterCache.test.js b/utils/searchPostFilterCache.test.js index 4fd81c6..7d2f41d 100644 --- a/utils/searchPostFilterCache.test.js +++ b/utils/searchPostFilterCache.test.js @@ -143,4 +143,30 @@ describe('searchPostFilterCache', () => { expect(searchPostFilterCache.get(key, 1001)).toBe(value); expect(searchPostFilterCache.get(key, 61001)).toBeNull(); }); + + test('reports entry and cached CVE ID counts while pruning expired entries', () => { + searchPostFilterCache.set( + 'first', + { + filteredCveIds: ['CVE-2024-0001', 'CVE-2024-0002'] + }, + 1000 + ); + searchPostFilterCache.set( + 'second', + { + filteredCveIds: ['CVE-2024-0003'] + }, + 2000 + ); + + expect(searchPostFilterCache.getState(3000)).toStrictEqual({ + entryCount: 2, + cachedCveIdCount: 3 + }); + expect(searchPostFilterCache.getState(62000)).toStrictEqual({ + entryCount: 0, + cachedCveIdCount: 0 + }); + }); }); diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index dda5ca9..4f86dd9 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -71,7 +71,11 @@ function buildSearchRequest(params) { return request; } -function buildOpenSearchBody(searchOptions, queryStringObjects = [], { sourceIncludes, trackTotalHits = true } = {}) { +function buildOpenSearchBody( + searchOptions, + queryStringObjects = [], + { includeSort = true, sourceIncludes, trackTotalHits = true } = {} +) { const normalizedQueryStringObjects = Array.isArray(queryStringObjects) ? queryStringObjects : []; const queryObj = { must: [...normalizedQueryStringObjects], @@ -81,12 +85,14 @@ function buildOpenSearchBody(searchOptions, queryStringObjects = [], { sourceInc query: { bool: queryObj }, - sort: buildDeterministicSort(normalizedQueryStringObjects), track_total_hits: trackTotalHits, size: searchOptions.resultsPerPage, from: searchOptions.from }; + if (includeSort) { + body.sort = buildDeterministicSort(normalizedQueryStringObjects); + } if (sourceIncludes?.length) { body._source = sourceIncludes; } @@ -101,18 +107,22 @@ function buildCpeCandidateSearchBody(searchOptions, queryStringObjects = []) { }); } -function buildCveIdsSearchRequest(cveIds) { +function buildCveIdsSearchBody(cveIds) { const filter = buildCveIdsFilter(cveIds.join(','), constants.FIELDS.cveIds); - return { - searchOptions: { + return buildOpenSearchBody( + { filters: filter ? [filter] : [], rangeObjects: [], resultsPerPage: Math.max(cveIds.length, 1), from: 0 }, - queryStringObjects: [] - }; + [], + { + includeSort: false, + trackTotalHits: false + } + ); } function buildPostFilters(params) { @@ -401,29 +411,10 @@ function buildCpeNameFilter(cpeName) { const wildcardVersionCriteria = buildCpeCriteriaFilter(criteriaKeywordField, parsedCpe, 4, { versionMode: 'wildcard' }); - const rangeFields = [ - fields.versionStartIncluding, - fields.versionStartExcluding, - fields.versionEndIncluding, - fields.versionEndExcluding - ]; return { bool: { - should: [ - exactVersionCriteria, - { - bool: { - filter: [wildcardVersionCriteria], - must_not: rangeFields.map(buildExistsFilter) - } - }, - { - bool: { - filter: [wildcardVersionCriteria, buildHasAnyRangeFieldFilter(rangeFields)] - } - } - ], + should: [exactVersionCriteria, wildcardVersionCriteria], minimum_should_match: 1 } }; @@ -489,23 +480,6 @@ function buildCpeCriteriaRegexpFilter(field, value) { }; } -function buildExistsFilter(field) { - return { - exists: { - field - } - }; -} - -function buildHasAnyRangeFieldFilter(rangeFields) { - return { - bool: { - should: rangeFields.map(buildExistsFilter), - minimum_should_match: 1 - } - }; -} - function buildCvssRegexpFilter(inputVectorString, param, fieldSpec) { const parsed = cvss.parseCvssMetrics(inputVectorString, param); if (!parsed.valid) { @@ -593,7 +567,7 @@ function cvssV2SeverityToRange(sev) { module.exports = { buildCpeCandidateSearchBody, - buildCveIdsSearchRequest, + buildCveIdsSearchBody, buildOpenSearchBody, buildQueryStrings, buildSearchRequest, diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index f930f38..ff7cfa1 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -353,21 +353,50 @@ describe('searchQueryBuilder buildSearchRequest()', () => { }); }); - test('buildCveIdsSearchRequest builds exact page fetch request for selected CVE IDs', () => { - expect(queryBuilder.buildCveIdsSearchRequest(['CVE-2024-0002', 'CVE-2024-0001'])).toStrictEqual({ - searchOptions: { - filters: [ - { - terms: { - 'cveMetadata.cveId.keyword': ['CVE-2024-0001', 'CVE-2024-0002'] + test('buildCveIdsSearchBody omits total counting and sorting for locally ordered page fetches', () => { + expect(queryBuilder.buildCveIdsSearchBody(['CVE-2024-0002', 'CVE-2024-0001'])).toStrictEqual({ + query: { + bool: { + must: [], + filter: [ + { + terms: { + 'cveMetadata.cveId.keyword': ['CVE-2024-0001', 'CVE-2024-0002'] + } } - } - ], - rangeObjects: [], - resultsPerPage: 2, - from: 0 + ] + } }, - queryStringObjects: [] + track_total_hits: false, + size: 2, + from: 0 + }); + }); + + test('buildOpenSearchBody can omit deterministic sorting for bounded internal fetches', () => { + expect( + queryBuilder.buildOpenSearchBody( + { + filters: [], + rangeObjects: [], + resultsPerPage: 1, + from: 0 + }, + [], + { + includeSort: false + } + ) + ).toStrictEqual({ + query: { + bool: { + must: [], + filter: [] + } + }, + track_total_hits: true, + size: 1, + from: 0 }); }); }); @@ -532,7 +561,7 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds CPE Name criteria filters for exact and range match strings.', () => { + test('getFilterFields() builds exact-version and wildcard-version CPE Name candidate filters.', () => { const output = queryBuilder.getFilterFields({ cpeName: 'cpe:2.3:a:artifex:ghostscript:10.05.1:*:*:*:*:*:*:*' }); @@ -551,124 +580,28 @@ describe('searchQueryBuilder getFilterFields() function', () => { 'cpe:2.3:*:*:*:10.05.1:' ]) ); - expect(output[0].bool.should[1]).toStrictEqual({ - bool: { - filter: [ - prefixShould([ - 'cpe:2.3:a:artifex:ghostscript:*:', - 'cpe:2.3:a:artifex:*:*:', - 'cpe:2.3:a:*:ghostscript:*:', - 'cpe:2.3:a:*:*:*:', - 'cpe:2.3:*:artifex:ghostscript:*:', - 'cpe:2.3:*:artifex:*:*:', - 'cpe:2.3:*:*:ghostscript:*:', - 'cpe:2.3:*:*:*:*:' - ]) - ], - must_not: [ - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartExcluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndExcluding' - } - } - ] - } - }); - expect(output[0].bool.should[2]).toStrictEqual({ - bool: { - filter: [ - prefixShould([ - 'cpe:2.3:a:artifex:ghostscript:*:', - 'cpe:2.3:a:artifex:*:*:', - 'cpe:2.3:a:*:ghostscript:*:', - 'cpe:2.3:a:*:*:*:', - 'cpe:2.3:*:artifex:ghostscript:*:', - 'cpe:2.3:*:artifex:*:*:', - 'cpe:2.3:*:*:ghostscript:*:', - 'cpe:2.3:*:*:*:*:' - ]), - { - bool: { - should: [ - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartExcluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndExcluding' - } - } - ], - minimum_should_match: 1 - } - } - ] - } - }); + expect(output[0].bool.should[1]).toStrictEqual( + prefixShould([ + 'cpe:2.3:a:artifex:ghostscript:*:', + 'cpe:2.3:a:artifex:*:*:', + 'cpe:2.3:a:*:ghostscript:*:', + 'cpe:2.3:a:*:*:*:', + 'cpe:2.3:*:artifex:ghostscript:*:', + 'cpe:2.3:*:artifex:*:*:', + 'cpe:2.3:*:*:ghostscript:*:', + 'cpe:2.3:*:*:*:*:' + ]) + ); + expect(output[0].bool.should).toHaveLength(2); }); - test('getFilterFields() avoids lexicographic keyword range clauses for CPE version boundaries.', () => { + test('getFilterFields() leaves CPE version range applicability to semantic post-filtering.', () => { const output = queryBuilder.getFilterFields({ cpeName: 'cpe:2.3:a:artifex:ghostscript:10.05.1:*:*:*:*:*:*:*' }); - const rangeBranchFilters = output[0].bool.should[2].bool.filter; - - expect(rangeBranchFilters).toHaveLength(2); - expect(rangeBranchFilters[1]).toStrictEqual({ - bool: { - should: [ - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartExcluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndIncluding' - } - }, - { - exists: { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndExcluding' - } - } - ], - minimum_should_match: 1 - } - }); + expect(output[0].bool.should).toHaveLength(2); + expect(JSON.stringify(output)).not.toContain('"exists"'); expect(JSON.stringify(output)).not.toContain('.keyword":{"lte"'); expect(JSON.stringify(output)).not.toContain('.keyword":{"lt"'); expect(JSON.stringify(output)).not.toContain('.keyword":{"gte"'); From 63959b4a9ff919c7a7db19b018f513161feecdeb Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Thu, 30 Jul 2026 18:33:22 -0400 Subject: [PATCH 17/33] Harden delivery and OpenSearch contracts --- .github/dependabot.yml | 19 + .github/workflows/opensearch-contract.yml | 46 + .github/workflows/quality.yml | 36 +- .trivyignore | 2 + Dockerfile | 7 +- Dockerfile.local | 7 +- README.md | 22 +- api-docs/openapi.json | 10 +- api-docs/openapi.test.js | 13 + config/runtime-config.test.js | 44 +- controllers/middleware.js | 2 +- package-lock.json | 2872 +++++++++++++-------- package.json | 3 +- routes/index.js | 4 +- routes/swagger.js | 6 +- test/contract/opensearch.js | 154 +- test/contract/opensearch.test.js | 172 ++ utils/constants.js | 3 + 18 files changed, 2255 insertions(+), 1167 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/opensearch-contract.yml create mode 100644 .trivyignore create mode 100644 test/contract/opensearch.test.js diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..083cace --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +version: 2 +updates: + - package-ecosystem: npm + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + + - package-ecosystem: docker + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 diff --git a/.github/workflows/opensearch-contract.yml b/.github/workflows/opensearch-contract.yml new file mode 100644 index 0000000..51ba093 --- /dev/null +++ b/.github/workflows/opensearch-contract.yml @@ -0,0 +1,46 @@ +name: OpenSearch Contract + +on: + workflow_dispatch: + inputs: + environment: + description: Protected GitHub environment containing the OpenSearch contract-test secrets + required: true + type: environment + +permissions: + contents: read + +concurrency: + group: search-api-opensearch-contract-${{ inputs.environment }} + cancel-in-progress: false + +jobs: + contract: + name: Read-only OpenSearch Contract + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: ${{ inputs.environment }} + + env: + NODE_ENV: production + NODE_CONFIG_ENV: prod + OpenSearchDomainEndpoint: ${{ secrets.OPENSEARCH_DOMAIN_ENDPOINT }} + OpenSearchCveIndex: ${{ secrets.OPENSEARCH_CVE_INDEX }} + OpenSearchAllowUnknownSslCerts: ${{ vars.OPENSEARCH_ALLOW_UNKNOWN_SSL_CERTS || 'false' }} + + steps: + - name: Check out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 24.14.1 + cache: npm + + - name: Install production dependencies + run: npm ci --omit=dev + + - name: Run read-only contract checks + run: npm run test:opensearch-contract diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 7671e70..1cf9ce2 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -2,15 +2,36 @@ name: Search API Quality on: pull_request: + push: + branches: + - dev + - main + merge_group: permissions: contents: read concurrency: - group: search-api-quality-${{ github.event.pull_request.number }} + group: search-api-quality-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: + dependency-review: + name: Dependency Review + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Review dependency changes + uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + fail-on-severity: high + fail-on-scopes: runtime, development + quality: name: Search API Quality runs-on: ubuntu-latest @@ -44,6 +65,19 @@ jobs: - name: Build production image run: docker build --tag cve-search-api:pr . + - name: Scan production image + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + scan-type: image + image-ref: cve-search-api:pr + format: table + exit-code: 1 + ignore-unfixed: true + severity: CRITICAL,HIGH + scanners: vuln + trivyignores: .trivyignore + version: v0.69.3 + - name: Start production image run: | docker run --detach \ diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 0000000..e4830c8 --- /dev/null +++ b/.trivyignore @@ -0,0 +1,2 @@ +# cve-core transitively pins adm-zip below 0.6.0. Remediation belongs in cve-core. +GHSA-xcpc-8h2w-3j85 diff --git a/Dockerfile b/Dockerfile index 10d044f..7f940b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,6 @@ # syntax=docker/dockerfile:1 -ARG NODE_VERSION=24.14.1 -ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot - -FROM node:${NODE_VERSION} AS deps +FROM node:24.14.1@sha256:80fc934952c8f1b2b4d39907af7211f8a9fff1a4c2cf673fb49099292c251cec AS deps ENV NODE_ENV=production \ NODE_CONFIG_ENV=prod @@ -17,7 +14,7 @@ COPY package.json package-lock.json ./ RUN npm ci --omit=dev \ && npm cache clean --force -FROM ${DISTROLESS_NODE_IMAGE} AS runtime +FROM gcr.io/distroless/nodejs24-debian13:nonroot@sha256:af85d11ce7ef10172855a6e3649e3e8125b1b9e3ca41849ec2918036f05cb212 AS runtime ENV NODE_ENV=production \ NODE_CONFIG_ENV=prod diff --git a/Dockerfile.local b/Dockerfile.local index a0977de..5a200d8 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,9 +1,6 @@ # syntax=docker/dockerfile:1 -ARG NODE_VERSION=24.14.1 -ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot - -FROM node:${NODE_VERSION} AS deps +FROM node:24.14.1@sha256:80fc934952c8f1b2b4d39907af7211f8a9fff1a4c2cf673fb49099292c251cec AS deps ENV NODE_ENV=production \ NODE_CONFIG_ENV=prod @@ -22,7 +19,7 @@ RUN --mount=type=secret,id=npm_ca,required=false \ fi \ && npm cache clean --force -FROM ${DISTROLESS_NODE_IMAGE} AS runtime +FROM gcr.io/distroless/nodejs24-debian13:nonroot@sha256:af85d11ce7ef10172855a6e3649e3e8125b1b9e3ca41849ec2918036f05cb212 AS runtime ENV NODE_ENV=production \ NODE_CONFIG_ENV=prod diff --git a/README.md b/README.md index b3d9e6b..0e5f2c8 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ Requests that exceed a limit return HTTP 400 before contacting OpenSearch. These CVSS vector filters accept full or partial vectors containing at least one unique `KEY:VALUE` metric token. An optional vector header must identify a version supported by the selected parameter. Requested metric tokens can appear in any order, but all tokens must occur in the same stored vector string. CVSS and CWE filters search CNA-provided data only; ADP-provided enrichment is not currently included. -Publication and last-modified date filters support one-sided ranges. When both values are supplied, `pubStartDate` must be less than or equal to `pubEndDate`, and `lastModStartDate` must be less than or equal to `lastModEndDate`. +Publication and last-modified date filters support one-sided ranges. Values must use the exact UTC format `yyyy-MM-ddTHH:mm:ss.SSSZ`, including three fractional-second digits and the trailing `Z`. When both values are supplied, `pubStartDate` must be less than or equal to `pubEndDate`, and `lastModStartDate` must be less than or equal to `lastModEndDate`. ### Search Ordering and Pagination @@ -285,7 +285,7 @@ While enabled, `POST /webSearch` requires a non-empty JSON `query` string. It al ## Swagger API docs -Swagger API docs are automatically generated on `docker compose` or `npm run dev`, and can be regenerated manually using: +The runtime container and Docker Compose serve the committed `api-docs/openapi.json` artifact. They do not regenerate it during image startup. `npm run dev` regenerates the artifact before starting the development server, and it can be regenerated manually using: ```bash npm run swagger-autogen @@ -310,7 +310,9 @@ npm run coverage npm run quality ``` -Pull requests run the quality gate and independently build the production `Dockerfile`. The container job starts the built image, checks process liveness and the packaged OpenAPI document, and verifies the runtime user and command. It intentionally does not call `/api/ready-check` or `/search`, because pull-request CI does not own or provision an OpenSearch index. +Pull requests, merge-queue validation, and pushes to `dev` or `main` run the quality gate and independently build the production `Dockerfile`. The container job scans the built image for fixed high and critical vulnerabilities, starts it, checks process liveness and the packaged OpenAPI document, and verifies the runtime user and command. It intentionally does not call `/api/ready-check` or `/search`, because the standard CI workflow does not own or provision an OpenSearch index. + +Pull requests also run GitHub dependency review and reject newly introduced high or critical vulnerabilities in runtime or development dependencies. The production image scan excludes only `GHSA-xcpc-8h2w-3j85`, an `adm-zip` advisory inherited from the separately owned `cve-core` dependency. New advisories are not covered by that exception. An opt-in, read-only OpenSearch contract check is available for an approved environment that can reach an existing index: @@ -318,9 +320,15 @@ An opt-in, read-only OpenSearch contract check is available for an approved envi npm run test:opensearch-contract ``` -Set `OpenSearchDomainEndpoint` and `OpenSearchCveIndex` in the environment or local `.env` before running it. `OpenSearchAllowUnknownSslCerts` remains optional and should be enabled only for local development against a self-signed certificate. The contract check sends a minimal readiness query and one single-hit search, validates the response shape, closes its OpenSearch client, and emits sanitized JSON output. It never creates, updates, maps, or deletes indexes. +Set `OpenSearchDomainEndpoint` and `OpenSearchCveIndex` in the environment or local `.env` before running it. `OpenSearchAllowUnknownSslCerts` remains optional and should be enabled only for local development against a self-signed certificate. The contract check sends a minimal readiness query followed by representative bounded searches, validates their response shapes, closes its OpenSearch client, and emits sanitized JSON output. It never creates, updates, maps, or deletes indexes. + +The contract check is not part of public pull-request CI. The `OpenSearch Contract` GitHub Actions workflow runs it only through a manual dispatch against a selected protected GitHub environment. Configure that environment with: -The contract check is not part of public pull-request CI. A future automated invocation should use a protected GitHub environment or self-hosted runner with network access to an approved test index. Connection values and credentials must remain in environment secrets rather than source files or workflow text. +- `OPENSEARCH_DOMAIN_ENDPOINT` secret: the approved OpenSearch endpoint, including deployment-managed credentials when required +- `OPENSEARCH_CVE_INDEX` secret: the existing CVE index name +- `OPENSEARCH_ALLOW_UNKNOWN_SSL_CERTS` variable: optional and normally `false`; use `true` only for an approved non-production endpoint with a self-signed certificate + +The selected runner must have network access to the endpoint. Change the workflow to an approved self-hosted runner before use when the index is not reachable from GitHub-hosted runners. The workflow performs readiness, source projection, exact-total, CVE ID keyword-subfield, keyword, publication-date, CVSS, and CPE candidate checks. Every query is read-only and returns at most one record. Connection values and credentials remain in environment secrets rather than source files or workflow text. The validation tiers have separate purposes: @@ -329,6 +337,8 @@ The validation tiers have separate purposes: - The optional OpenSearch contract check verifies that the installed client can communicate with an externally managed index using the response contract expected by this application. - Load and performance testing remain in the separate automation performance-testing repository. +Dependabot checks npm packages, Docker base-image digests, and GitHub Actions weekly. Both Dockerfiles pin the Node build image and Distroless runtime image to immutable multi-architecture digests; Dependabot updates those references through reviewable pull requests. + ### Operational Logging Every request receives an `X-Request-ID` response header. A caller-provided `X-Request-ID` is preserved when it contains only letters, numbers, `.`, `_`, `:`, or `-` and is no longer than 128 characters; otherwise the API generates a UUID. The same value appears as `request-id` in access logs and `requestId` in structured `/search` events. @@ -423,4 +433,4 @@ When building locally on a network that requires a corporate or self-signed CA b docker build -f Dockerfile.local --secret id=npm_ca,src=/path/to/ca.crt -t cve-search-api:runtime-smoke . ``` -The final runtime image uses Google Distroless Node, so it does not include a shell, package manager, or command lookup tools. +The final runtime image uses Google Distroless Node, so it does not include a shell, package manager, or command lookup tools. Both build stages are pinned by multi-architecture digest and are updated through Dependabot rather than mutable registry tags. diff --git a/api-docs/openapi.json b/api-docs/openapi.json index b889eed..8a0e900 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -3,7 +3,7 @@ "info": { "version": "0.4.0-beta", "title": "CVE Search API", - "description": "CVE Search API to retrieve all matching CVE records after applying request inputs as filters.", + "description": "CVE Search API for retrieving paginated CVE records after applying request inputs as filters.", "contact": { "name": "CVE Overview", "url": "https://www.cve.org/" @@ -20,8 +20,8 @@ "tags": [ "CVE Record" ], - "summary": "Retrieves all matching CVE records after applying the JSON request body as filters", - "description": "

Expected Behavior

Retrieves all matching CVE records after applying the JSON request body as filters

", + "summary": "Retrieves a page of CVE records after applying the JSON request body as filters", + "description": "

Expected Behavior

Retrieves one page of matching CVE records after applying the JSON request body as filters. Pagination metadata reports the accessible result window.

", "operationId": "search", "responses": { "200": { @@ -509,24 +509,28 @@ "lastModStartDate": { "type": "string", "format": "date-time", + "pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])T([01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d\\.\\d{3}Z$", "example": "2024-01-01T00:00:00.000Z", "description": "Returns CVE records with dateUpdated timestamps on or after this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When lastModEndDate is also provided, this value must be less than or equal to it." }, "lastModEndDate": { "type": "string", "format": "date-time", + "pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])T([01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d\\.\\d{3}Z$", "example": "2024-12-31T23:59:59.999Z", "description": "Returns CVE records with dateUpdated timestamps on or before this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When lastModStartDate is also provided, that value must be less than or equal to this one." }, "pubStartDate": { "type": "string", "format": "date-time", + "pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])T([01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d\\.\\d{3}Z$", "example": "2024-01-01T00:00:00.000Z", "description": "Returns CVE records with datePublished timestamps on or after this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When pubEndDate is also provided, this value must be less than or equal to it." }, "pubEndDate": { "type": "string", "format": "date-time", + "pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])T([01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d\\.\\d{3}Z$", "example": "2024-12-31T23:59:59.999Z", "description": "Returns CVE records with datePublished timestamps on or before this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ. When pubStartDate is also provided, that value must be less than or equal to this one." }, diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 5162c29..e51e77a 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -136,6 +136,10 @@ describe('OpenAPI contract', () => { expect(schema.properties.cvssV3Metrics.description).toContain('at least one unique KEY:VALUE token'); expect(schema.properties.pubStartDate.description).toContain('less than or equal'); expect(schema.properties.lastModEndDate.description).toContain('less than or equal'); + ['lastModStartDate', 'lastModEndDate', 'pubStartDate', 'pubEndDate'].forEach((field) => { + expect(schema.properties[field].format).toBe('date-time'); + expect(schema.properties[field].pattern).toBe(constants.DATE_TIME_PATTERN); + }); expect(Object.keys(mediaType.examples)).toStrictEqual([ 'allParameters', 'keywordSearch', @@ -180,6 +184,15 @@ describe('OpenAPI contract', () => { expect(openapi.paths['/search'].post.responses['429'].description).toContain('waiter limit'); }); + test('describes POST /search as a paginated result operation', () => { + const operation = openapi.paths['/search'].post; + + expect(operation.summary).toContain('a page of CVE records'); + expect(operation.description).toContain('one page of matching CVE records'); + expect(openapi.info.description).toContain('paginated CVE records'); + expect(operation.summary).not.toContain('all matching CVE records'); + }); + test('documents fail-closed request and dependency responses for POST /search', () => { const responses = openapi.paths['/search'].post.responses; diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index 1dc7f66..7f24e84 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -9,9 +9,10 @@ describe('runtime container and compose configuration', () => { test('Dockerfile keeps AWS-compatible production runtime hardening invariants', () => { const dockerfile = readProjectFile('Dockerfile'); - expect(dockerfile).toContain('ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot'); - expect(dockerfile).toContain('FROM node:${NODE_VERSION} AS deps'); - expect(dockerfile).toContain('FROM ${DISTROLESS_NODE_IMAGE} AS runtime'); + expect(dockerfile).toMatch(/^FROM node:24\.14\.1@sha256:[a-f0-9]{64} AS deps$/m); + expect(dockerfile).toMatch( + /^FROM gcr\.io\/distroless\/nodejs24-debian13:nonroot@sha256:[a-f0-9]{64} AS runtime$/m + ); expect(dockerfile).toContain('NODE_CONFIG_ENV=prod'); expect(dockerfile).toMatch(/^RUN npm ci --omit=dev \\/m); expect(dockerfile).toMatch(/^COPY --chown=65532:65532 config \.\/config\//m); @@ -34,9 +35,10 @@ describe('runtime container and compose configuration', () => { test('local Dockerfile keeps npm CA secret mount for local builds', () => { const dockerfile = readProjectFile('Dockerfile.local'); - expect(dockerfile).toContain('ARG DISTROLESS_NODE_IMAGE=gcr.io/distroless/nodejs24-debian13:nonroot'); - expect(dockerfile).toContain('FROM node:${NODE_VERSION} AS deps'); - expect(dockerfile).toContain('FROM ${DISTROLESS_NODE_IMAGE} AS runtime'); + expect(dockerfile).toMatch(/^FROM node:24\.14\.1@sha256:[a-f0-9]{64} AS deps$/m); + expect(dockerfile).toMatch( + /^FROM gcr\.io\/distroless\/nodejs24-debian13:nonroot@sha256:[a-f0-9]{64} AS runtime$/m + ); expect(dockerfile).toContain('NODE_CONFIG_ENV=prod'); expect(dockerfile).toMatch(/^RUN --mount=type=secret,id=npm_ca,required=false/m); expect(dockerfile).toContain('if [ -s /run/secrets/npm_ca ]; then'); @@ -77,11 +79,19 @@ describe('runtime container and compose configuration', () => { expect(dockerignore).toContain('**/reports'); }); - test('pull request CI builds and smoke tests the production runtime image', () => { + test('CI builds, scans, and smoke tests the production runtime image', () => { const workflow = readProjectFile('.github', 'workflows', 'quality.yml'); + expect(workflow).toContain('pull_request:'); + expect(workflow).toContain('push:'); + expect(workflow).toContain('merge_group:'); + expect(workflow).toContain('name: Dependency Review'); + expect(workflow).toContain('actions/dependency-review-action@'); expect(workflow).toContain('name: Runtime Container'); expect(workflow).toContain('docker build --tag cve-search-api:pr .'); + expect(workflow).toContain('aquasecurity/trivy-action@'); + expect(workflow).toContain('trivyignores: .trivyignore'); + expect(workflow).toContain('version: v0.69.3'); expect(workflow).toContain('http://127.0.0.1:4100/api/health-check'); expect(workflow).toContain('http://127.0.0.1:4100/api-docs/openapi.json'); expect(workflow).toContain(`--format '{{.Config.User}}'`); @@ -89,6 +99,26 @@ describe('runtime container and compose configuration', () => { expect(workflow).not.toContain('/api/ready-check'); }); + test('manual OpenSearch contract workflow uses a protected environment and secrets', () => { + const workflow = readProjectFile('.github', 'workflows', 'opensearch-contract.yml'); + + expect(workflow).toContain('workflow_dispatch:'); + expect(workflow).toContain('type: environment'); + expect(workflow).toContain('environment: ${{ inputs.environment }}'); + expect(workflow).toContain('${{ secrets.OPENSEARCH_DOMAIN_ENDPOINT }}'); + expect(workflow).toContain('${{ secrets.OPENSEARCH_CVE_INDEX }}'); + expect(workflow).toContain('npm run test:opensearch-contract'); + expect(workflow).not.toMatch(/https?:\/\//); + }); + + test('Dependabot tracks npm, Docker, and GitHub Actions dependencies', () => { + const dependabot = readProjectFile('.github', 'dependabot.yml'); + + expect(dependabot).toContain('package-ecosystem: npm'); + expect(dependabot).toContain('package-ecosystem: docker'); + expect(dependabot).toContain('package-ecosystem: github-actions'); + }); + test('server-only compose file joins the existing OpenSearch 3.5 network', () => { const compose = readProjectFile('docker-compose.yaml'); diff --git a/controllers/middleware.js b/controllers/middleware.js index f210169..a822cc3 100644 --- a/controllers/middleware.js +++ b/controllers/middleware.js @@ -58,7 +58,7 @@ const nonStringSearchParams = [ ]; const nonEmptyStringParams = constants.SEARCH_QUERY_PARAMS.filter((param) => !nonStringSearchParams.includes(param)); -const dateTimeFormat = /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])T([01]\d|2[0-3]):[0-5]\d:[0-5]\d\.\d{3}Z$/; +const dateTimeFormat = new RegExp(constants.DATE_TIME_PATTERN); function isTrueParam(value) { return value === true || String(value).toLowerCase() === 'true'; diff --git a/package-lock.json b/package-lock.json index 5a06297..ca79573 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,7 @@ "@eslint/js": "^10.0.1", "eslint": "^10.6.0", "globals": "^17.7.0", - "jest": "^28.1.3", + "jest": "^30.4.2", "nodemon": "^3.1.9", "prettier": "^3.9.3", "swagger-autogen": "^2.23.7" @@ -83,31 +83,6 @@ "url": "https://opencollective.com/babel" } }, - "node_modules/@babel/core/node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@babel/core/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@babel/core/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -205,9 +180,9 @@ } }, "node_modules/@babel/helper-plugin-utils": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", - "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", + "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", "dev": true, "license": "MIT", "engines": { @@ -330,13 +305,13 @@ } }, "node_modules/@babel/plugin-syntax-import-attributes": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.28.6.tgz", - "integrity": "sha512-jiLC0ma9XkQT3TKJ9uYvlakm66Pamywo+qwL+oL8HJOvc6TWdZXVfhqJr8CCzbSGUAbDOzlGHJC1U+vRfLQDvw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.29.7.tgz", + "integrity": "sha512-zGYcYfq/WmZ4V+kBIXQon9dSSc8ircGZqw9ZaNhhGj9nZkeBu1jHLBDQqYYi5WA9uawvA2sIMbry2nCFhf5Djg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -371,6 +346,22 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.29.7.tgz", + "integrity": "sha512-TSu8+mHCoEaaCDEZ0I3+6mvTBYR4PCxQwf2z9/r5Tbztv6NaLR3B9thGTTxX2WGuGHJqRiAbKPeGTJ5XWXVg6A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, "node_modules/@babel/plugin-syntax-logical-assignment-operators": { "version": "7.10.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", @@ -482,13 +473,13 @@ } }, "node_modules/@babel/plugin-syntax-typescript": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.28.6.tgz", - "integrity": "sha512-+nDNmQye7nlnuuHDboPbGm00Vqg3oO8niRRL27/4LYHUsHYh0zJ1xWOz0uRwNFmM1Avzk8wZbc6rdiYhomzv/A==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.29.7.tgz", + "integrity": "sha512-ngr+82Sh0xMz25TPCZi+nC2iTzjfCdWS2ONXTp/PtSCHCgaCNBpdMqgvJ2ccdLlClVZ7sisIgB914j/JFe+RZA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -540,24 +531,6 @@ "node": ">=6.9.0" } }, - "node_modules/@babel/traverse/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@babel/types": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", @@ -588,6 +561,40 @@ "commander": "10.0.x" } }, + "node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.9.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", @@ -645,24 +652,6 @@ "node": "^20.19.0 || ^22.13.0 || >=24" } }, - "node_modules/@eslint/config-array/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@eslint/config-helpers": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.6.0.tgz", @@ -827,9 +816,9 @@ } }, "node_modules/@istanbuljs/schema": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", - "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", + "integrity": "sha512-+Sg6GCR/wy1oSmQDFq4LQDAhm3ETKnorxN+y5nbLULOR3P0c14f2Wurzj3/xqPXtasLFfHd5iRFQ7AJt4KH2cw==", "dev": true, "license": "MIT", "engines": { @@ -837,62 +826,61 @@ } }, "node_modules/@jest/console": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-28.1.3.tgz", - "integrity": "sha512-QPAkP5EwKdK/bxIr6C1I4Vs0rm2nHiANzj/Z5X2JQkrZo6IqvC4ldZ9K95tF0HdidhA8Bo6egxSzUFPYKcEXLw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.4.1.tgz", + "integrity": "sha512-v3bhyxUh9Hgmo5p6hAOXe14/R3ZxZDOsvHleh4B07z3m/x4/ngPUXEm9XwK4sF4u+f+P2ORb0Ge+MgpaqRMVDA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "^28.1.3", + "@jest/types": "30.4.1", "@types/node": "*", - "chalk": "^4.0.0", - "jest-message-util": "^28.1.3", - "jest-util": "^28.1.3", + "chalk": "^4.1.2", + "jest-message-util": "30.4.1", + "jest-util": "30.4.1", "slash": "^3.0.0" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/core": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-28.1.3.tgz", - "integrity": "sha512-CIKBrlaKOzA7YG19BEqCw3SLIsEwjZkeJzf5bdooVnW4bH5cktqe3JX+G2YV1aK5vP8N9na1IGWFzYaTp6k6NA==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.4.2.tgz", + "integrity": "sha512-TZJA6cPJUFxoWhxaLo8t0VX/MZX2wPWr0uIDvLSHIvN4gu9h02vSzqI2kBADG1ExqQlC+cY09xKMSreivvrChQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "^28.1.3", - "@jest/reporters": "^28.1.3", - "@jest/test-result": "^28.1.3", - "@jest/transform": "^28.1.3", - "@jest/types": "^28.1.3", + "@jest/console": "30.4.1", + "@jest/pattern": "30.4.0", + "@jest/reporters": "30.4.1", + "@jest/test-result": "30.4.1", + "@jest/transform": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "jest-changed-files": "^28.1.3", - "jest-config": "^28.1.3", - "jest-haste-map": "^28.1.3", - "jest-message-util": "^28.1.3", - "jest-regex-util": "^28.0.2", - "jest-resolve": "^28.1.3", - "jest-resolve-dependencies": "^28.1.3", - "jest-runner": "^28.1.3", - "jest-runtime": "^28.1.3", - "jest-snapshot": "^28.1.3", - "jest-util": "^28.1.3", - "jest-validate": "^28.1.3", - "jest-watcher": "^28.1.3", - "micromatch": "^4.0.4", - "pretty-format": "^28.1.3", - "rimraf": "^3.0.0", - "slash": "^3.0.0", - "strip-ansi": "^6.0.0" + "ansi-escapes": "^4.3.2", + "chalk": "^4.1.2", + "ci-info": "^4.2.0", + "exit-x": "^0.2.2", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.11", + "jest-changed-files": "30.4.1", + "jest-config": "30.4.2", + "jest-haste-map": "30.4.1", + "jest-message-util": "30.4.1", + "jest-regex-util": "30.4.0", + "jest-resolve": "30.4.1", + "jest-resolve-dependencies": "30.4.2", + "jest-runner": "30.4.2", + "jest-runtime": "30.4.2", + "jest-snapshot": "30.4.1", + "jest-util": "30.4.1", + "jest-validate": "30.4.1", + "jest-watcher": "30.4.1", + "pretty-format": "30.4.1", + "slash": "^3.0.0" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" @@ -903,117 +891,150 @@ } } }, + "node_modules/@jest/diff-sequences": { + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.4.0.tgz", + "integrity": "sha512-zOpzlfUs45l6u7jm39qr87JCHUDsaeCtvL+kQe/Vn9jSnRB4/5IPXISm0h9I1vZW/o00Kn4UTJ2MOlhnUGwv3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, "node_modules/@jest/environment": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-28.1.3.tgz", - "integrity": "sha512-1bf40cMFTEkKyEf585R9Iz1WayDjHoHqvts0XFYEqyKM3cFWDpeMoqKKTAF9LSYQModPUlh8FKptoM2YcMWAXA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.4.1.tgz", + "integrity": "sha512-AK9yNRqgKxiabqMoe4oW+3/TSSeV8vkdC7BGaxZdU0AFXfOpofTLqdru2GXKZghP3sdgwE9XXpnVwfZ8JnFV4w==", "dev": true, "license": "MIT", "dependencies": { - "@jest/fake-timers": "^28.1.3", - "@jest/types": "^28.1.3", + "@jest/fake-timers": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", - "jest-mock": "^28.1.3" + "jest-mock": "30.4.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/expect": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-28.1.3.tgz", - "integrity": "sha512-lzc8CpUbSoE4dqT0U+g1qODQjBRHPpCPXissXD4mS9+sWQdmmpeJ9zSH1rS1HEkrsMN0fb7nKrJ9giAR1d3wBw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.4.1.tgz", + "integrity": "sha512-ginrj6TMgh2GshLUGCjO94Ptx9HhdZA/I6A9iUfyeLKFtdAjnKzHDgzgP9HYQgbxM1lbXScQ2eUBz2lGeVDPWA==", "dev": true, "license": "MIT", "dependencies": { - "expect": "^28.1.3", - "jest-snapshot": "^28.1.3" + "expect": "30.4.1", + "jest-snapshot": "30.4.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/expect-utils": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-28.1.3.tgz", - "integrity": "sha512-wvbi9LUrHJLn3NlDW6wF2hvIMtd4JUl2QNVrjq+IBSHirgfrR3o9RnVtxzdEGO2n9JyIWwHnLfby5KzqBGg2YA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.4.1.tgz", + "integrity": "sha512-ZBn5CglH8fBsQsvs4VWNzD4aWfUYks+IdOOQU3MEK71ol/BcVm+P+rtb1KpiFBpSWSCE27uOahyyf1vfqOVbcQ==", "dev": true, "license": "MIT", "dependencies": { - "jest-get-type": "^28.0.2" + "@jest/get-type": "30.1.0" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/fake-timers": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-28.1.3.tgz", - "integrity": "sha512-D/wOkL2POHv52h+ok5Oj/1gOG9HSywdoPtFsRCUmlCILXNn5eIWmcnd3DIiWlJnpGvQtmajqBP95Ei0EimxfLw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.4.1.tgz", + "integrity": "sha512-iW5umdmfPeWzehrVhugFQZqCchSCud5S1l2YT0O9ZhjRR0ExclANDZkiSBwzqtnlOn0J1JXvO+HZ6rkuyOVOgQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "^28.1.3", - "@sinonjs/fake-timers": "^9.1.2", + "@jest/types": "30.4.1", + "@sinonjs/fake-timers": "^15.4.0", "@types/node": "*", - "jest-message-util": "^28.1.3", - "jest-mock": "^28.1.3", - "jest-util": "^28.1.3" + "jest-message-util": "30.4.1", + "jest-mock": "30.4.1", + "jest-util": "30.4.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/get-type": { + "version": "30.1.0", + "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", + "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/globals": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-28.1.3.tgz", - "integrity": "sha512-XFU4P4phyryCXu1pbcqMO0GSQcYe1IsalYCDzRNyhetyeyxMcIxa11qPNDpVNLeretItNqEmYYQn1UYz/5x1NA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.4.1.tgz", + "integrity": "sha512-ZbuY4cmXC8DkxYjfvT2DbcHWL2T6vmsMhXCDcmTB2T0y0gaezBI77ufq5ZAIdcRkYZ7NEQEDg1xFeKbxUJ5v5Q==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "^28.1.3", - "@jest/expect": "^28.1.3", - "@jest/types": "^28.1.3" + "@jest/environment": "30.4.1", + "@jest/expect": "30.4.1", + "@jest/types": "30.4.1", + "jest-mock": "30.4.1" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/pattern": { + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/@jest/pattern/-/pattern-30.4.0.tgz", + "integrity": "sha512-RAWn3+f9u8BsHijKJ71uHcFp6vmyEt6VvoWXkl6hKF3qVIuWNmudVjg12DlBPGup/frIl5UcUlH5HfEuvHpEXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "jest-regex-util": "30.4.0" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/reporters": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-28.1.3.tgz", - "integrity": "sha512-JuAy7wkxQZVNU/V6g9xKzCGC5LVXx9FDcABKsSXp5MiKPEE2144a/vXTEDoyzjUpZKfVwp08Wqg5A4WfTMAzjg==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.4.1.tgz", + "integrity": "sha512-/SnkPCzEQpUaBH81kjdEdDdo2WZl5hxw+BmLDGWjRkm8o7XlhjwsU36cqwe5PGBE5WYpBvDzRSdXx9rbGuJtNA==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "^28.1.3", - "@jest/test-result": "^28.1.3", - "@jest/transform": "^28.1.3", - "@jest/types": "^28.1.3", - "@jridgewell/trace-mapping": "^0.3.13", + "@jest/console": "30.4.1", + "@jest/test-result": "30.4.1", + "@jest/transform": "30.4.1", + "@jest/types": "30.4.1", + "@jridgewell/trace-mapping": "^0.3.25", "@types/node": "*", - "chalk": "^4.0.0", - "collect-v8-coverage": "^1.0.0", - "exit": "^0.1.2", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", + "chalk": "^4.1.2", + "collect-v8-coverage": "^1.0.2", + "exit-x": "^0.2.2", + "glob": "^10.5.0", + "graceful-fs": "^4.2.11", "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-instrument": "^5.1.0", + "istanbul-lib-instrument": "^6.0.0", "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.0", + "istanbul-lib-source-maps": "^5.0.0", "istanbul-reports": "^3.1.3", - "jest-message-util": "^28.1.3", - "jest-util": "^28.1.3", - "jest-worker": "^28.1.3", + "jest-message-util": "30.4.1", + "jest-util": "30.4.1", + "jest-worker": "30.4.1", "slash": "^3.0.0", - "string-length": "^4.0.1", - "strip-ansi": "^6.0.0", - "terminal-link": "^2.0.0", + "string-length": "^4.0.2", "v8-to-istanbul": "^9.0.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" @@ -1024,144 +1045,306 @@ } } }, + "node_modules/@jest/reporters/node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@jest/reporters/node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@jest/reporters/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@jest/reporters/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, "node_modules/@jest/reporters/node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", "dev": true, "license": "ISC", "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" }, - "engines": { - "node": "*" + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@jest/reporters/node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" }, "funding": { "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" } }, + "node_modules/@jest/reporters/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, "node_modules/@jest/reporters/node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "dev": true, "license": "ISC", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.2" }, "engines": { - "node": "*" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@jest/reporters/node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@jest/reporters/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@jest/reporters/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@jest/reporters/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, "node_modules/@jest/schemas": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-28.1.3.tgz", - "integrity": "sha512-/l/VWsdt/aBXgjshLWOFyFt3IVdYypu5y2Wn2rOO1un6nkqIn8SLXzgIMYXFyYsRWDyF5EthmKJMIdJvk08grg==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.4.1.tgz", + "integrity": "sha512-i6b4qw5qnP8c5FEeBJg/uZQ4ddrkN6Ca8qISJh0pr7a5hfn3h3v5x60BEbOC7OYAGZNMs1LfFLwnW2CuK8F57Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.34.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/snapshot-utils": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.4.1.tgz", + "integrity": "sha512-ObY4ljvQ95mt6iwKtVLetR/4yXiAgl3H4nJxhztr0MTjrN97TwDYrnCp/kF60Ec9HdhkWTHSu+Hg05aXfngpOA==", "dev": true, "license": "MIT", "dependencies": { - "@sinclair/typebox": "^0.24.1" + "@jest/types": "30.4.1", + "chalk": "^4.1.2", + "graceful-fs": "^4.2.11", + "natural-compare": "^1.4.0" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/source-map": { - "version": "28.1.2", - "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-28.1.2.tgz", - "integrity": "sha512-cV8Lx3BeStJb8ipPHnqVw/IM2VCMWO3crWZzYodSIkxXnRcXJipCdx1JCK0K5MsJJouZQTH73mzf4vgxRaH9ww==", + "version": "30.0.1", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-30.0.1.tgz", + "integrity": "sha512-MIRWMUUR3sdbP36oyNyhbThLHyJ2eEDClPCiHVbrYAe5g3CHRArIVpBw7cdSB5fr+ofSfIb2Tnsw8iEHL0PYQg==", "dev": true, "license": "MIT", "dependencies": { - "@jridgewell/trace-mapping": "^0.3.13", - "callsites": "^3.0.0", - "graceful-fs": "^4.2.9" + "@jridgewell/trace-mapping": "^0.3.25", + "callsites": "^3.1.0", + "graceful-fs": "^4.2.11" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/test-result": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-28.1.3.tgz", - "integrity": "sha512-kZAkxnSE+FqE8YjW8gNuoVkkC9I7S1qmenl8sGcDOLropASP+BkcGKwhXoyqQuGOGeYY0y/ixjrd/iERpEXHNg==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.4.1.tgz", + "integrity": "sha512-/ZG7pgEiOmmWkN9TplKbOu4id2N5lh7FHwRwlkgBVAzGdRH+OkkQ8wX/kIxg4zmd3ZQvAL1RwL2yWsvNYYECTw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "^28.1.3", - "@jest/types": "^28.1.3", - "@types/istanbul-lib-coverage": "^2.0.0", - "collect-v8-coverage": "^1.0.0" + "@jest/console": "30.4.1", + "@jest/types": "30.4.1", + "@types/istanbul-lib-coverage": "^2.0.6", + "collect-v8-coverage": "^1.0.2" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/test-sequencer": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-28.1.3.tgz", - "integrity": "sha512-NIMPEqqa59MWnDi1kvXXpYbqsfQmSJsIbnd85mdVGkiDfQ9WQQTXOLsvISUfonmnBT+w85WEgneCigEEdHDFxw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.4.1.tgz", + "integrity": "sha512-PeYE+4td5rKjoRPxztObrXU+H8hsjZfxKMXOcmrr34JerSyB/ROOxbbicz8B7A5j9R9VayDnVPvBmedqCsFCdw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/test-result": "^28.1.3", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^28.1.3", + "@jest/test-result": "30.4.1", + "graceful-fs": "^4.2.11", + "jest-haste-map": "30.4.1", "slash": "^3.0.0" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/transform": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-28.1.3.tgz", - "integrity": "sha512-u5dT5di+oFI6hfcLOHGTAfmUxFRrjK+vnaP0kkVow9Md/M7V/MxqQMOz/VV25UZO8pzeA9PjfTpOu6BDuwSPQA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.4.1.tgz", + "integrity": "sha512-Wz0LyktlTvRefoymh+n64hQ84KNXsRGcwdoZ8CSa0Ea+fgYcHZlnk+hDP7v2MS7il2bQ5uTEIxf4/NNfhMN4KQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/core": "^7.11.6", - "@jest/types": "^28.1.3", - "@jridgewell/trace-mapping": "^0.3.13", - "babel-plugin-istanbul": "^6.1.1", - "chalk": "^4.0.0", - "convert-source-map": "^1.4.0", - "fast-json-stable-stringify": "^2.0.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^28.1.3", - "jest-regex-util": "^28.0.2", - "jest-util": "^28.1.3", - "micromatch": "^4.0.4", - "pirates": "^4.0.4", + "@babel/core": "^7.27.4", + "@jest/types": "30.4.1", + "@jridgewell/trace-mapping": "^0.3.25", + "babel-plugin-istanbul": "^7.0.1", + "chalk": "^4.1.2", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.11", + "jest-haste-map": "30.4.1", + "jest-regex-util": "30.4.0", + "jest-util": "30.4.1", + "pirates": "^4.0.7", "slash": "^3.0.0", - "write-file-atomic": "^4.0.1" + "write-file-atomic": "^5.0.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/types": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-28.1.3.tgz", - "integrity": "sha512-RyjiyMUZrKz/c+zlMFO1pm70DcIlST8AeWTkoUdZevew44wcNZQHsEVOiCVtgVnlFFD82FPaXycys58cf2muVQ==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.4.1.tgz", + "integrity": "sha512-f1x/vJXIfjOlEmejYpbkbgw1gOqpPECwMvMEtBqe47j7H2Hg8h8w3o3ikhSXq3MI15kg+oQ0exWO0uCtTNJLoQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/schemas": "^28.1.3", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", + "@jest/pattern": "30.4.0", + "@jest/schemas": "30.4.1", + "@types/istanbul-lib-coverage": "^2.0.6", + "@types/istanbul-reports": "^3.0.4", "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" + "@types/yargs": "^17.0.33", + "chalk": "^4.1.2" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jridgewell/gen-mapping": { @@ -1223,34 +1406,39 @@ "debug": "^4.1.1" } }, - "node_modules/@kwsites/file-exists/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@kwsites/promise-deferred": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz", "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==", "license": "MIT" }, - "node_modules/@opensearch-project/opensearch": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-2.13.0.tgz", - "integrity": "sha512-Bu3jJ7pKzumbMMeefu7/npAWAvFu5W9SlbBow1ulhluqUpqc7QoXe0KidDrMy7Dy3BQrkI6llR3cWL4lQTZOFw==", - "license": "Apache-2.0", + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.1.tgz", + "integrity": "sha512-KjZdi8Q1wh89gsVmghvbrMgWl6ZWmRmHV6wjB7/g4Zf0dyO+hH3neZUtuDNPO00qq5YE5RITVWvrIZKRaAmzGQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=23.5.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.3", + "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.3" + } + }, + "node_modules/@opensearch-project/opensearch": { + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-2.13.0.tgz", + "integrity": "sha512-Bu3jJ7pKzumbMMeefu7/npAWAvFu5W9SlbBow1ulhluqUpqc7QoXe0KidDrMy7Dy3BQrkI6llR3cWL4lQTZOFw==", + "license": "Apache-2.0", "dependencies": { "aws4": "^1.11.0", "debug": "^4.3.1", @@ -1264,21 +1452,28 @@ "yarn": "^1.22.10" } }, - "node_modules/@opensearch-project/opensearch/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, + "optional": true, "engines": { - "node": ">=6.0" + "node": ">=14" + } + }, + "node_modules/@pkgr/core": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.3.6.tgz", + "integrity": "sha512-SEeaJLb3qBNF/OaXnaR1NmmBbFYk1zC0ZH/52fATcRPLFg/p791YrcyFFy44Bo9sLaGuSuLp5Q6axbb/O+v/RA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.18.0 || >=16.0.0" }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } + "funding": { + "url": "https://opencollective.com/pkgr" } }, "node_modules/@scarf/scarf": { @@ -1304,16 +1499,16 @@ } }, "node_modules/@sinclair/typebox": { - "version": "0.24.51", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.24.51.tgz", - "integrity": "sha512-1P1OROm/rdubP5aFDSZQILU0vrLCJ4fvHt6EoqHEM+2D/G5MK3bIaymUKLit8Js9gbns5UyJnkP/TZROLw4tUA==", + "version": "0.34.52", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.52.tgz", + "integrity": "sha512-XiMQh7qqVlxZzcVD+kkGMNGMzcTrDMLWI7S4x7z1MkCkbDPrekpZXEUK0eZqZFMuHQg2a2DZOcDIh9o5v3Gonw==", "dev": true, "license": "MIT" }, "node_modules/@sinonjs/commons": { - "version": "1.8.6", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.6.tgz", - "integrity": "sha512-Ky+XkAkqPZSm3NLBeUng77EBQl3cmeJhITaGHdYH8kjVB+aun3S4XBRti2zt17mtt0mIUDiNxYeoJm6drVvBJQ==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -1321,13 +1516,24 @@ } }, "node_modules/@sinonjs/fake-timers": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-9.1.2.tgz", - "integrity": "sha512-BPS4ynJW/o92PUR4wgriz2Ud5gpST5vz6GQfMixEDK0Z8ZCUv2M7SkBLykH56T++Xs+8ln9zTGbOvNGIe02/jw==", + "version": "15.4.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-15.4.0.tgz", + "integrity": "sha512-DsG+8/LscQIQg68J6Ef3dv10u6nVyetYn923s3/sus5eaGfTo1of5WMZSLf0UJc9KDuKPilPH0UDJCjvNbDNCA==", "dev": true, "license": "BSD-3-Clause", "dependencies": { - "@sinonjs/commons": "^1.7.0" + "@sinonjs/commons": "^3.0.1" + } + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" } }, "node_modules/@types/babel__core": { @@ -1389,16 +1595,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/graceful-fs": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", - "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", @@ -1434,22 +1630,15 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "25.5.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.0.tgz", - "integrity": "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==", + "version": "26.1.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.2.tgz", + "integrity": "sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==", "dev": true, "license": "MIT", "dependencies": { - "undici-types": "~7.18.0" + "undici-types": "~8.3.0" } }, - "node_modules/@types/prettier": { - "version": "2.7.3", - "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.7.3.tgz", - "integrity": "sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", @@ -1474,6 +1663,356 @@ "dev": true, "license": "MIT" }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.3.tgz", + "integrity": "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg==", + "dev": true, + "license": "ISC" + }, + "node_modules/@unrs/resolver-binding-android-arm-eabi": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm-eabi/-/resolver-binding-android-arm-eabi-1.12.2.tgz", + "integrity": "sha512-g5T90pqg1bo/7mytQx6F4iBNC0Wsh9cu+z9veDbFjc7HjpesJFWD7QMS0NGStXM075+7dJPPVvBbpZlnrdpi/w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@unrs/resolver-binding-android-arm64": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm64/-/resolver-binding-android-arm64-1.12.2.tgz", + "integrity": "sha512-YGCRZv/9GLhwmz6mYDeTsm/92BAyR28l6c2ReweVW5pWgfsitWLY8upvfRlGdoyD8HjeTHSYJWyZGD4KJA/nFQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@unrs/resolver-binding-darwin-arm64": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-arm64/-/resolver-binding-darwin-arm64-1.12.2.tgz", + "integrity": "sha512-u9DiNT1auQMO20A9SyTuG3wUgQWB9Z7KjAg0uFuCDR1FsAY8A0CG2S6JpHS1xwm/w1G08bjXZDcyOCjv1WAm2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@unrs/resolver-binding-darwin-x64": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-x64/-/resolver-binding-darwin-x64-1.12.2.tgz", + "integrity": "sha512-f7rPLi/T1HVKZu/u6t87lroib16n8vrSzcyxI7lg4BGO9UF26KhQL44sd9eOUgrTYhvRXtWOIZT5PejdPyJfUA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@unrs/resolver-binding-freebsd-x64": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-freebsd-x64/-/resolver-binding-freebsd-x64-1.12.2.tgz", + "integrity": "sha512-BpcOjWCJub6nRZUS2zA20pmLvjtqAtGejETaIyRLiZiQf++cbrjltLA5NN/xaXfqeOBOSlMFbemIl5/S5tljmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm-gnueabihf": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-gnueabihf/-/resolver-binding-linux-arm-gnueabihf-1.12.2.tgz", + "integrity": "sha512-vZTDvdSISZjJx66OzJqtsOhzifbqRjbmI1Mnu49fQDwog5GtDI4QidRiEAYbZCRj9C8YZEW+3ZjqsyS9GR4k2A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm-musleabihf": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-musleabihf/-/resolver-binding-linux-arm-musleabihf-1.12.2.tgz", + "integrity": "sha512-BiPI+IrIlwcW4nLLMM21+B1dFPzd55yAVgVGrdgDjNef+ch03GdxrcyaIz8X9SsQirh/kCQ7mviyWlMxdh2D7g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm64-gnu": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-gnu/-/resolver-binding-linux-arm64-gnu-1.12.2.tgz", + "integrity": "sha512-zJc0H99FEPoFfSrNpa91HYfxzfAJCr502oxNK1cfdC9hlaFI43RT+JFCann9JUgZmLzzntChHyn13Sgn9ljHNg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm64-musl": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-musl/-/resolver-binding-linux-arm64-musl-1.12.2.tgz", + "integrity": "sha512-KQ3Lki6l+Pz1k/eBipN41ES+YUK30beLGb9YqcB1O542cyLCNE6GaxrfcY3T6EezmGGk84wb5XyO9loTM9tkcA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-loong64-gnu": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-loong64-gnu/-/resolver-binding-linux-loong64-gnu-1.12.2.tgz", + "integrity": "sha512-3SJGEh1DborhG6pyxvhPzCT4bbSIVihsvgJc13P1bHG7KLdNDaF9T3gsTwFc7Jw/5Y5/iWOjkEx7Zy0NvCGX3Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-loong64-musl": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-loong64-musl/-/resolver-binding-linux-loong64-musl-1.12.2.tgz", + "integrity": "sha512-jiuG/Obbel7uw1PwHNFfrkiKhLAF6mnyZ6aWlOAVN9WqKm8v0OFGnciJIHu8+CMvXLQ8AD51LPzAoUfT21D5Ew==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-ppc64-gnu": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-ppc64-gnu/-/resolver-binding-linux-ppc64-gnu-1.12.2.tgz", + "integrity": "sha512-q7xRvVpmcfeL+LlZg8Pbbo6QaTZwDU5BaGZbwfhkEsXJn3Was8xYfE0RBH266xZt0rM6B7i8xAYIvjthuUIWHg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-riscv64-gnu": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-gnu/-/resolver-binding-linux-riscv64-gnu-1.12.2.tgz", + "integrity": "sha512-0CVdx6lcnT3Q9inOH8tsMIOJ6ImndllMjqJHg8RLVdB7Vq4SfkEXl9mCSsVNuNA4MCYycRicCUxPCabVHJRr6A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-riscv64-musl": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-musl/-/resolver-binding-linux-riscv64-musl-1.12.2.tgz", + "integrity": "sha512-iOwlRo9vnp6R6ohHQS11n0NnfdXx/omhkocmIfaPRpQhKZ+3BDMkkdRVh53qjkFkpPddf+FETA28NwGN7l5l+w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-s390x-gnu": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-s390x-gnu/-/resolver-binding-linux-s390x-gnu-1.12.2.tgz", + "integrity": "sha512-HYJtLfXq94q8iZNFT1lknx258wlkkWhZeUXJRqzKBBUJ00CvZ+N33zgbCqimLjsyw5Va6uUxhVa12mI+kaveEw==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-x64-gnu": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-gnu/-/resolver-binding-linux-x64-gnu-1.12.2.tgz", + "integrity": "sha512-mPsUhunKKDih5O96Y6enDQyHc1SqBPlY1E/SfMWDM3EdJ95Z9CArPeCVwCCqbP45ljvivdEk8Fxn+SIb1rDAJQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-x64-musl": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-musl/-/resolver-binding-linux-x64-musl-1.12.2.tgz", + "integrity": "sha512-azrt6+5ydLd8Vt210AAFis/lZevSfPw93EJRIJG+xPu4WCJ8K0kppCTpMyLPcKT7H15M4Jnt2tMp5bOvCkRC6A==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-openharmony-arm64": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-openharmony-arm64/-/resolver-binding-openharmony-arm64-1.12.2.tgz", + "integrity": "sha512-YZ9hP4O0X9PQb8eO980qmLNGH4zT3I9+SZTdt0Pr0YyuGQhYKoOZkV02VzrzyOZJ5xIJ3UFIenKkUkGg8GjgWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-wasm32-wasi/-/resolver-binding-wasm32-wasi-1.12.2.tgz", + "integrity": "sha512-tYFDIkMxSflfEc/h92ZWNsZlHSwgimbNHSO3PL2JWQHfCuC2q316jMyYU9TIWZsFK2bQwyK5VAdYgn8ygPj69A==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.10.0", + "@emnapi/runtime": "1.10.0", + "@napi-rs/wasm-runtime": "^1.1.4" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.12.2.tgz", + "integrity": "sha512-qzNyg3xL0VPQmCaUh+N5jSitce6k+uCBfMDesWRnlULOZaqUkaJ0ybdT+UqlAWJoQjuqfIU/0Ptx9bteN4D82g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@unrs/resolver-binding-win32-ia32-msvc": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-ia32-msvc/-/resolver-binding-win32-ia32-msvc-1.12.2.tgz", + "integrity": "sha512-WD9sY00OfpHVGfsnHZoA8jVT+esS/Bg8z8jzxp5BnDCjjwsuKsPQrzswwpFy4J1AUJbXPRfkpcX0mXrzeXW79g==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@unrs/resolver-binding-win32-x64-msvc": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-x64-msvc/-/resolver-binding-win32-x64-msvc-1.12.2.tgz", + "integrity": "sha512-nAB74NfSNKknqQ1RrYj6uz8FcXEomu/MATJZxh/x+BArzN2U3JbOYC0APYzUIGhVY3m5hRxA8VPNdPBoG8txlA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -1593,14 +2132,11 @@ } }, "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } + "license": "Python-2.0" }, "node_modules/array-flatten": { "version": "1.1.1", @@ -1645,58 +2181,58 @@ } }, "node_modules/babel-jest": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-28.1.3.tgz", - "integrity": "sha512-epUaPOEWMk3cWX0M/sPvCHHCe9fMFAa/9hXEgKP8nFfNl/jlGkE9ucq9NqkZGXLDduCJYS0UvSlPUwC0S+rH6Q==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.4.1.tgz", + "integrity": "sha512-fATAbM8piYxkiXQp3RBXmZHxZVNJZAVXXfyeyCN2Tida3+qJ8ea9UxhiJ2y4fLO90ZImKt6k9FlcH2+rLkJGhw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/transform": "^28.1.3", - "@types/babel__core": "^7.1.14", - "babel-plugin-istanbul": "^6.1.1", - "babel-preset-jest": "^28.1.3", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", + "@jest/transform": "30.4.1", + "@types/babel__core": "^7.20.5", + "babel-plugin-istanbul": "^7.0.1", + "babel-preset-jest": "30.4.0", + "chalk": "^4.1.2", + "graceful-fs": "^4.2.11", "slash": "^3.0.0" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { - "@babel/core": "^7.8.0" + "@babel/core": "^7.11.0 || ^8.0.0-0" } }, "node_modules/babel-plugin-istanbul": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", - "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-7.0.1.tgz", + "integrity": "sha512-D8Z6Qm8jCvVXtIRkBnqNHX0zJ37rQcFJ9u8WOS6tkYOsRdHBzypCstaxWiu5ZIlqQtviRYbgnRLSoCEvjqcqbA==", "dev": true, "license": "BSD-3-Clause", + "workspaces": [ + "test/babel-8" + ], "dependencies": { "@babel/helper-plugin-utils": "^7.0.0", "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-instrument": "^5.0.4", + "@istanbuljs/schema": "^0.1.3", + "istanbul-lib-instrument": "^6.0.2", "test-exclude": "^6.0.0" }, "engines": { - "node": ">=8" + "node": ">=12" } }, "node_modules/babel-plugin-jest-hoist": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-28.1.3.tgz", - "integrity": "sha512-Ys3tUKAmfnkRUpPdpa98eYrAR0nV+sSFUZZEGuQ2EbFd1y4SOLtD5QDNHAq+bb9a+bbXvYQC4b+ID/THIMcU6Q==", + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-30.4.0.tgz", + "integrity": "sha512-9EdtWM/sSfXLOGLwSn+GS6pIXyBnL07/8gyJlwFXjWy4DxMOyItqyUT29d4lQiS380EZwYlX7/At4PgBS+m2aA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/template": "^7.3.3", - "@babel/types": "^7.3.3", - "@types/babel__core": "^7.1.14", - "@types/babel__traverse": "^7.0.6" + "@types/babel__core": "^7.20.5" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/babel-preset-current-node-syntax": { @@ -1727,20 +2263,20 @@ } }, "node_modules/babel-preset-jest": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-28.1.3.tgz", - "integrity": "sha512-L+fupJvlWAHbQfn74coNX3zf60LXMJsezNvvx8eIh7iOR1luJ1poxYgQk1F8PYtNq/6QODDHCqsSnTFSWC491A==", + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-30.4.0.tgz", + "integrity": "sha512-lBY4jxsNmCnSiu7kquw8ZC9F4+XLMOKypT3RnNHPvU2Kpd4W0xaPuLr5ZkRyOsvLYAY4yaW1ZwTW4xB7NIiZzg==", "dev": true, "license": "MIT", "dependencies": { - "babel-plugin-jest-hoist": "^28.1.3", - "babel-preset-current-node-syntax": "^1.0.0" + "babel-plugin-jest-hoist": "30.4.0", + "babel-preset-current-node-syntax": "^1.2.0" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { - "@babel/core": "^7.0.0" + "@babel/core": "^7.11.0 || ^8.0.0-beta.1" } }, "node_modules/balanced-match": { @@ -1753,9 +2289,9 @@ } }, "node_modules/baseline-browser-mapping": { - "version": "2.10.40", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.40.tgz", - "integrity": "sha512-BSSLZ9/Cjjv7Gtj5B68ZzXcXUg8iOf3fme+FCuh8rC/Go+Kmh8cox7M3A8dolou16s64QjLPOSdngh7GxXvkSw==", + "version": "2.11.8", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.8.tgz", + "integrity": "sha512-zAgkquC2WYF0PIc6XbNYkA2uuxxFavzgmX61R+dHDUa558V8Ejf8ozTZFR6QzM24RWu4kBcRkhJ5kpz77j9fnQ==", "dev": true, "license": "Apache-2.0", "bin": { @@ -1881,9 +2417,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.4", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.4.tgz", - "integrity": "sha512-MTc8i/x9jBQd1iMw2CFGS+rwMa07eYjLR0CCTLDACl9xhxy+nIs3KeML/biicXtk9JrZ6dnnTatmc7ErPXIxqw==", + "version": "4.28.7", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.7.tgz", + "integrity": "sha512-JxV13hNrFxqjOc8alRbq9dK1MM79NEXYpma2B2J4wAtpWS5zIEIKqWPGCl7N4o7Uc7B7itylh7SuDujATRyyTw==", "dev": true, "funding": [ { @@ -1901,10 +2437,10 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.10.38", - "caniuse-lite": "^1.0.30001799", - "electron-to-chromium": "^1.5.376", - "node-releases": "^2.0.48", + "baseline-browser-mapping": "^2.10.44", + "caniuse-lite": "^1.0.30001806", + "electron-to-chromium": "^1.5.393", + "node-releases": "^2.0.51", "update-browserslist-db": "^1.2.3" }, "bin": { @@ -1990,9 +2526,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001799", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001799.tgz", - "integrity": "sha512-hG1bReV+OUU+MOqK4t/ZWI0tZOyz3rqS9XuhOUz1cIcbwBKjOyJEJuw9ER5JuNyqxNk8u/JUVbGibBOL1yrjFw==", + "version": "1.0.30001806", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001806.tgz", + "integrity": "sha512-72Cuvd95zbSYPKq6Fhg8eDJRlzgWDf7/mtoZv6Qe/DYNCEBdNxoA3+rZAU2ZhGCpZlns3EssFavaZomckT5Uuw==", "dev": true, "funding": [ { @@ -2063,9 +2599,9 @@ } }, "node_modules/ci-info": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", - "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.4.0.tgz", + "integrity": "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==", "dev": true, "funding": [ { @@ -2079,9 +2615,9 @@ } }, "node_modules/cjs-module-lexer": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", - "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", + "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", "dev": true, "license": "MIT" }, @@ -2193,9 +2729,9 @@ } }, "node_modules/convert-source-map": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", - "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "dev": true, "license": "MIT" }, @@ -2321,12 +2857,37 @@ "date-fns": "2.x" } }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/dedent": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz", - "integrity": "sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==", + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.2.tgz", + "integrity": "sha512-WzMx3mW98SN+zn3hgemf4OzdmyNhhhKz5Ay0pUfQiMQ3e1g+xmTJWp/pKdwKVXhdSkAEGIIzqeuWrL3mV/AXbA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "babel-plugin-macros": "^3.1.0" + }, + "peerDependenciesMeta": { + "babel-plugin-macros": { + "optional": true + } + } }, "node_modules/deep-is": { "version": "0.1.4", @@ -2383,16 +2944,6 @@ "node": ">=8" } }, - "node_modules/diff-sequences": { - "version": "28.1.1", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-28.1.1.tgz", - "integrity": "sha512-FU0iFaH/E23a+a718l8Qa/19bF9p06kgE0KipMOMadwa3SjnaElKzPaUC0vnibs6/B/9ni97s61mcejk8W1fQw==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" - } - }, "node_modules/dotenv": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.2.0.tgz", @@ -2416,6 +2967,13 @@ "node": ">= 0.4" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -2423,16 +2981,16 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.381", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.381.tgz", - "integrity": "sha512-n9Wa6yB+vDsGuA8AKbl/0z7HbvWqt5jxIdvr1IUicd0ryPrk7/xzwqLv8D9AbbvZ6avVNtXYLTfmgFHkwkyelg==", + "version": "1.5.398", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.398.tgz", + "integrity": "sha512-AsvhAxopJGh6museTDMIjn6JpDYOfgu4RLlygomt87MUwBUqTfd/1EiPtx10/LZE8xpTvkP2E9Gafq7lkLtodQ==", "dev": true, "license": "ISC" }, "node_modules/emittery": { - "version": "0.10.2", - "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.10.2.tgz", - "integrity": "sha512-aITqOwnLanpHLNXZJENbOgjUBeHocD+xsSJmNrjovKBW5HbSpW3d1pEls7GFQPUWXiwG9+0P4GtHfEqC/4M0Iw==", + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", + "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", "dev": true, "license": "MIT", "engines": { @@ -2630,24 +3188,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/eslint/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/eslint/node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -2754,20 +3294,6 @@ "node": ">=0.4.0" } }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/esquery": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", @@ -2854,30 +3380,32 @@ "dev": true, "license": "ISC" }, - "node_modules/exit": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", - "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", + "node_modules/exit-x": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/exit-x/-/exit-x-0.2.2.tgz", + "integrity": "sha512-+I6B/IkJc1o/2tiURyz/ivu/O0nKNEArIUB5O7zBrlDVJr22SCLH3xTeEry428LvFhRzIA1g8izguxJ/gbNcVQ==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8.0" } }, "node_modules/expect": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/expect/-/expect-28.1.3.tgz", - "integrity": "sha512-eEh0xn8HlsuOBxFgIss+2mX85VAS4Qy3OSkjV7rlBWljtA4oWH37glVGyOZSZvErDT/yBywZdPGwCXuTvSG85g==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/expect/-/expect-30.4.1.tgz", + "integrity": "sha512-PMARsyh/JtqC20HoGqlFcIlQAyqUtW4PlI1rup1uhYJtKuwAjbvWi3GQMAn+STdHum/dk8xrKfUM1+5SAwpolA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/expect-utils": "^28.1.3", - "jest-get-type": "^28.0.2", - "jest-matcher-utils": "^28.1.3", - "jest-message-util": "^28.1.3", - "jest-util": "^28.1.3" + "@jest/expect-utils": "30.4.1", + "@jest/get-type": "30.1.0", + "jest-matcher-utils": "30.4.1", + "jest-message-util": "30.4.1", + "jest-mock": "30.4.1", + "jest-util": "30.4.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/express": { @@ -2945,23 +3473,6 @@ "express": ">= 4.11" } }, - "node_modules/express-rate-limit/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/express-validator": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/express-validator/-/express-validator-7.3.1.tgz", @@ -3481,23 +3992,6 @@ "node": ">= 6.0.0" } }, - "node_modules/https-proxy-agent/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", @@ -3623,22 +4117,6 @@ "node": ">=8" } }, - "node_modules/is-core-module": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", - "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", - "dev": true, - "license": "MIT", - "dependencies": { - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -3734,30 +4212,20 @@ } }, "node_modules/istanbul-lib-instrument": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", - "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", + "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", "dev": true, "license": "BSD-3-Clause", "dependencies": { - "@babel/core": "^7.12.3", - "@babel/parser": "^7.14.7", - "@istanbuljs/schema": "^0.1.2", + "@babel/core": "^7.23.9", + "@babel/parser": "^7.23.9", + "@istanbuljs/schema": "^0.1.3", "istanbul-lib-coverage": "^3.2.0", - "semver": "^6.3.0" + "semver": "^7.5.4" }, "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-instrument/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" + "node": ">=10" } }, "node_modules/istanbul-lib-report": { @@ -3776,38 +4244,20 @@ } }, "node_modules/istanbul-lib-source-maps": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", - "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz", + "integrity": "sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==", "dev": true, "license": "BSD-3-Clause", "dependencies": { + "@jridgewell/trace-mapping": "^0.3.23", "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0", - "source-map": "^0.6.1" + "istanbul-lib-coverage": "^3.0.0" }, "engines": { "node": ">=10" } }, - "node_modules/istanbul-lib-source-maps/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/istanbul-reports": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", @@ -3838,22 +4288,22 @@ } }, "node_modules/jest": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest/-/jest-28.1.3.tgz", - "integrity": "sha512-N4GT5on8UkZgH0O5LUavMRV1EDEhNTL0KEfRmDIeZHSV7p2XgLoY9t9VDUgL6o+yfdgYHVxuz81G8oB9VG5uyA==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest/-/jest-30.4.2.tgz", + "integrity": "sha512-Yi1jqNC/Oq0N4hBgNH/YvBpP1P57QqundgytzYqy3yqAa7NZPNjSoi4SGbRAXDMdBzNE6xBCi5U7RgfrvMEUVQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/core": "^28.1.3", - "@jest/types": "^28.1.3", - "import-local": "^3.0.2", - "jest-cli": "^28.1.3" + "@jest/core": "30.4.2", + "@jest/types": "30.4.1", + "import-local": "^3.2.0", + "jest-cli": "30.4.2" }, "bin": { "jest": "bin/jest.js" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" @@ -3865,75 +4315,75 @@ } }, "node_modules/jest-changed-files": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-28.1.3.tgz", - "integrity": "sha512-esaOfUWJXk2nfZt9SPyC8gA1kNfdKLkQWyzsMlqq8msYSlNKfmZxfRgZn4Cd4MGVUF+7v6dBs0d5TOAKa7iIiA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-30.4.1.tgz", + "integrity": "sha512-IuctmYrxi21iOSOaIXpJWalHyPAsVv0GeBHKDn8C1CA4W5htHn7INL+wdnL4Bo0+olEndvAFkmb++tIQJG+vvg==", "dev": true, "license": "MIT", "dependencies": { - "execa": "^5.0.0", + "execa": "^5.1.1", + "jest-util": "30.4.1", "p-limit": "^3.1.0" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-circus": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-28.1.3.tgz", - "integrity": "sha512-cZ+eS5zc79MBwt+IhQhiEp0OeBddpc1n8MBo1nMB8A7oPMKEO+Sre+wHaLJexQUj9Ya/8NOBY0RESUgYjB6fow==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.4.2.tgz", + "integrity": "sha512-rvHH7VlY6LgbJXJTQ87GW62g1FntOtbhh0zT+v04kC+pgL6aBKyYINXxWukCpj3dcIBMw5/XUbtDS9dU9JTXeQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "^28.1.3", - "@jest/expect": "^28.1.3", - "@jest/test-result": "^28.1.3", - "@jest/types": "^28.1.3", + "@jest/environment": "30.4.1", + "@jest/expect": "30.4.1", + "@jest/test-result": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", - "chalk": "^4.0.0", + "chalk": "^4.1.2", "co": "^4.6.0", - "dedent": "^0.7.0", - "is-generator-fn": "^2.0.0", - "jest-each": "^28.1.3", - "jest-matcher-utils": "^28.1.3", - "jest-message-util": "^28.1.3", - "jest-runtime": "^28.1.3", - "jest-snapshot": "^28.1.3", - "jest-util": "^28.1.3", + "dedent": "^1.6.0", + "is-generator-fn": "^2.1.0", + "jest-each": "30.4.1", + "jest-matcher-utils": "30.4.1", + "jest-message-util": "30.4.1", + "jest-runtime": "30.4.2", + "jest-snapshot": "30.4.1", + "jest-util": "30.4.1", "p-limit": "^3.1.0", - "pretty-format": "^28.1.3", + "pretty-format": "30.4.1", + "pure-rand": "^7.0.0", "slash": "^3.0.0", - "stack-utils": "^2.0.3" + "stack-utils": "^2.0.6" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-cli": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-28.1.3.tgz", - "integrity": "sha512-roY3kvrv57Azn1yPgdTebPAXvdR2xfezaKKYzVxZ6It/5NCxzJym6tUI5P1zkdWhfUYkxEI9uZWcQdaFLo8mJQ==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.4.2.tgz", + "integrity": "sha512-jfA2ocvVHMXS2QijrJ0d31ektP+d/W0T5RpcTX2Pq+3sVqHlsXVCM2+FmwpL+bdY8OfHpIg9xMxLF17Zg0U49Q==", "dev": true, "license": "MIT", "dependencies": { - "@jest/core": "^28.1.3", - "@jest/test-result": "^28.1.3", - "@jest/types": "^28.1.3", - "chalk": "^4.0.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "import-local": "^3.0.2", - "jest-config": "^28.1.3", - "jest-util": "^28.1.3", - "jest-validate": "^28.1.3", - "prompts": "^2.0.1", - "yargs": "^17.3.1" + "@jest/core": "30.4.2", + "@jest/test-result": "30.4.1", + "@jest/types": "30.4.1", + "chalk": "^4.1.2", + "exit-x": "^0.2.2", + "import-local": "^3.2.0", + "jest-config": "30.4.2", + "jest-util": "30.4.1", + "jest-validate": "30.4.1", + "yargs": "^17.7.2" }, "bin": { "jest": "bin/jest.js" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" @@ -3945,249 +4395,392 @@ } }, "node_modules/jest-config": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-28.1.3.tgz", - "integrity": "sha512-MG3INjByJ0J4AsNBm7T3hsuxKQqFIiRo/AUqb1q9LRKI5UU6Aar9JHbr9Ivn1TVwfUD9KirRoM/T6u8XlcQPHQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@jest/test-sequencer": "^28.1.3", - "@jest/types": "^28.1.3", - "babel-jest": "^28.1.3", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "deepmerge": "^4.2.2", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "jest-circus": "^28.1.3", - "jest-environment-node": "^28.1.3", - "jest-get-type": "^28.0.2", - "jest-regex-util": "^28.0.2", - "jest-resolve": "^28.1.3", - "jest-runner": "^28.1.3", - "jest-util": "^28.1.3", - "jest-validate": "^28.1.3", - "micromatch": "^4.0.4", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.4.2.tgz", + "integrity": "sha512-rNHAShJQqQwFNoL0hbf3BphSBOWnpOUAKvidLS/AjNVLPfoj5mSf4jQMfW3cYOs6hXeZC7nF7mDHaBnbxELOzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.27.4", + "@jest/get-type": "30.1.0", + "@jest/pattern": "30.4.0", + "@jest/test-sequencer": "30.4.1", + "@jest/types": "30.4.1", + "babel-jest": "30.4.1", + "chalk": "^4.1.2", + "ci-info": "^4.2.0", + "deepmerge": "^4.3.1", + "glob": "^10.5.0", + "graceful-fs": "^4.2.11", + "jest-circus": "30.4.2", + "jest-docblock": "30.4.0", + "jest-environment-node": "30.4.1", + "jest-regex-util": "30.4.0", + "jest-resolve": "30.4.1", + "jest-runner": "30.4.2", + "jest-util": "30.4.1", + "jest-validate": "30.4.1", "parse-json": "^5.2.0", - "pretty-format": "^28.1.3", + "pretty-format": "30.4.1", "slash": "^3.0.0", "strip-json-comments": "^3.1.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "@types/node": "*", + "esbuild-register": ">=3.4.0", "ts-node": ">=9.0.0" }, "peerDependenciesMeta": { "@types/node": { "optional": true }, + "esbuild-register": { + "optional": true + }, "ts-node": { "optional": true } } }, + "node_modules/jest-config/node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/jest-config/node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/jest-config/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-config/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, "node_modules/jest-config/node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", "dev": true, "license": "ISC", "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" }, - "engines": { - "node": "*" + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-config/node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" }, "funding": { "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" } }, + "node_modules/jest-config/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, "node_modules/jest-config/node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "dev": true, "license": "ISC", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.2" }, "engines": { - "node": "*" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/jest-diff": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-28.1.3.tgz", - "integrity": "sha512-8RqP1B/OXzjjTWkqMX67iqgwBVJRgCyKD3L9nq+6ZqJMdvjE8RgHktqZ6jNrkdMT+dJuYNI3rhQpxaz7drJHfw==", + "node_modules/jest-config/node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-config/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", "dev": true, "license": "MIT", "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "^28.1.1", - "jest-get-type": "^28.0.2", - "pretty-format": "^28.1.3" + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/jest-docblock": { - "version": "28.1.1", - "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-28.1.1.tgz", - "integrity": "sha512-3wayBVNiOYx0cwAbl9rwm5kKFP8yHH3d/fkEaL02NPTkDojPtheGB7HZSFY4wzX+DxyrvhXz0KSCVksmCknCuA==", + "node_modules/jest-config/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", "dev": true, "license": "MIT", "dependencies": { - "detect-newline": "^3.0.0" + "ansi-regex": "^6.2.2" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, - "node_modules/jest-each": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-28.1.3.tgz", - "integrity": "sha512-arT1z4sg2yABU5uogObVPvSlSMQlDA48owx07BDPAiasW0yYpYHYOo4HHLz9q0BVzDVU4hILFjzJw0So9aCL/g==", + "node_modules/jest-config/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "^28.1.3", - "chalk": "^4.0.0", - "jest-get-type": "^28.0.2", - "jest-util": "^28.1.3", - "pretty-format": "^28.1.3" + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/jest-environment-node": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-28.1.3.tgz", - "integrity": "sha512-ugP6XOhEpjAEhGYvp5Xj989ns5cB1K6ZdjBYuS30umT4CQEETaxSiPcZ/E1kFktX4GkrcM4qu07IIlDYX1gp+A==", + "node_modules/jest-diff": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.4.1.tgz", + "integrity": "sha512-CRpFK0RtLriVDGcPPAnR6HMVI8bSR2jnUIgralhauzYQZIb4RH9AtEInTuQr65LmmGggGcRT6HIASxwqsVsmlA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "^28.1.3", - "@jest/fake-timers": "^28.1.3", - "@jest/types": "^28.1.3", - "@types/node": "*", - "jest-mock": "^28.1.3", - "jest-util": "^28.1.3" + "@jest/diff-sequences": "30.4.0", + "@jest/get-type": "30.1.0", + "chalk": "^4.1.2", + "pretty-format": "30.4.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, - "node_modules/jest-get-type": { - "version": "28.0.2", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-28.0.2.tgz", - "integrity": "sha512-ioj2w9/DxSYHfOm5lJKCdcAmPJzQXmbM/Url3rhlghrPvT3tt+7a/+oXc9azkKmLvoiXjtV83bEWqi+vs5nlPA==", + "node_modules/jest-docblock": { + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-30.4.0.tgz", + "integrity": "sha512-ZPMabUZCx5MpbZ2eBYSvZ0J8fvo3dR9oM+eeUpb3aKNQFuS2tu3Duw1TNlMoP8k3WQgKGJuhcMFvwcVuq6T7oA==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-newline": "^3.1.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-each": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.4.1.tgz", + "integrity": "sha512-/8MJbH6fuj48TstjrMf+u/pd06Qezz5xOXvZA6442heNOWr8bdeoGZX2d9fCn028CoMgYmroH9//zky5GfyYmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/get-type": "30.1.0", + "@jest/types": "30.4.1", + "chalk": "^4.1.2", + "jest-util": "30.4.1", + "pretty-format": "30.4.1" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-environment-node": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.4.1.tgz", + "integrity": "sha512-4FZYVOk85hz2AyT6BbarKy9u37g6DbrDyCdFhsnDdXqyrueYQvB+0zO4f/kqLCRD0BsPRXPMNJeQwihKZV8naw==", "dev": true, "license": "MIT", + "dependencies": { + "@jest/environment": "30.4.1", + "@jest/fake-timers": "30.4.1", + "@jest/types": "30.4.1", + "@types/node": "*", + "jest-mock": "30.4.1", + "jest-util": "30.4.1", + "jest-validate": "30.4.1" + }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-haste-map": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-28.1.3.tgz", - "integrity": "sha512-3S+RQWDXccXDKSWnkHa/dPwt+2qwA8CJzR61w3FoYCvoo3Pn8tvGcysmMF0Bj0EX5RYvAI2EIvC57OmotfdtKA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.4.1.tgz", + "integrity": "sha512-rFrcONd8jeFsyw+Z9CrScJgglRf2+NFmNam8dKu7n+SoHqNYT47mn0DdEcVUZJpvh7Iz6/si7f7yUH7GJHVgnw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "^28.1.3", - "@types/graceful-fs": "^4.1.3", + "@jest/types": "30.4.1", "@types/node": "*", - "anymatch": "^3.0.3", - "fb-watchman": "^2.0.0", - "graceful-fs": "^4.2.9", - "jest-regex-util": "^28.0.2", - "jest-util": "^28.1.3", - "jest-worker": "^28.1.3", - "micromatch": "^4.0.4", + "anymatch": "^3.1.3", + "fb-watchman": "^2.0.2", + "graceful-fs": "^4.2.11", + "jest-regex-util": "30.4.0", + "jest-util": "30.4.1", + "jest-worker": "30.4.1", + "picomatch": "^4.0.3", "walker": "^1.0.8" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "optionalDependencies": { - "fsevents": "^2.3.2" + "fsevents": "^2.3.3" } }, "node_modules/jest-leak-detector": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-28.1.3.tgz", - "integrity": "sha512-WFVJhnQsiKtDEo5lG2mM0v40QWnBM+zMdHHyJs8AWZ7J0QZJS59MsyKeJHWhpBZBH32S48FOVvGyOFT1h0DlqA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.4.1.tgz", + "integrity": "sha512-IpmyiioeHxiWDhesHnUFmOxcTzwCwKpgACgWajtAP+nYQXiY7DakTxB6Bx9JFiRMljr0AX1PvnQdaU1KFoz6NQ==", "dev": true, "license": "MIT", "dependencies": { - "jest-get-type": "^28.0.2", - "pretty-format": "^28.1.3" + "@jest/get-type": "30.1.0", + "pretty-format": "30.4.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-matcher-utils": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-28.1.3.tgz", - "integrity": "sha512-kQeJ7qHemKfbzKoGjHHrRKH6atgxMk8Enkk2iPQ3XwO6oE/KYD8lMYOziCkeSB9G4adPM4nR1DE8Tf5JeWH6Bw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.4.1.tgz", + "integrity": "sha512-zvYfX5CaeEkFrrLS9suWe9rvJrm9J1Iv3ua8kIBv9GEPzcnsfBf0bob37la7s67fs0nlBC3EuvkOLnXQKxtx4A==", "dev": true, "license": "MIT", "dependencies": { - "chalk": "^4.0.0", - "jest-diff": "^28.1.3", - "jest-get-type": "^28.0.2", - "pretty-format": "^28.1.3" + "@jest/get-type": "30.1.0", + "chalk": "^4.1.2", + "jest-diff": "30.4.1", + "pretty-format": "30.4.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-message-util": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-28.1.3.tgz", - "integrity": "sha512-PFdn9Iewbt575zKPf1286Ht9EPoJmYT7P0kY+RibeYZ2XtOr53pDLEFoTWXbd1h4JiGiWpTBC84fc8xMXQMb7g==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.4.1.tgz", + "integrity": "sha512-kwCKIvq0MCW1HzLoGola9Te6JUdzgV0loyKJ3Qghrkz9i5/RRIHsL95BMQc2HBBhlBKC4j22K9p11TGHH8RBpQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "^28.1.3", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.4", - "pretty-format": "^28.1.3", + "@babel/code-frame": "^7.27.1", + "@jest/types": "30.4.1", + "@types/stack-utils": "^2.0.3", + "chalk": "^4.1.2", + "graceful-fs": "^4.2.11", + "jest-util": "30.4.1", + "picomatch": "^4.0.3", + "pretty-format": "30.4.1", "slash": "^3.0.0", - "stack-utils": "^2.0.3" + "stack-utils": "^2.0.6" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-mock": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-28.1.3.tgz", - "integrity": "sha512-o3J2jr6dMMWYVH4Lh/NKmDXdosrsJgi4AviS8oXLujcjpCMBb1FMsblDnOXKZKfSiHLxYub1eS0IHuRXsio9eA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.4.1.tgz", + "integrity": "sha512-/i8SVb8/NSB7RfNi8gfqu8gxLV23KaL5EpAttyb9iz8qWRIqXRLflycz/32wXsYkOnaUlx8NAKnJYtpsmXUmfw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "^28.1.3", - "@types/node": "*" + "@jest/types": "30.4.1", + "@types/node": "*", + "jest-util": "30.4.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-pnp-resolver": { @@ -4209,221 +4802,365 @@ } }, "node_modules/jest-regex-util": { - "version": "28.0.2", - "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-28.0.2.tgz", - "integrity": "sha512-4s0IgyNIy0y9FK+cjoVYoxamT7Zeo7MhzqRGx7YDYmaQn1wucY9rotiGkBzzcMXTtjrCAP/f7f+E0F7+fxPNdw==", + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-30.4.0.tgz", + "integrity": "sha512-mWlvLviKIgIQ8VCuM1xRdD0TWp3zlzionlmDBjuXVBs+VkmXq6FgW9T4Emr7oGz/Rk6feDCGyiugolcQEyp3mg==", "dev": true, "license": "MIT", "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-resolve": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-28.1.3.tgz", - "integrity": "sha512-Z1W3tTjE6QaNI90qo/BJpfnvpxtaFTFw5CDgwpyE/Kz8U/06N1Hjf4ia9quUhCh39qIGWF1ZuxFiBiJQwSEYKQ==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.4.1.tgz", + "integrity": "sha512-Zry8Yq/yJcNAZ7dJ5F2heic8AheXvbFZ7XI5V+h28nrYZ7Qoyy4dItq8OodjnYD270mvX+ZudmrNV9cysqhW5Q==", "dev": true, "license": "MIT", "dependencies": { - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^28.1.3", - "jest-pnp-resolver": "^1.2.2", - "jest-util": "^28.1.3", - "jest-validate": "^28.1.3", - "resolve": "^1.20.0", - "resolve.exports": "^1.1.0", - "slash": "^3.0.0" + "chalk": "^4.1.2", + "graceful-fs": "^4.2.11", + "jest-haste-map": "30.4.1", + "jest-pnp-resolver": "^1.2.3", + "jest-util": "30.4.1", + "jest-validate": "30.4.1", + "slash": "^3.0.0", + "unrs-resolver": "^1.7.11" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-resolve-dependencies": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-28.1.3.tgz", - "integrity": "sha512-qa0QO2Q0XzQoNPouMbCc7Bvtsem8eQgVPNkwn9LnS+R2n8DaVDPL/U1gngC0LTl1RYXJU0uJa2BMC2DbTfFrHA==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.4.2.tgz", + "integrity": "sha512-gDiVh1I+GxYzz9oXlyw+1wv6VOYX1WYxMOfjsA3iGKePV2oxmbHhwxfkALxNxYy1ciw6APWwkW2zZONwP97aEQ==", "dev": true, "license": "MIT", "dependencies": { - "jest-regex-util": "^28.0.2", - "jest-snapshot": "^28.1.3" + "jest-regex-util": "30.4.0", + "jest-snapshot": "30.4.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-runner": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-28.1.3.tgz", - "integrity": "sha512-GkMw4D/0USd62OVO0oEgjn23TM+YJa2U2Wu5zz9xsQB1MxWKDOlrnykPxnMsN0tnJllfLPinHTka61u0QhaxBA==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.4.2.tgz", + "integrity": "sha512-2dw0PslVYXxffXGpLo+Ejad+KcI1Qkjn7f4X4619gf21oCUmL+SPfjqIa/losUem3yEOvfNZe/F1HWUcNpODcg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "^28.1.3", - "@jest/environment": "^28.1.3", - "@jest/test-result": "^28.1.3", - "@jest/transform": "^28.1.3", - "@jest/types": "^28.1.3", + "@jest/console": "30.4.1", + "@jest/environment": "30.4.1", + "@jest/test-result": "30.4.1", + "@jest/transform": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", - "chalk": "^4.0.0", - "emittery": "^0.10.2", - "graceful-fs": "^4.2.9", - "jest-docblock": "^28.1.1", - "jest-environment-node": "^28.1.3", - "jest-haste-map": "^28.1.3", - "jest-leak-detector": "^28.1.3", - "jest-message-util": "^28.1.3", - "jest-resolve": "^28.1.3", - "jest-runtime": "^28.1.3", - "jest-util": "^28.1.3", - "jest-watcher": "^28.1.3", - "jest-worker": "^28.1.3", + "chalk": "^4.1.2", + "emittery": "^0.13.1", + "exit-x": "^0.2.2", + "graceful-fs": "^4.2.11", + "jest-docblock": "30.4.0", + "jest-environment-node": "30.4.1", + "jest-haste-map": "30.4.1", + "jest-leak-detector": "30.4.1", + "jest-message-util": "30.4.1", + "jest-resolve": "30.4.1", + "jest-runtime": "30.4.2", + "jest-util": "30.4.1", + "jest-watcher": "30.4.1", + "jest-worker": "30.4.1", "p-limit": "^3.1.0", "source-map-support": "0.5.13" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-runtime": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-28.1.3.tgz", - "integrity": "sha512-NU+881ScBQQLc1JHG5eJGU7Ui3kLKrmwCPPtYsJtBykixrM2OhVQlpMmFWJjMyDfdkGgBMNjXCGB/ebzsgNGQw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^28.1.3", - "@jest/fake-timers": "^28.1.3", - "@jest/globals": "^28.1.3", - "@jest/source-map": "^28.1.2", - "@jest/test-result": "^28.1.3", - "@jest/transform": "^28.1.3", - "@jest/types": "^28.1.3", - "chalk": "^4.0.0", - "cjs-module-lexer": "^1.0.0", - "collect-v8-coverage": "^1.0.0", - "execa": "^5.0.0", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^28.1.3", - "jest-message-util": "^28.1.3", - "jest-mock": "^28.1.3", - "jest-regex-util": "^28.0.2", - "jest-resolve": "^28.1.3", - "jest-snapshot": "^28.1.3", - "jest-util": "^28.1.3", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.4.2.tgz", + "integrity": "sha512-3/5e8iPz2k/VLqlr8DgTftYyLUv8Su3FkCAO2/Od81UsUTpSxOrS6O5x5KkoQwyUjmpYyDJKeyAvg2T2nvpNkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "30.4.1", + "@jest/fake-timers": "30.4.1", + "@jest/globals": "30.4.1", + "@jest/source-map": "30.0.1", + "@jest/test-result": "30.4.1", + "@jest/transform": "30.4.1", + "@jest/types": "30.4.1", + "@types/node": "*", + "chalk": "^4.1.2", + "cjs-module-lexer": "^2.1.0", + "collect-v8-coverage": "^1.0.2", + "glob": "^10.5.0", + "graceful-fs": "^4.2.11", + "jest-haste-map": "30.4.1", + "jest-message-util": "30.4.1", + "jest-mock": "30.4.1", + "jest-regex-util": "30.4.0", + "jest-resolve": "30.4.1", + "jest-snapshot": "30.4.1", + "jest-util": "30.4.1", "slash": "^3.0.0", "strip-bom": "^4.0.0" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-runtime/node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" } }, + "node_modules/jest-runtime/node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/jest-runtime/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-runtime/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, "node_modules/jest-runtime/node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", "dev": true, "license": "ISC", "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" }, - "engines": { - "node": "*" + "bin": { + "glob": "dist/esm/bin.mjs" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/jest-runtime/node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/jest-runtime/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, "node_modules/jest-runtime/node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "dev": true, "license": "ISC", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.2" }, "engines": { - "node": "*" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-runtime/node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-runtime/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-runtime/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/jest-runtime/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, "node_modules/jest-snapshot": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-28.1.3.tgz", - "integrity": "sha512-4lzMgtiNlc3DU/8lZfmqxN3AYD6GGLbl+72rdBpXvcV+whX7mDrREzkPdp2RnmfIiWBg1YbuFSkXduF2JcafJg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@babel/generator": "^7.7.2", - "@babel/plugin-syntax-typescript": "^7.7.2", - "@babel/traverse": "^7.7.2", - "@babel/types": "^7.3.3", - "@jest/expect-utils": "^28.1.3", - "@jest/transform": "^28.1.3", - "@jest/types": "^28.1.3", - "@types/babel__traverse": "^7.0.6", - "@types/prettier": "^2.1.5", - "babel-preset-current-node-syntax": "^1.0.0", - "chalk": "^4.0.0", - "expect": "^28.1.3", - "graceful-fs": "^4.2.9", - "jest-diff": "^28.1.3", - "jest-get-type": "^28.0.2", - "jest-haste-map": "^28.1.3", - "jest-matcher-utils": "^28.1.3", - "jest-message-util": "^28.1.3", - "jest-util": "^28.1.3", - "natural-compare": "^1.4.0", - "pretty-format": "^28.1.3", - "semver": "^7.3.5" + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.4.1.tgz", + "integrity": "sha512-tEOkkfOMppUyeiHwjZswOQ3lcnoTnws/q5FnGIaeIh/jmoU0ZlgMYRR8sTlTj+nNGCoJ0RDq6SfxGxCsyMTPmw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.27.4", + "@babel/generator": "^7.27.5", + "@babel/plugin-syntax-jsx": "^7.27.1", + "@babel/plugin-syntax-typescript": "^7.27.1", + "@babel/types": "^7.27.3", + "@jest/expect-utils": "30.4.1", + "@jest/get-type": "30.1.0", + "@jest/snapshot-utils": "30.4.1", + "@jest/transform": "30.4.1", + "@jest/types": "30.4.1", + "babel-preset-current-node-syntax": "^1.2.0", + "chalk": "^4.1.2", + "expect": "30.4.1", + "graceful-fs": "^4.2.11", + "jest-diff": "30.4.1", + "jest-matcher-utils": "30.4.1", + "jest-message-util": "30.4.1", + "jest-util": "30.4.1", + "pretty-format": "30.4.1", + "semver": "^7.7.2", + "synckit": "^0.11.8" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-util": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-28.1.3.tgz", - "integrity": "sha512-XdqfpHwpcSRko/C35uLYFM2emRAltIIKZiJ9eAmhjsj0CqZMa0p1ib0R5fWIqGhn1a103DebTbpqIaP1qCQ6tQ==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.4.1.tgz", + "integrity": "sha512-vjQb1sACEiv13DKJMDToJpzVW0joCsIQrmbg0fi7CyOOt+g9jTuQl2A216pWRBYhOVt53XbL/2LbMKg1BECWOw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "^28.1.3", + "@jest/types": "30.4.1", "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "graceful-fs": "^4.2.9", - "picomatch": "^2.2.3" + "chalk": "^4.1.2", + "ci-info": "^4.2.0", + "graceful-fs": "^4.2.11", + "picomatch": "^4.0.3" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-validate": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-28.1.3.tgz", - "integrity": "sha512-SZbOGBWEsaTxBGCOpsRWlXlvNkvTkY0XxRfh7zYmvd8uL5Qzyg0CHAXiXKROflh801quA6+/DsT4ODDthOC/OA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.4.1.tgz", + "integrity": "sha512-PDWi4SOwLnwqNDfHZjOcsEFyZ4fc/2W2gVL3DEoyqnB6jCQMLRtfBong8s6omIw3lI0HWOus12xfnFmQtjW3fw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "^28.1.3", - "camelcase": "^6.2.0", - "chalk": "^4.0.0", - "jest-get-type": "^28.0.2", + "@jest/get-type": "30.1.0", + "@jest/types": "30.4.1", + "camelcase": "^6.3.0", + "chalk": "^4.1.2", "leven": "^3.1.0", - "pretty-format": "^28.1.3" + "pretty-format": "30.4.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-validate/node_modules/camelcase": { @@ -4440,38 +5177,40 @@ } }, "node_modules/jest-watcher": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-28.1.3.tgz", - "integrity": "sha512-t4qcqj9hze+jviFPUN3YAtAEeFnr/azITXQEMARf5cMwKY2SMBRnCQTXLixTl20OR6mLh9KLMrgVJgJISym+1g==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.4.1.tgz", + "integrity": "sha512-/l9UonmvCwjHH7d2h3iAwIloLc1H0S8mJZ/LNK3i86hqwPAz8otUJjP9MfYtz9Tt77Su5FD2xGjZn8d31IZHlw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/test-result": "^28.1.3", - "@jest/types": "^28.1.3", + "@jest/test-result": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "emittery": "^0.10.2", - "jest-util": "^28.1.3", - "string-length": "^4.0.1" + "ansi-escapes": "^4.3.2", + "chalk": "^4.1.2", + "emittery": "^0.13.1", + "jest-util": "30.4.1", + "string-length": "^4.0.2" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-worker": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-28.1.3.tgz", - "integrity": "sha512-CqRA220YV/6jCo8VWvAt1KKx6eek1VIHMPeLEbpcfSfkEeWyBNppynM/o6q+Wmw+sOhos2ml34wZbSX3G13//g==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.4.1.tgz", + "integrity": "sha512-SHynN/q/QD++iNyvMdy+WMmbCGk8jIsNcRxycXbWubSOhvo6T+j2afcfUSl+3hYsiBebOTo0cT7c2H7CXugu1g==", "dev": true, "license": "MIT", "dependencies": { "@types/node": "*", + "@ungap/structured-clone": "^1.3.0", + "jest-util": "30.4.1", "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" + "supports-color": "^8.1.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-worker/node_modules/supports-color": { @@ -4498,14 +5237,23 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", + "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" @@ -4592,16 +5340,6 @@ "json-buffer": "3.0.1" } }, - "node_modules/kleur": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", - "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -4781,20 +5519,6 @@ "node": ">= 0.6" } }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, - "license": "MIT", - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, "node_modules/mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", @@ -4903,6 +5627,22 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/napi-postinstall": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.4.tgz", + "integrity": "sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ==", + "dev": true, + "license": "MIT", + "bin": { + "napi-postinstall": "lib/cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/napi-postinstall" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -4927,9 +5667,9 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.50", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.50.tgz", - "integrity": "sha512-J6l92tKHX6w8Jy5nO1Vuc01NoIiRGi/d6qBKVxh+IQ8Cr3b6HbVNfKiF8ZpFKufTwpwxMmce2W3iQZ861ZRyTg==", + "version": "2.0.51", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.51.tgz", + "integrity": "sha512-wRNIrw4DmVLKQlbgOMdkMx27Wrpzes2hh5Jtbi2bjPd+4wJstWIqP5A+lscnqbm0xxmT5Bpg8Lec5ItEBwx6BQ==", "dev": true, "license": "MIT", "engines": { @@ -4965,24 +5705,6 @@ "url": "https://opencollective.com/nodemon" } }, - "node_modules/nodemon/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/nodemon/node_modules/has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -5233,13 +5955,6 @@ "node": ">=8" } }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, - "license": "MIT" - }, "node_modules/path-scurry": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", @@ -5341,19 +6056,19 @@ } }, "node_modules/pretty-format": { - "version": "28.1.3", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-28.1.3.tgz", - "integrity": "sha512-8gFb/To0OmxHR9+ZTb14Df2vNxdGCX8g1xWGUTqUw5TiZvcQf5sHKObd5UcPyLLyowNwDAMTF3XWOG1B6mxl1Q==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.4.1.tgz", + "integrity": "sha512-K6KiKMHTL4jjX4u3Kir2EW07nRfcqVTXIImx50wbjHQTcZPgg+gjVeNTIT3l3L1Rd4UefxfogquC9J37SoFyyw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/schemas": "^28.1.3", - "ansi-regex": "^5.0.1", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" + "@jest/schemas": "30.4.1", + "ansi-styles": "^5.2.0", + "react-is-18": "npm:react-is@^18.3.1", + "react-is-19": "npm:react-is@^19.2.5" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/pretty-format/node_modules/ansi-styles": { @@ -5369,20 +6084,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/prompts": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", - "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "kleur": "^3.0.3", - "sisteransi": "^1.0.5" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -5422,6 +6123,23 @@ "node": ">=6" } }, + "node_modules/pure-rand": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-7.0.1.tgz", + "integrity": "sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, "node_modules/qs": { "version": "6.15.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", @@ -5481,13 +6199,22 @@ "url": "https://opencollective.com/express" } }, - "node_modules/react-is": { + "node_modules/react-is-18": { + "name": "react-is", "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", "dev": true, "license": "MIT" }, + "node_modules/react-is-19": { + "name": "react-is", + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.2.8.tgz", + "integrity": "sha512-s5un28nYxKJw5gvUHyW5PCC28CvBqLu9r3cWgzHT4Vo/5fqqkFcdRYsGcKf50WMPpjjFZS5d76fn3YCo2njKwQ==", + "dev": true, + "license": "MIT" + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -5511,27 +6238,6 @@ "node": ">=0.10.0" } }, - "node_modules/resolve": { - "version": "1.22.11", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", - "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-core-module": "^2.16.1", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/resolve-cwd": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", @@ -5555,68 +6261,6 @@ "node": ">=8" } }, - "node_modules/resolve.exports": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-1.1.1.tgz", - "integrity": "sha512-/NtpHNDN7jWhAaQ9BvBUYZ6YTXsRBgfqWFWP7BZBaoMJO/I3G5OFzvTuWNlZC3aPjins1F+TNrLKsGbH4rfsRQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/rimraf": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", - "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "deprecated": "Rimraf versions prior to v4 are no longer supported", - "dev": true, - "license": "ISC", - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/rimraf/node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/rimraf/node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -5863,23 +6507,6 @@ "url": "https://github.com/steveukx/git-js?sponsor=1" } }, - "node_modules/simple-git/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/simple-update-notifier": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", @@ -5893,13 +6520,6 @@ "node": ">=10" } }, - "node_modules/sisteransi": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", - "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", - "dev": true, - "license": "MIT" - }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -5931,13 +6551,6 @@ "source-map": "^0.6.0" } }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true, - "license": "BSD-3-Clause" - }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", @@ -5989,6 +6602,22 @@ "node": ">=8" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -6002,6 +6631,20 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-bom": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", @@ -6048,33 +6691,6 @@ "node": ">=8" } }, - "node_modules/supports-hyperlinks": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz", - "integrity": "sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0", - "supports-color": "^7.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/swagger-autogen": { "version": "2.23.7", "resolved": "https://registry.npmjs.org/swagger-autogen/-/swagger-autogen-2.23.7.tgz", @@ -6147,21 +6763,20 @@ "express": ">=4.0.0 || >=5.0.0-beta" } }, - "node_modules/terminal-link": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", - "integrity": "sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ==", + "node_modules/synckit": { + "version": "0.11.13", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.13.tgz", + "integrity": "sha512-eNRKgb3z66Yp3D2CixVujOUvXLFUTij/zVnV8KRyvFdQwpz7I5DS8UfRkTeLzb64u+dkzDSdelE24izu+zSSUg==", "dev": true, "license": "MIT", "dependencies": { - "ansi-escapes": "^4.2.1", - "supports-hyperlinks": "^2.0.0" + "@pkgr/core": "^0.3.6" }, "engines": { - "node": ">=8" + "node": "^14.18.0 || >=16.0.0" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "url": "https://opencollective.com/synckit" } }, "node_modules/test-exclude": { @@ -6322,9 +6937,9 @@ "license": "MIT" }, "node_modules/undici-types": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", - "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", "dev": true, "license": "MIT" }, @@ -6337,6 +6952,44 @@ "node": ">= 0.8" } }, + "node_modules/unrs-resolver": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/unrs-resolver/-/unrs-resolver-1.12.2.tgz", + "integrity": "sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "napi-postinstall": "^0.3.4" + }, + "funding": { + "url": "https://opencollective.com/unrs-resolver" + }, + "optionalDependencies": { + "@unrs/resolver-binding-android-arm-eabi": "1.12.2", + "@unrs/resolver-binding-android-arm64": "1.12.2", + "@unrs/resolver-binding-darwin-arm64": "1.12.2", + "@unrs/resolver-binding-darwin-x64": "1.12.2", + "@unrs/resolver-binding-freebsd-x64": "1.12.2", + "@unrs/resolver-binding-linux-arm-gnueabihf": "1.12.2", + "@unrs/resolver-binding-linux-arm-musleabihf": "1.12.2", + "@unrs/resolver-binding-linux-arm64-gnu": "1.12.2", + "@unrs/resolver-binding-linux-arm64-musl": "1.12.2", + "@unrs/resolver-binding-linux-loong64-gnu": "1.12.2", + "@unrs/resolver-binding-linux-loong64-musl": "1.12.2", + "@unrs/resolver-binding-linux-ppc64-gnu": "1.12.2", + "@unrs/resolver-binding-linux-riscv64-gnu": "1.12.2", + "@unrs/resolver-binding-linux-riscv64-musl": "1.12.2", + "@unrs/resolver-binding-linux-s390x-gnu": "1.12.2", + "@unrs/resolver-binding-linux-x64-gnu": "1.12.2", + "@unrs/resolver-binding-linux-x64-musl": "1.12.2", + "@unrs/resolver-binding-openharmony-arm64": "1.12.2", + "@unrs/resolver-binding-wasm32-wasi": "1.12.2", + "@unrs/resolver-binding-win32-arm64-msvc": "1.12.2", + "@unrs/resolver-binding-win32-ia32-msvc": "1.12.2", + "@unrs/resolver-binding-win32-x64-msvc": "1.12.2" + } + }, "node_modules/update-browserslist-db": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", @@ -6402,13 +7055,6 @@ "node": ">=10.12.0" } }, - "node_modules/v8-to-istanbul/node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, "node_modules/validator": { "version": "13.15.26", "resolved": "https://registry.npmjs.org/validator/-/validator-13.15.26.tgz", @@ -6480,6 +7126,25 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -6488,26 +7153,19 @@ "license": "ISC" }, "node_modules/write-file-atomic": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", - "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", + "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dev": true, "license": "ISC", "dependencies": { "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.7" + "signal-exit": "^4.0.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, - "node_modules/write-file-atomic/node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true, - "license": "ISC" - }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -6526,9 +7184,9 @@ "license": "ISC" }, "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 1b752e5..406ea41 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "@eslint/js": "^10.0.1", "eslint": "^10.6.0", "globals": "^17.7.0", - "jest": "^28.1.3", + "jest": "^30.4.2", "nodemon": "^3.1.9", "prettier": "^3.9.3", "swagger-autogen": "^2.23.7" @@ -36,6 +36,7 @@ "overrides": { "axios": "1.18.0", "brace-expansion": "5.0.8", + "js-yaml": "4.3.0", "lodash": "4.18.1", "lodash.unset": "4.18.0", "path-to-regexp": "0.1.13", diff --git a/routes/index.js b/routes/index.js index 966044f..e76a818 100644 --- a/routes/index.js +++ b/routes/index.js @@ -171,10 +171,10 @@ router.post( /* #swagger.tags = ['CVE Record'] #swagger.operationId = 'search' -#swagger.summary = "Retrieves all matching CVE records after applying the JSON request body as filters" +#swagger.summary = "Retrieves a page of CVE records after applying the JSON request body as filters" #swagger.description = "

Expected Behavior

-

Retrieves all matching CVE records after applying the JSON request body as filters

+

Retrieves one page of matching CVE records after applying the JSON request body as filters. Pagination metadata reports the accessible result window.

#swagger.requestBody = { required: false, description: "Optional JSON filters and pagination controls. The JSON request body cannot exceed 100 KB.", diff --git a/routes/swagger.js b/routes/swagger.js index 674ae6b..aba5b68 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -145,6 +145,7 @@ const searchRequestProperties = { { type: 'string', format: 'date-time', + pattern: constants.DATE_TIME_PATTERN, example: '2024-01-01T00:00:00.000Z' } ), @@ -153,6 +154,7 @@ const searchRequestProperties = { { type: 'string', format: 'date-time', + pattern: constants.DATE_TIME_PATTERN, example: '2024-12-31T23:59:59.999Z' } ), @@ -161,6 +163,7 @@ const searchRequestProperties = { { type: 'string', format: 'date-time', + pattern: constants.DATE_TIME_PATTERN, example: '2024-01-01T00:00:00.000Z' } ), @@ -169,6 +172,7 @@ const searchRequestProperties = { { type: 'string', format: 'date-time', + pattern: constants.DATE_TIME_PATTERN, example: '2024-12-31T23:59:59.999Z' } ), @@ -373,7 +377,7 @@ const doc = { info: { version: packageInfo.version, title: 'CVE Search API', - description: 'CVE Search API to retrieve all matching CVE records after applying request inputs as filters.', + description: 'CVE Search API for retrieving paginated CVE records after applying request inputs as filters.', contact: { name: 'CVE Overview', url: 'https://www.cve.org/' diff --git a/test/contract/opensearch.js b/test/contract/opensearch.js index 0467d2d..bf3606f 100644 --- a/test/contract/opensearch.js +++ b/test/contract/opensearch.js @@ -1,10 +1,11 @@ -const assert = require('assert'); +const assert = require('assert/strict'); require('dotenv').config(); const REQUIRED_ENVIRONMENT_VARIABLES = ['OpenSearchDomainEndpoint', 'OpenSearchCveIndex']; +const CONTRACT_CVE_ID = 'CVE-0000-0000'; -function requireConfiguration() { - const missing = REQUIRED_ENVIRONMENT_VARIABLES.filter((name) => !process.env[name]?.trim()); +function requireConfiguration(environment = process.env) { + const missing = REQUIRED_ENVIRONMENT_VARIABLES.filter((name) => !environment[name]?.trim()); if (missing.length > 0) { throw new Error(`Missing required environment variables: ${missing.join(', ')}`); } @@ -22,36 +23,123 @@ function getSanitizedFailure(err) { }; } -async function run() { - requireConfiguration(); +function buildContractChecks(queryBuilder = require('../../utils/searchQueryBuilder')) { + const buildStandardBody = (params, options = {}) => { + const request = queryBuilder.buildSearchRequest(params); + return queryBuilder.buildOpenSearchBody(request.searchOptions, request.queryStringObjects, { + trackTotalHits: false, + ...options + }); + }; + const buildCandidateBody = (params) => { + const request = queryBuilder.buildSearchRequest(params); + return queryBuilder.buildCpeCandidateSearchBody(request.searchOptions, request.queryStringObjects); + }; - const searchProvider = require('../../utils/searchProvider'); - try { - await searchProvider.checkOpenSearchReadiness(); - const body = await searchProvider.searchOpenSearch( - { - query: { - match_all: {} + return [ + { + name: 'source-projection', + body: buildStandardBody( + { + resultsPerPage: 1, + pageNumber: 1 }, - size: 1, - track_total_hits: false, - _source: ['cveMetadata.cveId'] - }, - { - operation: 'contract-check' + { + sourceIncludes: ['cveMetadata.cveId'] + } + ), + validate(body) { + body.hits.hits.forEach((hit) => { + assert.equal(typeof hit._id, 'string'); + assert.equal(typeof hit._source?.cveMetadata?.cveId, 'string'); + }); } - ); + }, + { + name: 'cve-id-keyword-and-total', + body: buildStandardBody( + { + cveIds: CONTRACT_CVE_ID, + resultsPerPage: 1, + pageNumber: 1 + }, + { + trackTotalHits: true + } + ), + operation: 'standard', + validate(body) { + assert.equal(Number.isInteger(body.hits.total.value), true); + assert.equal(body.hits.total.value >= 0, true); + } + }, + { + name: 'keyword', + body: buildStandardBody({ + keywordSearch: 'contract-probe-value-that-should-not-exist', + resultsPerPage: 1, + pageNumber: 1 + }) + }, + { + name: 'publication-date', + body: buildStandardBody({ + pubStartDate: '1970-01-01T00:00:00.000Z', + resultsPerPage: 1, + pageNumber: 1 + }) + }, + { + name: 'cvss', + body: buildStandardBody({ + cvssV3Severity: 'CRITICAL', + resultsPerPage: 1, + pageNumber: 1 + }) + }, + { + name: 'cpe-candidate', + body: { + ...buildCandidateBody({ + virtualMatchString: 'cpe:2.3:a:contract-probe:contract-probe', + resultsPerPage: 1, + pageNumber: 1 + }), + size: 0 + } + } + ]; +} + +function validateContractResponse(check, body) { + assert.ok(Array.isArray(body.hits.hits), `${check.name} response did not include hits.hits`); + check.validate?.(body); +} + +async function run({ + environment = process.env, + output = process.stdout, + searchProvider = require('../../utils/searchProvider'), + queryBuilder = require('../../utils/searchQueryBuilder') +} = {}) { + requireConfiguration(environment); + const completedChecks = []; - assert.ok(Array.isArray(body.hits.hits)); - if (body.hits.hits.length > 0) { - assert.strictEqual(typeof body.hits.hits[0]._id, 'string'); + try { + await searchProvider.checkOpenSearchReadiness(); + for (const check of buildContractChecks(queryBuilder)) { + const body = await searchProvider.searchOpenSearch(check.body, { + operation: check.operation ?? `contract-${check.name}` + }); + validateContractResponse(check, body); + completedChecks.push(check.name); } - process.stdout.write( + output.write( `${JSON.stringify({ event: 'opensearch.contract', outcome: 'passed', - returnedHits: body.hits.hits.length + checks: completedChecks })}\n` ); } finally { @@ -59,7 +147,17 @@ async function run() { } } -run().catch((err) => { - process.stderr.write(`${JSON.stringify(getSanitizedFailure(err))}\n`); - process.exitCode = 1; -}); +if (require.main === module) { + run().catch((err) => { + process.stderr.write(`${JSON.stringify(getSanitizedFailure(err))}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + buildContractChecks, + getSanitizedFailure, + requireConfiguration, + run, + validateContractResponse +}; diff --git a/test/contract/opensearch.test.js b/test/contract/opensearch.test.js new file mode 100644 index 0000000..9f76d44 --- /dev/null +++ b/test/contract/opensearch.test.js @@ -0,0 +1,172 @@ +const contract = require('./opensearch'); + +function validResponse(overrides = {}) { + return { + timed_out: false, + _shards: { + failed: 0 + }, + hits: { + total: { + value: 0, + relation: 'eq' + }, + hits: [] + }, + ...overrides + }; +} + +describe('OpenSearch contract check', () => { + test('requires the endpoint and index without reporting their values', () => { + expect(() => contract.requireConfiguration({})).toThrow( + 'Missing required environment variables: OpenSearchDomainEndpoint, OpenSearchCveIndex' + ); + expect(() => + contract.requireConfiguration({ + OpenSearchDomainEndpoint: 'https://search.example.com', + OpenSearchCveIndex: 'cves' + }) + ).not.toThrow(); + }); + + test('builds representative read-only production query shapes', () => { + const checks = contract.buildContractChecks(); + + expect(checks.map(({ name }) => name)).toStrictEqual([ + 'source-projection', + 'cve-id-keyword-and-total', + 'keyword', + 'publication-date', + 'cvss', + 'cpe-candidate' + ]); + expect(checks.every(({ body }) => body.size <= 1)).toBe(true); + expect(checks.find(({ name }) => name === 'source-projection').body._source).toStrictEqual([ + 'cveMetadata.cveId' + ]); + expect(checks.find(({ name }) => name === 'cve-id-keyword-and-total').body).toMatchObject({ + track_total_hits: true, + sort: [ + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ] + }); + expect(checks.find(({ name }) => name === 'keyword').body.query.bool.must[0]).toHaveProperty( + 'simple_query_string' + ); + expect(checks.find(({ name }) => name === 'publication-date').body.query.bool.filter[0].range).toHaveProperty([ + 'cveMetadata.datePublished' + ]); + expect(checks.find(({ name }) => name === 'cvss').body.query.bool.filter[0]).toHaveProperty('bool.should'); + expect(checks.find(({ name }) => name === 'cpe-candidate').body).toMatchObject({ + size: 0, + track_total_hits: false + }); + }); + + test('runs every contract check and closes the provider', async () => { + const output = { + write: jest.fn() + }; + const searchProvider = { + checkOpenSearchReadiness: jest.fn().mockResolvedValue(true), + searchOpenSearch: jest.fn().mockImplementation((_body, { operation }) => { + if (operation === 'standard') return Promise.resolve(validResponse()); + if (operation === 'contract-source-projection') { + return Promise.resolve( + validResponse({ + hits: { + hits: [ + { + _id: 'CVE-2024-0001', + _source: { + cveMetadata: { + cveId: 'CVE-2024-0001' + } + } + } + ] + } + }) + ); + } + return Promise.resolve(validResponse()); + }), + closeSearchProviderClients: jest.fn().mockResolvedValue(undefined) + }; + + await contract.run({ + environment: { + OpenSearchDomainEndpoint: 'https://search.example.com', + OpenSearchCveIndex: 'cves' + }, + output, + searchProvider + }); + + expect(searchProvider.checkOpenSearchReadiness).toHaveBeenCalledTimes(1); + expect(searchProvider.searchOpenSearch).toHaveBeenCalledTimes(6); + expect(searchProvider.closeSearchProviderClients).toHaveBeenCalledTimes(1); + expect(JSON.parse(output.write.mock.calls[0][0])).toStrictEqual({ + event: 'opensearch.contract', + outcome: 'passed', + checks: [ + 'source-projection', + 'cve-id-keyword-and-total', + 'keyword', + 'publication-date', + 'cvss', + 'cpe-candidate' + ] + }); + }); + + test('closes the provider and fails when a contract response is invalid', async () => { + const searchProvider = { + checkOpenSearchReadiness: jest.fn().mockResolvedValue(true), + searchOpenSearch: jest.fn().mockResolvedValue({ + hits: {} + }), + closeSearchProviderClients: jest.fn().mockResolvedValue(undefined) + }; + + await expect( + contract.run({ + environment: { + OpenSearchDomainEndpoint: 'https://search.example.com', + OpenSearchCveIndex: 'cves' + }, + output: { + write: jest.fn() + }, + searchProvider + }) + ).rejects.toThrow(); + expect(searchProvider.searchOpenSearch).toHaveBeenCalledTimes(1); + expect(searchProvider.closeSearchProviderClients).toHaveBeenCalledTimes(1); + }); + + test('sanitizes reported failures', () => { + const failure = contract.getSanitizedFailure({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'partial_shards', + message: 'https://user:secret@search.example.com', + meta: { + statusCode: 502 + } + }); + + expect(failure).toStrictEqual({ + event: 'opensearch.contract', + outcome: 'failed', + errorCode: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'partial_shards', + statusCode: 502 + }); + expect(JSON.stringify(failure)).not.toContain('secret'); + }); +}); diff --git a/utils/constants.js b/utils/constants.js index b422d8a..483a1b2 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -10,6 +10,8 @@ const MAX_CVE_ID_SEQUENCE_DIGITS = 19; const MAX_KEYWORD_SEARCH_TOKENS = 64; const JSON_REQUEST_BODY_LIMIT_KB = 100; const JSON_REQUEST_BODY_LIMIT_BYTES = JSON_REQUEST_BODY_LIMIT_KB * 1024; +const DATE_TIME_PATTERN = + '^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])T([01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d\\.\\d{3}Z$'; const SEARCH_STRING_LENGTH_LIMITS = Object.freeze({ keywordSearch: 512, cveIds: 4096, @@ -141,6 +143,7 @@ module.exports = { MAX_KEYWORD_SEARCH_TOKENS, JSON_REQUEST_BODY_LIMIT_KB, JSON_REQUEST_BODY_LIMIT_BYTES, + DATE_TIME_PATTERN, SEARCH_STRING_LENGTH_LIMITS, SEARCH_POST_FILTER_CACHE_TTL_MS, SEARCH_POST_FILTER_CACHE_MAX_ENTRIES, From 0c5a2ccbf7a7f91dc49d7d9daee9ea423c7b5d7e Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 10:31:33 -0400 Subject: [PATCH 18/33] Harden OpenSearch contract delivery checks --- .github/workflows/opensearch-contract.yml | 8 +- .github/workflows/quality.yml | 16 ++- .trivyignore | 3 +- README.md | 12 +- test/contract/opensearch.js | 148 +++++++++++++++++----- test/contract/opensearch.test.js | 125 +++++++++++++++++- test/delivery-policy.test.js | 84 ++++++++++++ utils/searchProvider.js | 34 +++++ utils/searchProvider.test.js | 50 ++++++++ 9 files changed, 430 insertions(+), 50 deletions(-) create mode 100644 test/delivery-policy.test.js diff --git a/.github/workflows/opensearch-contract.yml b/.github/workflows/opensearch-contract.yml index 51ba093..062f5dd 100644 --- a/.github/workflows/opensearch-contract.yml +++ b/.github/workflows/opensearch-contract.yml @@ -18,6 +18,7 @@ concurrency: jobs: contract: name: Read-only OpenSearch Contract + if: github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 15 environment: ${{ inputs.environment }} @@ -25,9 +26,6 @@ jobs: env: NODE_ENV: production NODE_CONFIG_ENV: prod - OpenSearchDomainEndpoint: ${{ secrets.OPENSEARCH_DOMAIN_ENDPOINT }} - OpenSearchCveIndex: ${{ secrets.OPENSEARCH_CVE_INDEX }} - OpenSearchAllowUnknownSslCerts: ${{ vars.OPENSEARCH_ALLOW_UNKNOWN_SSL_CERTS || 'false' }} steps: - name: Check out repository @@ -44,3 +42,7 @@ jobs: - name: Run read-only contract checks run: npm run test:opensearch-contract + env: + OpenSearchDomainEndpoint: ${{ secrets.OPENSEARCH_DOMAIN_ENDPOINT }} + OpenSearchCveIndex: ${{ secrets.OPENSEARCH_CVE_INDEX }} + OpenSearchAllowUnknownSslCerts: ${{ vars.OPENSEARCH_ALLOW_UNKNOWN_SSL_CERTS || 'false' }} diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 1cf9ce2..068d766 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -65,7 +65,21 @@ jobs: - name: Build production image run: docker build --tag cve-search-api:pr . - - name: Scan production image + - name: Report all high and critical image vulnerabilities + continue-on-error: true + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + scan-type: image + image-ref: cve-search-api:pr + format: table + exit-code: 0 + ignore-unfixed: false + severity: CRITICAL,HIGH + scanners: vuln + trivyignores: /dev/null + version: v0.69.3 + + - name: Enforce fixable high and critical image vulnerabilities uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: scan-type: image diff --git a/.trivyignore b/.trivyignore index e4830c8..f5a0176 100644 --- a/.trivyignore +++ b/.trivyignore @@ -1,2 +1,3 @@ +# package=adm-zip owner=cve-core fixed=0.6.0 review-by=2026-10-31 # cve-core transitively pins adm-zip below 0.6.0. Remediation belongs in cve-core. -GHSA-xcpc-8h2w-3j85 +CVE-2026-39244 diff --git a/README.md b/README.md index 0e5f2c8..146efb0 100644 --- a/README.md +++ b/README.md @@ -310,9 +310,9 @@ npm run coverage npm run quality ``` -Pull requests, merge-queue validation, and pushes to `dev` or `main` run the quality gate and independently build the production `Dockerfile`. The container job scans the built image for fixed high and critical vulnerabilities, starts it, checks process liveness and the packaged OpenAPI document, and verifies the runtime user and command. It intentionally does not call `/api/ready-check` or `/search`, because the standard CI workflow does not own or provision an OpenSearch index. +Pull requests, merge-queue validation, and pushes to `dev` or `main` run the quality gate and independently build the production `Dockerfile`. The container job reports all high and critical image vulnerabilities without blocking, then rejects fixed high and critical vulnerabilities that are not covered by an approved exception. It starts the image, checks process liveness and the packaged OpenAPI document, and verifies the runtime user and command. It intentionally does not call `/api/ready-check` or `/search`, because the standard CI workflow does not own or provision an OpenSearch index. -Pull requests also run GitHub dependency review and reject newly introduced high or critical vulnerabilities in runtime or development dependencies. The production image scan excludes only `GHSA-xcpc-8h2w-3j85`, an `adm-zip` advisory inherited from the separately owned `cve-core` dependency. New advisories are not covered by that exception. +Pull requests also run GitHub dependency review and reject newly introduced high or critical vulnerabilities in runtime or development dependencies. The blocking production image scan excludes only `CVE-2026-39244`, an `adm-zip` advisory inherited from the separately owned `cve-core` dependency. The exception records its package owner, fixed version, and review deadline in `.trivyignore`; repository tests fail after that deadline or if the dependency provenance changes. The non-blocking scan still reports the exception and other unfixed severe findings. New advisories are not covered by the exception. An opt-in, read-only OpenSearch contract check is available for an approved environment that can reach an existing index: @@ -320,7 +320,7 @@ An opt-in, read-only OpenSearch contract check is available for an approved envi npm run test:opensearch-contract ``` -Set `OpenSearchDomainEndpoint` and `OpenSearchCveIndex` in the environment or local `.env` before running it. `OpenSearchAllowUnknownSslCerts` remains optional and should be enabled only for local development against a self-signed certificate. The contract check sends a minimal readiness query followed by representative bounded searches, validates their response shapes, closes its OpenSearch client, and emits sanitized JSON output. It never creates, updates, maps, or deletes indexes. +Set `OpenSearchDomainEndpoint` and `OpenSearchCveIndex` in the environment or local `.env` before running it. `OpenSearchAllowUnknownSslCerts` remains optional and should be enabled only for local development against a self-signed certificate. The contract check sends a minimal readiness query, validates representative field capabilities, requires a concrete CVE record, and uses that record for a positive CVE-ID search before running the remaining bounded query shapes. It closes its OpenSearch client and emits sanitized JSON output. It never creates, updates, maps, or deletes indexes. The contract check is not part of public pull-request CI. The `OpenSearch Contract` GitHub Actions workflow runs it only through a manual dispatch against a selected protected GitHub environment. Configure that environment with: @@ -328,13 +328,15 @@ The contract check is not part of public pull-request CI. The `OpenSearch Contra - `OPENSEARCH_CVE_INDEX` secret: the existing CVE index name - `OPENSEARCH_ALLOW_UNKNOWN_SSL_CERTS` variable: optional and normally `false`; use `true` only for an approved non-production endpoint with a self-signed certificate -The selected runner must have network access to the endpoint. Change the workflow to an approved self-hosted runner before use when the index is not reachable from GitHub-hosted runners. The workflow performs readiness, source projection, exact-total, CVE ID keyword-subfield, keyword, publication-date, CVSS, and CPE candidate checks. Every query is read-only and returns at most one record. Connection values and credentials remain in environment secrets rather than source files or workflow text. +The workflow can run only from the trusted `dev` or `main` branch. The selected GitHub environment should additionally require reviewers and restrict deployment branches. OpenSearch secrets are exposed only to the final contract execution step, after checkout, Node setup, and dependency installation have completed. + +The selected runner must have network access to the endpoint. Change the workflow to an approved self-hosted runner before use when the index is not reachable from GitHub-hosted runners. The workflow performs readiness, field-capability, positive source-projection, exact-total, CVE ID keyword-subfield, keyword, publication-date, CVSS, and CPE candidate checks. Every search is read-only and returns at most one record. Connection values and credentials remain in environment secrets rather than source files or workflow text. The validation tiers have separate purposes: - Jest and `npm run quality` verify application behavior, API contracts, and regression coverage with controlled test doubles. - Pull-request container validation verifies production-image packaging, non-root execution, startup, liveness, and packaged API documentation. -- The optional OpenSearch contract check verifies that the installed client can communicate with an externally managed index using the response contract expected by this application. +- The optional OpenSearch contract check verifies that the installed client can communicate with a non-empty externally managed index whose representative searchable fields satisfy the application contract. - Load and performance testing remain in the separate automation performance-testing repository. Dependabot checks npm packages, Docker base-image digests, and GitHub Actions weekly. Both Dockerfiles pin the Node build image and Distroless runtime image to immutable multi-architecture digests; Dependabot updates those references through reviewable pull requests. diff --git a/test/contract/opensearch.js b/test/contract/opensearch.js index bf3606f..56e71a8 100644 --- a/test/contract/opensearch.js +++ b/test/contract/opensearch.js @@ -2,7 +2,36 @@ const assert = require('assert/strict'); require('dotenv').config(); const REQUIRED_ENVIRONMENT_VARIABLES = ['OpenSearchDomainEndpoint', 'OpenSearchCveIndex']; -const CONTRACT_CVE_ID = 'CVE-0000-0000'; +const FIELD_CAPABILITY_REQUIREMENTS = Object.freeze([ + { + name: 'cve-id-keyword', + anyOf: [{ field: 'cveMetadata.cveId.keyword', types: ['keyword'] }] + }, + { + name: 'description-search', + anyOf: [{ field: 'containers.cna.descriptions.value', types: ['text', 'keyword'] }] + }, + { + name: 'publication-date', + anyOf: [{ field: 'cveMetadata.datePublished', types: ['date', 'date_nanos'] }] + }, + { + name: 'cvss-v3-severity', + anyOf: [ + { field: 'containers.cna.metrics.cvssV3_0.baseSeverity', types: ['text', 'keyword'] }, + { field: 'containers.cna.metrics.cvssV3_1.baseSeverity', types: ['text', 'keyword'] } + ] + }, + { + name: 'cpe-criteria-keyword', + anyOf: [ + { + field: 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword', + types: ['keyword'] + } + ] + } +]); function requireConfiguration(environment = process.env) { const missing = REQUIRED_ENVIRONMENT_VARIABLES.filter((name) => !environment[name]?.trim()); @@ -23,13 +52,34 @@ function getSanitizedFailure(err) { }; } -function buildContractChecks(queryBuilder = require('../../utils/searchQueryBuilder')) { - const buildStandardBody = (params, options = {}) => { - const request = queryBuilder.buildSearchRequest(params); - return queryBuilder.buildOpenSearchBody(request.searchOptions, request.queryStringObjects, { - trackTotalHits: false, - ...options - }); +function buildStandardBody(queryBuilder, params, options = {}) { + const request = queryBuilder.buildSearchRequest(params); + return queryBuilder.buildOpenSearchBody(request.searchOptions, request.queryStringObjects, { + trackTotalHits: false, + ...options + }); +} + +function buildSourceProjectionCheck(queryBuilder = require('../../utils/searchQueryBuilder')) { + return { + name: 'source-projection', + body: buildStandardBody( + queryBuilder, + { + resultsPerPage: 1, + pageNumber: 1 + }, + { + sourceIncludes: ['cveMetadata.cveId'] + } + ) + }; +} + +function buildContractChecks(contractCveId, queryBuilder = require('../../utils/searchQueryBuilder')) { + assert.equal(typeof contractCveId, 'string'); + const buildBody = (params, options = {}) => { + return buildStandardBody(queryBuilder, params, options); }; const buildCandidateBody = (params) => { const request = queryBuilder.buildSearchRequest(params); @@ -37,29 +87,11 @@ function buildContractChecks(queryBuilder = require('../../utils/searchQueryBuil }; return [ - { - name: 'source-projection', - body: buildStandardBody( - { - resultsPerPage: 1, - pageNumber: 1 - }, - { - sourceIncludes: ['cveMetadata.cveId'] - } - ), - validate(body) { - body.hits.hits.forEach((hit) => { - assert.equal(typeof hit._id, 'string'); - assert.equal(typeof hit._source?.cveMetadata?.cveId, 'string'); - }); - } - }, { name: 'cve-id-keyword-and-total', - body: buildStandardBody( + body: buildBody( { - cveIds: CONTRACT_CVE_ID, + cveIds: contractCveId, resultsPerPage: 1, pageNumber: 1 }, @@ -70,12 +102,13 @@ function buildContractChecks(queryBuilder = require('../../utils/searchQueryBuil operation: 'standard', validate(body) { assert.equal(Number.isInteger(body.hits.total.value), true); - assert.equal(body.hits.total.value >= 0, true); + assert.equal(body.hits.total.value >= 1, true); + assert.equal(body.hits.hits[0]?._source?.cveMetadata?.cveId, contractCveId); } }, { name: 'keyword', - body: buildStandardBody({ + body: buildBody({ keywordSearch: 'contract-probe-value-that-should-not-exist', resultsPerPage: 1, pageNumber: 1 @@ -83,7 +116,7 @@ function buildContractChecks(queryBuilder = require('../../utils/searchQueryBuil }, { name: 'publication-date', - body: buildStandardBody({ + body: buildBody({ pubStartDate: '1970-01-01T00:00:00.000Z', resultsPerPage: 1, pageNumber: 1 @@ -91,7 +124,7 @@ function buildContractChecks(queryBuilder = require('../../utils/searchQueryBuil }, { name: 'cvss', - body: buildStandardBody({ + body: buildBody({ cvssV3Severity: 'CRITICAL', resultsPerPage: 1, pageNumber: 1 @@ -111,6 +144,37 @@ function buildContractChecks(queryBuilder = require('../../utils/searchQueryBuil ]; } +function getRequiredFieldNames(requirements = FIELD_CAPABILITY_REQUIREMENTS) { + return [...new Set(requirements.flatMap((requirement) => requirement.anyOf.map(({ field }) => field)))]; +} + +function validateFieldCapabilities(body, requirements = FIELD_CAPABILITY_REQUIREMENTS) { + assert.ok(body && typeof body === 'object' && !Array.isArray(body), 'field capabilities response was invalid'); + assert.ok( + body.fields && typeof body.fields === 'object' && !Array.isArray(body.fields), + 'field capabilities response did not include fields' + ); + + requirements.forEach((requirement) => { + const satisfied = requirement.anyOf.some(({ field, types }) => + Object.values(body.fields[field] ?? {}).some( + (capability) => capability?.searchable === true && types.includes(capability.type) + ) + ); + assert.ok(satisfied, `required field capability was not satisfied: ${requirement.name}`); + }); +} + +function getContractCveId(body) { + assert.equal(body.hits.hits.length, 1, 'source projection did not return a CVE record'); + const [hit] = body.hits.hits; + const sourceCveId = hit?._source?.cveMetadata?.cveId; + + assert.equal(typeof hit?._id, 'string'); + assert.equal(typeof sourceCveId, 'string'); + return sourceCveId; +} + function validateContractResponse(check, body) { assert.ok(Array.isArray(body.hits.hits), `${check.name} response did not include hits.hits`); check.validate?.(body); @@ -127,7 +191,19 @@ async function run({ try { await searchProvider.checkOpenSearchReadiness(); - for (const check of buildContractChecks(queryBuilder)) { + const fieldCapabilities = await searchProvider.getOpenSearchFieldCapabilities(getRequiredFieldNames()); + validateFieldCapabilities(fieldCapabilities); + completedChecks.push('field-capabilities'); + + const sourceProjectionCheck = buildSourceProjectionCheck(queryBuilder); + const sourceProjectionBody = await searchProvider.searchOpenSearch(sourceProjectionCheck.body, { + operation: `contract-${sourceProjectionCheck.name}` + }); + validateContractResponse(sourceProjectionCheck, sourceProjectionBody); + const contractCveId = getContractCveId(sourceProjectionBody); + completedChecks.push(sourceProjectionCheck.name); + + for (const check of buildContractChecks(contractCveId, queryBuilder)) { const body = await searchProvider.searchOpenSearch(check.body, { operation: check.operation ?? `contract-${check.name}` }); @@ -156,8 +232,14 @@ if (require.main === module) { module.exports = { buildContractChecks, + buildSourceProjectionCheck, + buildStandardBody, + FIELD_CAPABILITY_REQUIREMENTS, + getContractCveId, getSanitizedFailure, + getRequiredFieldNames, requireConfiguration, run, + validateFieldCapabilities, validateContractResponse }; diff --git a/test/contract/opensearch.test.js b/test/contract/opensearch.test.js index 9f76d44..e95bb39 100644 --- a/test/contract/opensearch.test.js +++ b/test/contract/opensearch.test.js @@ -17,6 +17,29 @@ function validResponse(overrides = {}) { }; } +function validFieldCapabilities() { + return { + fields: Object.fromEntries( + contract.getRequiredFieldNames().map((field) => { + let type = 'keyword'; + if (field === 'containers.cna.descriptions.value') type = 'text'; + if (field === 'cveMetadata.datePublished') type = 'date'; + if (field.includes('cvssV3')) type = 'text'; + + return [ + field, + { + [type]: { + searchable: true, + type + } + } + ]; + }) + ) + }; +} + describe('OpenSearch contract check', () => { test('requires the endpoint and index without reporting their values', () => { expect(() => contract.requireConfiguration({})).toThrow( @@ -31,9 +54,10 @@ describe('OpenSearch contract check', () => { }); test('builds representative read-only production query shapes', () => { - const checks = contract.buildContractChecks(); + const sourceProjection = contract.buildSourceProjectionCheck(); + const checks = contract.buildContractChecks('CVE-2024-0001'); - expect(checks.map(({ name }) => name)).toStrictEqual([ + expect([sourceProjection, ...checks].map(({ name }) => name)).toStrictEqual([ 'source-projection', 'cve-id-keyword-and-total', 'keyword', @@ -41,10 +65,8 @@ describe('OpenSearch contract check', () => { 'cvss', 'cpe-candidate' ]); - expect(checks.every(({ body }) => body.size <= 1)).toBe(true); - expect(checks.find(({ name }) => name === 'source-projection').body._source).toStrictEqual([ - 'cveMetadata.cveId' - ]); + expect([sourceProjection, ...checks].every(({ body }) => body.size <= 1)).toBe(true); + expect(sourceProjection.body._source).toStrictEqual(['cveMetadata.cveId']); expect(checks.find(({ name }) => name === 'cve-id-keyword-and-total').body).toMatchObject({ track_total_hits: true, sort: [ @@ -68,14 +90,76 @@ describe('OpenSearch contract check', () => { }); }); + test('requires every representative field capability', () => { + expect(() => contract.validateFieldCapabilities(validFieldCapabilities())).not.toThrow(); + + const missingCpe = validFieldCapabilities(); + delete missingCpe.fields['containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword']; + expect(() => contract.validateFieldCapabilities(missingCpe)).toThrow( + 'required field capability was not satisfied: cpe-criteria-keyword' + ); + + const unsearchableDate = validFieldCapabilities(); + unsearchableDate.fields['cveMetadata.datePublished'].date.searchable = false; + expect(() => contract.validateFieldCapabilities(unsearchableDate)).toThrow( + 'required field capability was not satisfied: publication-date' + ); + }); + + test('requires source projection to return a concrete CVE record', () => { + expect(() => contract.getContractCveId(validResponse())).toThrow( + 'source projection did not return a CVE record' + ); + expect( + contract.getContractCveId( + validResponse({ + hits: { + hits: [ + { + _id: 'record-1', + _source: { + cveMetadata: { + cveId: 'CVE-2024-0001' + } + } + } + ] + } + }) + ) + ).toBe('CVE-2024-0001'); + }); + test('runs every contract check and closes the provider', async () => { const output = { write: jest.fn() }; const searchProvider = { checkOpenSearchReadiness: jest.fn().mockResolvedValue(true), + getOpenSearchFieldCapabilities: jest.fn().mockResolvedValue(validFieldCapabilities()), searchOpenSearch: jest.fn().mockImplementation((_body, { operation }) => { - if (operation === 'standard') return Promise.resolve(validResponse()); + if (operation === 'standard') { + return Promise.resolve( + validResponse({ + hits: { + total: { + value: 1, + relation: 'eq' + }, + hits: [ + { + _id: 'CVE-2024-0001', + _source: { + cveMetadata: { + cveId: 'CVE-2024-0001' + } + } + } + ] + } + }) + ); + } if (operation === 'contract-source-projection') { return Promise.resolve( validResponse({ @@ -109,12 +193,14 @@ describe('OpenSearch contract check', () => { }); expect(searchProvider.checkOpenSearchReadiness).toHaveBeenCalledTimes(1); + expect(searchProvider.getOpenSearchFieldCapabilities).toHaveBeenCalledWith(contract.getRequiredFieldNames()); expect(searchProvider.searchOpenSearch).toHaveBeenCalledTimes(6); expect(searchProvider.closeSearchProviderClients).toHaveBeenCalledTimes(1); expect(JSON.parse(output.write.mock.calls[0][0])).toStrictEqual({ event: 'opensearch.contract', outcome: 'passed', checks: [ + 'field-capabilities', 'source-projection', 'cve-id-keyword-and-total', 'keyword', @@ -128,6 +214,7 @@ describe('OpenSearch contract check', () => { test('closes the provider and fails when a contract response is invalid', async () => { const searchProvider = { checkOpenSearchReadiness: jest.fn().mockResolvedValue(true), + getOpenSearchFieldCapabilities: jest.fn().mockResolvedValue(validFieldCapabilities()), searchOpenSearch: jest.fn().mockResolvedValue({ hits: {} }), @@ -150,6 +237,30 @@ describe('OpenSearch contract check', () => { expect(searchProvider.closeSearchProviderClients).toHaveBeenCalledTimes(1); }); + test('closes the provider and fails when field capabilities are incomplete', async () => { + const searchProvider = { + checkOpenSearchReadiness: jest.fn().mockResolvedValue(true), + getOpenSearchFieldCapabilities: jest.fn().mockResolvedValue({ fields: {} }), + searchOpenSearch: jest.fn(), + closeSearchProviderClients: jest.fn().mockResolvedValue(undefined) + }; + + await expect( + contract.run({ + environment: { + OpenSearchDomainEndpoint: 'https://search.example.com', + OpenSearchCveIndex: 'cves' + }, + output: { + write: jest.fn() + }, + searchProvider + }) + ).rejects.toThrow('required field capability was not satisfied'); + expect(searchProvider.searchOpenSearch).not.toHaveBeenCalled(); + expect(searchProvider.closeSearchProviderClients).toHaveBeenCalledTimes(1); + }); + test('sanitizes reported failures', () => { const failure = contract.getSanitizedFailure({ code: 'OPEN_SEARCH_RESPONSE_INVALID', diff --git a/test/delivery-policy.test.js b/test/delivery-policy.test.js new file mode 100644 index 0000000..7b4d88c --- /dev/null +++ b/test/delivery-policy.test.js @@ -0,0 +1,84 @@ +const fs = require('fs'); +const path = require('path'); +const yaml = require('js-yaml'); + +const repositoryRoot = path.resolve(__dirname, '..'); + +function readRepositoryFile(file) { + return fs.readFileSync(path.join(repositoryRoot, file), 'utf8'); +} + +function readWorkflow(file) { + return yaml.load(readRepositoryFile(file)); +} + +describe('delivery security policy', () => { + test('exposes OpenSearch secrets only to the trusted contract execution step', () => { + const workflow = readWorkflow('.github/workflows/opensearch-contract.yml'); + const job = workflow.jobs.contract; + const installStep = job.steps.find(({ name }) => name === 'Install production dependencies'); + const contractStep = job.steps.find(({ name }) => name === 'Run read-only contract checks'); + + expect(job.if).toBe("github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main'"); + expect(job.env).toStrictEqual({ + NODE_ENV: 'production', + NODE_CONFIG_ENV: 'prod' + }); + expect(installStep.env).toBeUndefined(); + expect(contractStep.env).toStrictEqual({ + OpenSearchDomainEndpoint: '${{ secrets.OPENSEARCH_DOMAIN_ENDPOINT }}', + OpenSearchCveIndex: '${{ secrets.OPENSEARCH_CVE_INDEX }}', + OpenSearchAllowUnknownSslCerts: "${{ vars.OPENSEARCH_ALLOW_UNKNOWN_SSL_CERTS || 'false' }}" + }); + }); + + test('reports all severe image findings before enforcing the fixable vulnerability gate', () => { + const workflow = readWorkflow('.github/workflows/quality.yml'); + const steps = workflow.jobs.container.steps; + const reportIndex = steps.findIndex( + ({ name }) => name === 'Report all high and critical image vulnerabilities' + ); + const gateIndex = steps.findIndex( + ({ name }) => name === 'Enforce fixable high and critical image vulnerabilities' + ); + + expect(reportIndex).toBeGreaterThan(-1); + expect(gateIndex).toBeGreaterThan(reportIndex); + expect(steps[reportIndex]).toMatchObject({ + 'continue-on-error': true, + with: { + 'exit-code': 0, + 'ignore-unfixed': false, + severity: 'CRITICAL,HIGH', + trivyignores: '/dev/null' + } + }); + expect(steps[gateIndex].with).toMatchObject({ + 'exit-code': 1, + 'ignore-unfixed': true, + trivyignores: '.trivyignore' + }); + }); + + test('limits the Trivy exception to the owned transitive dependency and review window', () => { + const ignorePolicy = readRepositoryFile('.trivyignore'); + const advisoryIds = ignorePolicy + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')); + const metadata = ignorePolicy.match( + /package=adm-zip owner=cve-core fixed=0\.6\.0 review-by=(\d{4}-\d{2}-\d{2})/ + ); + const lockfile = JSON.parse(readRepositoryFile('package-lock.json')); + const rootPackage = lockfile.packages['']; + const cveCorePackage = lockfile.packages['node_modules/cve-core']; + const admZipPackage = lockfile.packages['node_modules/adm-zip']; + + expect(advisoryIds).toStrictEqual(['CVE-2026-39244']); + expect(metadata).not.toBeNull(); + expect(Date.parse(`${metadata[1]}T23:59:59Z`)).toBeGreaterThan(Date.now()); + expect(rootPackage.dependencies?.['adm-zip']).toBeUndefined(); + expect(cveCorePackage.dependencies?.['adm-zip']).toBeDefined(); + expect(admZipPackage.version.startsWith('0.5.')).toBe(true); + }); +}); diff --git a/utils/searchProvider.js b/utils/searchProvider.js index eb4393b..8a92326 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -205,6 +205,39 @@ async function searchOpenSearch(body, { signal, operation = 'search', onStart, r return searchResponseValidator.validateOpenSearchResponse(response.body, { operation }); } +async function getOpenSearchFieldCapabilities(fields, { signal } = {}) { + searchDeadline.throwIfAborted(signal); + if (!Array.isArray(fields) || fields.length === 0 || fields.some((field) => typeof field !== 'string' || !field)) { + throw new SearchProviderContractError(); + } + + const reader = getSearchReader(); + if (typeof reader._client?.fieldCaps !== 'function') { + throw new SearchProviderContractError(); + } + + const response = await executeOpenSearchRequest( + () => + reader._client.fieldCaps( + { + index: reader._cveIndex, + fields, + include_unmapped: true + }, + { + maxRetries: 0, + requestTimeout: getOpenSearchRequestTimeoutMs() + } + ), + { + signal, + operation: 'field-capabilities' + } + ); + + return response.body; +} + async function checkOpenSearchReadiness({ signal } = {}) { searchDeadline.throwIfAborted(signal); const reader = getSearchReader(); @@ -306,6 +339,7 @@ module.exports = { checkOpenSearchReadiness, closeSearchProviderClients, getBasicSearchManager, + getOpenSearchFieldCapabilities, getOpenSearchRequestState, getOpenSearchReadinessTimeoutMs, getInFlightMaxWaitersPerOperation, diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index ed13c55..84f4cc0 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -11,6 +11,7 @@ const mockBasicSearchManager = jest.fn(function BasicSearchManager(spec) { }; }); const mockClientSearch = jest.fn(); +const mockClientFieldCaps = jest.fn(); const mockClientClose = jest.fn(); const mockSearchReader = jest.fn(function SearchReader(endpoint, index, options) { this._openSearchDomainEndpoint = endpoint; @@ -18,6 +19,7 @@ const mockSearchReader = jest.fn(function SearchReader(endpoint, index, options) this._options = options; this._client = { close: mockClientClose, + fieldCaps: mockClientFieldCaps, search: mockClientSearch, transport: { maxRetries: 3, @@ -90,6 +92,7 @@ describe('searchProvider', () => { mockBasicSearchManager.mockClear(); mockSearchReader.mockClear(); mockClientSearch.mockReset(); + mockClientFieldCaps.mockReset(); mockClientClose.mockReset(); mockClientClose.mockResolvedValue(); mockGetDefaultSearchProviderSpec.mockReset(); @@ -213,6 +216,53 @@ describe('searchProvider', () => { expect(mockSearchReader).toHaveBeenCalledTimes(2); }); + test('requests bounded field capabilities through the configured index', async () => { + const body = { + fields: { + 'cveMetadata.cveId.keyword': { + keyword: { + searchable: true, + type: 'keyword' + } + } + } + }; + mockClientFieldCaps.mockResolvedValue({ body }); + + await expect(searchProvider.getOpenSearchFieldCapabilities(['cveMetadata.cveId.keyword'])).resolves.toBe(body); + + expect(mockClientFieldCaps).toHaveBeenCalledWith( + { + index: 'test-index', + fields: ['cveMetadata.cveId.keyword'], + include_unmapped: true + }, + { + maxRetries: 0, + requestTimeout: 20000 + } + ); + expect(searchProvider.getOpenSearchRequestState()).toStrictEqual({ + activeCount: 0, + activeByOperation: {} + }); + }); + + test('rejects invalid field capability requests and incompatible clients', async () => { + await expect(searchProvider.getOpenSearchFieldCapabilities([])).rejects.toMatchObject({ + code: 'SEARCH_PROVIDER_CONTRACT' + }); + + const reader = searchProvider.getSearchReader(); + reader._client.fieldCaps = undefined; + + await expect( + searchProvider.getOpenSearchFieldCapabilities(['cveMetadata.cveId.keyword']) + ).rejects.toMatchObject({ + code: 'SEARCH_PROVIDER_CONTRACT' + }); + }); + test('attempts to close every initialized OpenSearch client before reporting a close failure', async () => { const manager = searchProvider.getBasicSearchManager(); searchProvider.getSearchReader(); From 269f79cdbf856e76db40b5283174aab189426932 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 10:42:09 -0400 Subject: [PATCH 19/33] Fail closed on incomplete OpenSearch responses --- README.md | 4 +- controllers/searchController.js | 33 ++++++-- controllers/searchController.test.js | 91 +++++++++++++++++++++ utils/searchPostFilterCache.js | 5 ++ utils/searchPostFilterCache.test.js | 12 +++ utils/searchProvider.test.js | 114 +++++++++++++-------------- utils/searchResponseValidator.js | 23 +++--- 7 files changed, 203 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 146efb0..84d0dd4 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ The Search API will be served at `localhost:3000`. `GET /api/health-check` is a process liveness check. It returns HTTP 200 when the Node.js process is running and deliberately does not contact OpenSearch. Use it where a dependency outage should not restart otherwise healthy API tasks. -`GET /api/ready-check` sends a minimal zero-result search to the configured CVE index. This verifies connectivity, index availability, and the same search permission used by the API without retrieving CVE records or counting all matches. It returns HTTP 200 only when search requests can still be served after the dependency check completes. Configuration errors, connection failures, timeouts, incompatible `cve-core` SearchReader contracts, a missing index, and an in-progress process shutdown return a sanitized HTTP 503 `SEARCH_NOT_READY` response with `Retry-After: 5`. The OpenSearch request is limited to five seconds by default and can be changed with `SearchReadinessTimeoutMs`. If shutdown is already active, readiness fails immediately without contacting OpenSearch. If shutdown begins while the dependency check is pending, the completed check is not reported as ready. +`GET /api/ready-check` sends a minimal zero-result search to the configured CVE index. This verifies connectivity, index availability, and the same search permission used by the API without retrieving CVE records or counting all matches. It returns HTTP 200 only when search requests can still be served after the dependency check completes and OpenSearch explicitly reports that the request did not time out and no shards failed. Configuration errors, connection failures, timeouts, incomplete response metadata, failed shards, incompatible `cve-core` SearchReader contracts, a missing index, and an in-progress process shutdown return a sanitized HTTP 503 `SEARCH_NOT_READY` response with `Retry-After: 5`. The OpenSearch request is limited to five seconds by default and can be changed with `SearchReadinessTimeoutMs`. If shutdown is already active, readiness fails immediately without contacting OpenSearch. If shutdown begins while the dependency check is pending, the completed check is not reported as ready. An AWS load balancer can use `/api/ready-check` to stop routing search traffic to an unready task while the container orchestrator continues using `/api/health-check` for process liveness. Coordinate that probe selection with the infrastructure team before changing deployed health-check behavior. @@ -369,7 +369,7 @@ Searches that use `cpeName`, `virtualMatchString`, or `virtualMatchString` with The post-filter cache stores only the filtered CVE ID list and candidate count for that expensive candidate query. It does not cache full CVE records. On a cache hit, the API skips the 10,000-candidate OpenSearch query and semantic post-filtering, slices the cached CVE IDs for the requested page, and fetches that page of full CVE records by ID. -Candidate retrieval asks OpenSearch for exact-version or wildcard-version criteria and leaves range applicability decisions to the semantic post-filter. The full-record page fetch uses an exact CVE ID set without total-hit counting or OpenSearch sorting; the API restores the requested CVE ID order locally. Standard searches retain exact total counting and deterministic sorting. +Candidate retrieval asks OpenSearch for exact-version or wildcard-version criteria and leaves range applicability decisions to the semantic post-filter. The full-record page fetch uses an exact CVE ID set without total-hit counting or OpenSearch sorting; the API requires one unique full record for every requested CVE ID and restores the requested CVE ID order locally. Missing, duplicate, malformed, or unexpected page records fail closed with a sanitized HTTP 502 response and evict the associated semantic candidate cache entry so the next request must repeat candidate discovery. Standard searches retain exact total counting and deterministic sorting. The cache is intentionally small and short-lived: diff --git a/controllers/searchController.js b/controllers/searchController.js index 64152cc..8853867 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -8,6 +8,7 @@ const searchInFlightRegistry = require('../utils/searchInFlightRegistry'); const searchPostFilterWorkQueue = require('../utils/searchPostFilterWorkQueue'); const searchProvider = require('../utils/searchProvider'); const searchQueryBuilder = require('../utils/searchQueryBuilder'); +const { OPEN_SEARCH_RESPONSE_FAILURE_TYPES, OpenSearchResponseError } = require('../utils/searchResponseValidator'); const constants = require('../utils/constants'); const error = new errors.errors(); const openSearchUnavailableErrorNames = new Set(['ConnectionError', 'NoLivingConnectionsError']); @@ -135,10 +136,24 @@ function getHitCveIds(hits) { return hits.map(getHitCveId).filter(Boolean); } -function orderHitsByCveIds(hits, cveIds) { - const hitsByCveId = new Map(hits.map((hit) => [getHitCveId(hit), hit])); +function validateAndOrderPageHits(hits, cveIds) { + const requestedCveIds = new Set(cveIds); + const hitsByCveId = new Map(); - return cveIds.map((cveId) => hitsByCveId.get(cveId)).filter(Boolean); + if (requestedCveIds.size !== cveIds.length || hits.length !== cveIds.length) { + throw new OpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.INCOMPLETE_PAGE_FETCH); + } + + for (const hit of hits) { + const hitCveId = hit?._source?.cveMetadata?.cveId; + if (typeof hitCveId !== 'string' || !requestedCveIds.has(hitCveId) || hitsByCveId.has(hitCveId)) { + throw new OpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.INCOMPLETE_PAGE_FETCH); + } + + hitsByCveId.set(hitCveId, hit); + } + + return cveIds.map((cveId) => hitsByCveId.get(cveId)); } function combineTook(...values) { @@ -193,7 +208,7 @@ async function fetchFullHitsByCveIds(cveIds, metrics, signal) { metrics.pageFetchElapsedMs = searchPerformance.elapsedMs(pageFetchStart); metrics.pageFetchTookMs = pageData?.took; - return orderHitsByCveIds(pageData?.hits?.hits ?? [], cveIds); + return validateAndOrderPageHits(pageData.hits.hits, cveIds); }, { signal, @@ -303,7 +318,15 @@ async function executeOptimizedPostFilterSearch( metrics.filteredCount = candidateResult.filteredCount ?? candidateResult.filteredCveIds.length; metrics.returnedCount = pageCveIds.length; - const fullPageHits = await fetchFullHitsByCveIds(pageCveIds, metrics, signal); + let fullPageHits; + try { + fullPageHits = await fetchFullHitsByCveIds(pageCveIds, metrics, signal); + } catch (err) { + if (err?.failureType === OPEN_SEARCH_RESPONSE_FAILURE_TYPES.INCOMPLETE_PAGE_FETCH) { + searchPostFilterCache.remove(cacheKey); + } + throw err; + } const response = { status: 'ok', data: { diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index cd3be86..cb1233f 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -176,6 +176,17 @@ function cpeHit(id, cpeMatch) { }; } +function fullCveHit(id) { + return { + _id: id, + _source: { + cveMetadata: { + cveId: id + } + } + }; +} + describe('Search controller SEARCH handler', () => { let res, next; @@ -920,6 +931,86 @@ describe('Search controller SEARCH handler', () => { }); }); + test.each([ + ['missing', ['CVE-2024-0001', 'CVE-2024-0002'], [fullCveHit('CVE-2024-0001')]], + ['duplicate', ['CVE-2024-0001', 'CVE-2024-0002'], [fullCveHit('CVE-2024-0001'), fullCveHit('CVE-2024-0001')]], + ['malformed', ['CVE-2024-0001'], [{ _id: 'CVE-2024-0001', _source: { cveMetadata: {} } }]], + ['unexpected', ['CVE-2024-0001'], [fullCveHit('CVE-2024-9999')]] + ])('SEARCH fails closed for a %s full-record page response', async (_caseName, candidateIds, pageHits) => { + const candidates = candidateIds.map((id) => + cpeHit(id, { + criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' + }) + ); + mockSearchRaw.mockResolvedValue(rawResponse(candidates.length, candidates)); + mockApiSearch.mockResolvedValue(advancedResponse(pageHits.length, pageHits)); + + await controller.SEARCH( + { + requestId: `${_caseName}-page-fetch-request`, + searchParams: { + virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', + resultsPerPage: candidateIds.length, + pageNumber: 1 + } + }, + res, + next + ); + + expect(res.status).toHaveBeenCalledWith(502); + expect(res.json).toHaveBeenCalledWith({ + error: 'UPSTREAM_SEARCH_ERROR', + message: 'Search backend returned an error. Please try again later.' + }); + expect(searchPostFilterCache.getState()).toStrictEqual({ + entryCount: 0, + cachedCveIdCount: 0 + }); + expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ + event: 'search.performance', + requestId: `${_caseName}-page-fetch-request`, + outcome: 'upstream_invalid_response', + statusCode: 502, + upstreamFailureType: 'incomplete_page_fetch' + }); + expect(next).not.toHaveBeenCalled(); + }); + + test('SEARCH re-runs candidate discovery after an incomplete page fetch evicts the cache entry', async () => { + const candidate = cpeHit('CVE-2024-0001', { + criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' + }); + const fullHit = fullCveHit('CVE-2024-0001'); + const req = { + searchParams: { + virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', + resultsPerPage: 1, + pageNumber: 1 + } + }; + mockSearchRaw.mockResolvedValue(rawResponse(1, [candidate])); + mockApiSearch + .mockResolvedValueOnce(advancedResponse(0, [])) + .mockResolvedValueOnce(advancedResponse(1, [fullHit])); + + await controller.SEARCH(req, res, next); + const secondRes = createResponse(); + await controller.SEARCH(req, secondRes, next); + + expect(mockSearchRaw).toHaveBeenCalledTimes(2); + expect(mockApiSearch).toHaveBeenCalledTimes(2); + expect(res.status).toHaveBeenCalledWith(502); + expect(secondRes.status).toHaveBeenCalledWith(200); + expect(secondRes.json).toHaveBeenCalledWith( + expect.objectContaining({ + page: 1, + pageSize: 1, + status: 'ok' + }) + ); + }); + test('SEARCH coalesces concurrent identical cold semantic post-filter searches', async () => { const candidateSearch = deferred(); const applicable = cpeHit('CVE-2024-0001', { diff --git a/utils/searchPostFilterCache.js b/utils/searchPostFilterCache.js index ee90e58..19e6c55 100644 --- a/utils/searchPostFilterCache.js +++ b/utils/searchPostFilterCache.js @@ -77,6 +77,10 @@ function set(cacheKey, value, now = Date.now()) { } } +function remove(cacheKey) { + return entries.delete(cacheKey); +} + function getState(now = Date.now()) { for (const [key, entry] of entries) { if (entry.expiresAt <= now) entries.delete(key); @@ -101,5 +105,6 @@ module.exports = { get, getState, normalizeForKey, + remove, set }; diff --git a/utils/searchPostFilterCache.test.js b/utils/searchPostFilterCache.test.js index 7d2f41d..bc5173b 100644 --- a/utils/searchPostFilterCache.test.js +++ b/utils/searchPostFilterCache.test.js @@ -144,6 +144,18 @@ describe('searchPostFilterCache', () => { expect(searchPostFilterCache.get(key, 61001)).toBeNull(); }); + test('remove evicts only the selected cache entry', () => { + searchPostFilterCache.set('first', { filteredCveIds: ['CVE-2024-0001'] }, 1000); + searchPostFilterCache.set('second', { filteredCveIds: ['CVE-2024-0002'] }, 1000); + + expect(searchPostFilterCache.remove('first')).toBe(true); + expect(searchPostFilterCache.get('first', 1001)).toBeNull(); + expect(searchPostFilterCache.get('second', 1001)).toStrictEqual({ + filteredCveIds: ['CVE-2024-0002'] + }); + expect(searchPostFilterCache.remove('missing')).toBe(false); + }); + test('reports entry and cached CVE ID counts while pruning expired entries', () => { searchPostFilterCache.set( 'first', diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index 84f4cc0..78616c5 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -86,6 +86,22 @@ function cpeCandidateHit(cpeMatch = {}) { }; } +function completeSearchResponse({ hits = [], total } = {}) { + const body = { + timed_out: false, + _shards: { + failed: 0 + }, + hits: { + hits + } + }; + + if (total !== undefined) body.hits.total = total; + + return body; +} + describe('searchProvider', () => { beforeEach(() => { searchProvider.resetSearchProviderForTests(); @@ -138,16 +154,14 @@ describe('searchProvider', () => { }); test('reuses one search reader per process for OpenSearch searches', async () => { - mockClientSearch.mockResolvedValue({ - body: { - hits: { - total: { - value: 0 - }, - hits: [] - } + const responseBody = completeSearchResponse({ + total: { + value: 0 } }); + mockClientSearch.mockResolvedValue({ + body: responseBody + }); const first = searchProvider.getSearchReader(); const second = searchProvider.getSearchReader(); @@ -180,14 +194,7 @@ describe('searchProvider', () => { ); expect(first._client.transport.maxRetries).toBe(0); expect(first._client.transport.requestTimeout).toBe(20000); - expect(result).toStrictEqual({ - hits: { - total: { - value: 0 - }, - hits: [] - } - }); + expect(result).toBe(responseBody); expect(onStart).toHaveBeenCalledWith({ activeCount: 1, activeByOperation: { @@ -283,11 +290,7 @@ describe('searchProvider', () => { test('caps an OpenSearch request timeout to the remaining request budget', async () => { mockClientSearch.mockResolvedValue({ - body: { - hits: { - hits: [] - } - } + body: completeSearchResponse() }); await searchProvider.searchOpenSearch( @@ -309,11 +312,7 @@ describe('searchProvider', () => { test('does not extend an OpenSearch request beyond the configured timeout', async () => { mockClientSearch.mockResolvedValue({ - body: { - hits: { - hits: [] - } - } + body: completeSearchResponse() }); await searchProvider.searchOpenSearch( @@ -347,6 +346,7 @@ describe('searchProvider', () => { [ 'partial-shard', { + timed_out: false, _shards: { failed: 1 }, @@ -359,6 +359,10 @@ describe('searchProvider', () => { [ 'malformed', { + timed_out: false, + _shards: { + failed: 0 + }, hits: {} }, 'malformed_response' @@ -372,6 +376,22 @@ describe('searchProvider', () => { }); }); + test.each([ + ['missing timed_out', { _shards: { failed: 0 }, hits: { hits: [] } }], + ['non-boolean timed_out', { timed_out: 'false', _shards: { failed: 0 }, hits: { hits: [] } }], + ['missing _shards', { timed_out: false, hits: { hits: [] } }], + ['non-object _shards', { timed_out: false, _shards: 0, hits: { hits: [] } }], + ['missing failed shard count', { timed_out: false, _shards: {}, hits: { hits: [] } }], + ['non-integer failed shard count', { timed_out: false, _shards: { failed: '0' }, hits: { hits: [] } }] + ])('rejects an OpenSearch response with %s', async (_caseName, body) => { + mockClientSearch.mockResolvedValue({ body }); + + await expect(searchProvider.searchOpenSearch({ query: { match_all: {} } })).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_response' + }); + }); + test.each([ ['missing', undefined], ['non-object', 1], @@ -381,12 +401,7 @@ describe('searchProvider', () => { ['string', { value: '1' }] ])('rejects a standard OpenSearch response with a %s total', async (_caseName, total) => { mockClientSearch.mockResolvedValue({ - body: { - hits: { - total, - hits: [] - } - } + body: completeSearchResponse({ total }) }); await expect( @@ -407,11 +422,7 @@ describe('searchProvider', () => { }); test('accepts schema-shaped CPE candidate hits for candidate operations', async () => { - const body = { - hits: { - hits: [cpeCandidateHit()] - } - }; + const body = completeSearchResponse({ hits: [cpeCandidateHit()] }); mockClientSearch.mockResolvedValue({ body }); await expect( @@ -429,11 +440,7 @@ describe('searchProvider', () => { }); test('accepts an empty CPE candidate result', async () => { - const body = { - hits: { - hits: [] - } - }; + const body = completeSearchResponse(); mockClientSearch.mockResolvedValue({ body }); await expect( @@ -453,11 +460,7 @@ describe('searchProvider', () => { test('does not synchronously traverse CPE candidate elements during response validation', async () => { const malformedHit = cpeCandidateHit({ criteria: undefined }); mockClientSearch.mockResolvedValue({ - body: { - hits: { - hits: [malformedHit] - } - } + body: completeSearchResponse({ hits: [malformedHit] }) }); await expect( @@ -471,20 +474,12 @@ describe('searchProvider', () => { operation: 'cpe-candidate' } ) - ).resolves.toStrictEqual({ - hits: { - hits: [malformedHit] - } - }); + ).resolves.toStrictEqual(completeSearchResponse({ hits: [malformedHit] })); }); test('checks OpenSearch readiness through the configured CVE index', async () => { mockClientSearch.mockResolvedValue({ - body: { - hits: { - hits: [] - } - } + body: completeSearchResponse() }); await expect(searchProvider.checkOpenSearchReadiness()).resolves.toBe(true); @@ -516,6 +511,7 @@ describe('searchProvider', () => { test('fails readiness when OpenSearch reports a partial response', async () => { mockClientSearch.mockResolvedValue({ body: { + timed_out: false, _shards: { failed: 1 }, @@ -656,8 +652,8 @@ describe('searchProvider', () => { } }); - candidate.resolve({ body: { hits: { hits: [] } } }); - pageFetch.resolve({ body: { hits: { hits: [] } } }); + candidate.resolve({ body: completeSearchResponse() }); + pageFetch.resolve({ body: completeSearchResponse() }); await Promise.all([candidateSearch, pageSearch]); expect(searchProvider.getOpenSearchRequestState()).toStrictEqual({ diff --git a/utils/searchResponseValidator.js b/utils/searchResponseValidator.js index 47a4acc..8e6668b 100644 --- a/utils/searchResponseValidator.js +++ b/utils/searchResponseValidator.js @@ -1,4 +1,5 @@ const OPEN_SEARCH_RESPONSE_FAILURE_TYPES = Object.freeze({ + INCOMPLETE_PAGE_FETCH: 'incomplete_page_fetch', MALFORMED_CANDIDATE: 'malformed_candidate', MALFORMED_RESPONSE: 'malformed_response', PARTIAL_SHARDS: 'partial_shards', @@ -36,20 +37,16 @@ function validateOpenSearchResponse(body, { operation } = {}) { throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.TIMEOUT); } - if (body._shards !== undefined) { - if (!isObject(body._shards)) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); - } + if (body.timed_out !== false || !isObject(body._shards)) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } - if (body._shards.failed !== undefined) { - const failedShards = Number(body._shards.failed); - if (!Number.isInteger(failedShards) || failedShards < 0) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); - } - if (failedShards > 0) { - throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.PARTIAL_SHARDS); - } - } + const failedShards = body._shards.failed; + if (!Number.isInteger(failedShards) || failedShards < 0) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_RESPONSE); + } + if (failedShards > 0) { + throwOpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.PARTIAL_SHARDS); } if (!isObject(body.hits) || !Array.isArray(body.hits.hits)) { From 65fdf6d111ba0cba8a867712351bc920e2ca4b5e Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 10:52:04 -0400 Subject: [PATCH 20/33] Fail closed on invalid search construction --- .github/dependabot.yml | 3 + README.md | 2 +- controllers/searchController.test.js | 22 ++++++ test/delivery-policy.test.js | 11 +++ utils/searchQueryBuilder.js | 79 ++++++++++++++++----- utils/searchQueryBuilder.test.js | 101 +++++++++++++++++++++++++++ 6 files changed, 201 insertions(+), 17 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 083cace..b1b27aa 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -2,18 +2,21 @@ version: 2 updates: - package-ecosystem: npm directory: / + target-branch: dev schedule: interval: weekly open-pull-requests-limit: 5 - package-ecosystem: docker directory: / + target-branch: dev schedule: interval: weekly open-pull-requests-limit: 5 - package-ecosystem: github-actions directory: / + target-branch: dev schedule: interval: weekly open-pull-requests-limit: 5 diff --git a/README.md b/README.md index 84d0dd4..5ddf165 100644 --- a/README.md +++ b/README.md @@ -339,7 +339,7 @@ The validation tiers have separate purposes: - The optional OpenSearch contract check verifies that the installed client can communicate with a non-empty externally managed index whose representative searchable fields satisfy the application contract. - Load and performance testing remain in the separate automation performance-testing repository. -Dependabot checks npm packages, Docker base-image digests, and GitHub Actions weekly. Both Dockerfiles pin the Node build image and Distroless runtime image to immutable multi-architecture digests; Dependabot updates those references through reviewable pull requests. +Dependabot checks npm packages, Docker base-image digests, and GitHub Actions weekly. Version-update pull requests target the `dev` integration branch; GitHub security updates continue to follow the repository default-branch behavior. Both Dockerfiles pin the Node build image and Distroless runtime image to immutable multi-architecture digests; Dependabot updates those references through reviewable pull requests. ### Operational Logging diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index cb1233f..b5c4ec7 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -977,6 +977,28 @@ describe('Search controller SEARCH handler', () => { expect(next).not.toHaveBeenCalled(); }); + test.each([ + ['cpeName', { cpeName: 'not-a-cpe-name' }], + ['virtualMatchString', { virtualMatchString: 'not-a-cpe-match-string' }], + ['virtualMatchString range', { versionStart: '1.0', versionStartType: 'including' }] + ])( + 'SEARCH does not contact OpenSearch when invalid %s input bypasses middleware', + async (_caseName, searchParams) => { + await controller.SEARCH( + { + searchParams + }, + res, + next + ); + + expect(mockSearchOpenSearch).not.toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect(res.json).not.toHaveBeenCalled(); + expect(next).toHaveBeenCalledWith(expect.any(TypeError)); + } + ); + test('SEARCH re-runs candidate discovery after an incomplete page fetch evicts the cache entry', async () => { const candidate = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' diff --git a/test/delivery-policy.test.js b/test/delivery-policy.test.js index 7b4d88c..65af699 100644 --- a/test/delivery-policy.test.js +++ b/test/delivery-policy.test.js @@ -13,6 +13,17 @@ function readWorkflow(file) { } describe('delivery security policy', () => { + test('routes Dependabot version updates through the dev integration branch', () => { + const dependabot = readWorkflow('.github/dependabot.yml'); + + expect(dependabot.updates.map((update) => update['package-ecosystem'])).toStrictEqual([ + 'npm', + 'docker', + 'github-actions' + ]); + expect(dependabot.updates.every((update) => update['target-branch'] === 'dev')).toBe(true); + }); + test('exposes OpenSearch secrets only to the trusted contract execution step', () => { const workflow = readWorkflow('.github/workflows/opensearch-contract.yml'); const job = workflow.jobs.contract; diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index 4f86dd9..2a1f62b 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -23,6 +23,8 @@ const cveIdAscendingSort = { order: 'asc' } }; +const versionRangeParamNames = ['versionStart', 'versionStartType', 'versionEnd', 'versionEndType']; +const versionRangeTypes = new Set(['including', 'excluding']); function buildDeterministicSort(queryStringObjects) { const sort = []; @@ -156,35 +158,70 @@ function getFrom(params, resultsPerPage) { } function getCpeNamePostFilter(params) { - if (!params?.cpeName) return null; + if (!hasParam(params, 'cpeName')) return null; const parsed = cpe.parseCpeName(params.cpeName); - return parsed.valid ? parsed : null; + if (!parsed.valid) throwInvalidQueryValue('cpeName'); + + return parsed; } function hasVirtualMatchStringRangeParams(params) { - return Boolean(params?.versionStart || params?.versionStartType || params?.versionEnd || params?.versionEndType); + return versionRangeParamNames.some((param) => hasParam(params, param)); } function getVirtualMatchStringPostFilter(params) { - if (!params?.virtualMatchString || hasVirtualMatchStringRangeParams(params)) return null; + if (!hasParam(params, 'virtualMatchString') || hasVirtualMatchStringRangeParams(params)) return null; const parsed = cpe.parseVirtualMatchString(params.virtualMatchString); - return parsed.valid ? parsed : null; + if (!parsed.valid) throwInvalidQueryValue('virtualMatchString'); + + return parsed; } function getVirtualMatchStringRangePostFilter(params) { - if (!params?.virtualMatchString || !hasVirtualMatchStringRangeParams(params)) return null; + if (!hasVirtualMatchStringRangeParams(params)) return null; + if (!hasParam(params, 'virtualMatchString')) throwInvalidVirtualMatchStringRange(); const matchString = cpe.parseVirtualMatchString(params.virtualMatchString); - if (!matchString.valid || matchString.hasVersionComponent) return null; + if (!matchString.valid || matchString.hasVersionComponent) throwInvalidVirtualMatchStringRange(); + + const versionStart = normalizeVersionRangeBound(params, 'versionStart', 'versionStartType'); + const versionEnd = normalizeVersionRangeBound(params, 'versionEnd', 'versionEndType'); + if (versionStart.value !== undefined && versionEnd.value !== undefined) { + const comparison = cpe.compareCpeVersions(versionStart.value, versionEnd.value); + const isEmptyEqualRange = + comparison === 0 && (versionStart.type !== 'including' || versionEnd.type !== 'including'); + if (comparison > 0 || isEmptyEqualRange) throwInvalidVirtualMatchStringRange(); + } return { matchString, - versionStart: params.versionStart, - versionStartType: params.versionStartType?.toLowerCase(), - versionEnd: params.versionEnd, - versionEndType: params.versionEndType?.toLowerCase() + versionStart: versionStart.value, + versionStartType: versionStart.type, + versionEnd: versionEnd.value, + versionEndType: versionEnd.type + }; +} + +function normalizeVersionRangeBound(params, valueParam, typeParam) { + const hasValue = hasParam(params, valueParam); + const hasType = hasParam(params, typeParam); + if (hasValue !== hasType) throwInvalidVirtualMatchStringRange(); + if (!hasValue) return {}; + + const value = params[valueParam]; + const type = params[typeParam]; + if (typeof value !== 'string' || !value.trim() || typeof type !== 'string') { + throwInvalidVirtualMatchStringRange(); + } + + const normalizedType = type.toLowerCase(); + if (!versionRangeTypes.has(normalizedType)) throwInvalidVirtualMatchStringRange(); + + return { + type: normalizedType, + value }; } @@ -248,6 +285,18 @@ function isTrueValue(value) { return value === true || String(value).toLowerCase() === 'true'; } +function hasParam(params, param) { + return Object.prototype.hasOwnProperty.call(params ?? {}, param) && params[param] !== undefined; +} + +function throwInvalidQueryValue(param) { + throw new TypeError(`Invalid ${param} value reached query construction.`); +} + +function throwInvalidVirtualMatchStringRange() { + throw new TypeError('Invalid virtualMatchString version range reached query construction.'); +} + function addWildCard(searchString) { const terms = String(searchString ?? '') .trim() @@ -403,7 +452,7 @@ function buildNoRejectedFilter(fieldSpec) { function buildCpeNameFilter(cpeName) { const parsedCpe = cpe.parseCpeName(cpeName); - if (!parsedCpe.valid) return null; + if (!parsedCpe.valid) throwInvalidQueryValue('cpeName'); const fields = constants.CPE_MATCH_FIELDS; const criteriaKeywordField = `${fields.criteria}.keyword`; @@ -422,7 +471,7 @@ function buildCpeNameFilter(cpeName) { function buildVirtualMatchStringFilter(virtualMatchString) { const parsedCpe = cpe.parseVirtualMatchString(virtualMatchString); - if (!parsedCpe.valid) return null; + if (!parsedCpe.valid) throwInvalidQueryValue('virtualMatchString'); const criteriaKeywordField = `${constants.CPE_MATCH_FIELDS.criteria}.keyword`; if (cpe.hasConcreteLeadingComponents(parsedCpe, 3) && !cpe.hasConcreteComponentsAfter(parsedCpe, 2)) { @@ -482,9 +531,7 @@ function buildCpeCriteriaRegexpFilter(field, value) { function buildCvssRegexpFilter(inputVectorString, param, fieldSpec) { const parsed = cvss.parseCvssMetrics(inputVectorString, param); - if (!parsed.valid) { - throw new TypeError(`Invalid ${param} value reached query construction.`); - } + if (!parsed.valid) throwInvalidQueryValue(param); let fields = Array.isArray(fieldSpec) ? fieldSpec : [fieldSpec]; if (parsed.version) { diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index ff7cfa1..71e580b 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -228,6 +228,98 @@ describe('searchQueryBuilder buildSearchRequest()', () => { expect(output.queryStringObjects).toStrictEqual([]); }); + test.each([ + ['cpeName', { cpeName: 'not-a-cpe-name' }, 'Invalid cpeName value reached query construction.'], + [ + 'virtualMatchString', + { virtualMatchString: 'not-a-cpe-match-string' }, + 'Invalid virtualMatchString value reached query construction.' + ] + ])('buildSearchRequest fails closed when invalid %s bypasses middleware validation', (_param, params, message) => { + expect(() => queryBuilder.buildSearchRequest(params)).toThrow(message); + }); + + test.each([ + ['missing virtualMatchString', { versionStart: '1.0', versionStartType: 'including' }], + [ + 'invalid virtualMatchString', + { + virtualMatchString: 'not-a-cpe-match-string', + versionStart: '1.0', + versionStartType: 'including' + } + ], + [ + 'version-bearing virtualMatchString', + { + virtualMatchString: 'cpe:2.3:a:vendor:product:*', + versionStart: '1.0', + versionStartType: 'including' + } + ], + ['unpaired bound', { virtualMatchString: 'cpe:2.3:a:vendor:product', versionStart: '1.0' }], + [ + 'empty bound', + { + virtualMatchString: 'cpe:2.3:a:vendor:product', + versionStart: '', + versionStartType: 'including' + } + ], + [ + 'invalid bound type', + { + virtualMatchString: 'cpe:2.3:a:vendor:product', + versionStart: '1.0', + versionStartType: 'inclusive' + } + ], + [ + 'reversed bounds', + { + virtualMatchString: 'cpe:2.3:a:vendor:product', + versionStart: '2.0', + versionStartType: 'including', + versionEnd: '1.0', + versionEndType: 'including' + } + ], + [ + 'empty equal bounds', + { + virtualMatchString: 'cpe:2.3:a:vendor:product', + versionStart: '1.0', + versionStartType: 'including', + versionEnd: '1.0', + versionEndType: 'excluding' + } + ] + ])('buildSearchRequest fails closed for a virtualMatchString range with %s', (_caseName, params) => { + expect(() => queryBuilder.buildSearchRequest(params)).toThrow( + 'Invalid virtualMatchString version range reached query construction.' + ); + }); + + test.each([ + ['start', { versionStart: '1.0', versionStartType: 'INCLUDING' }], + ['end', { versionEnd: '2.0', versionEndType: 'EXCLUDING' }] + ])('buildSearchRequest preserves a valid one-sided virtualMatchString %s range', (_bound, rangeParams) => { + const output = queryBuilder.buildSearchRequest({ + virtualMatchString: 'cpe:2.3:a:vendor:product', + ...rangeParams + }); + + expect(output.postFilters.virtualMatchStringRange).toMatchObject({ + matchString: expect.objectContaining({ + product: 'product' + }), + versionStart: rangeParams.versionStart, + versionStartType: rangeParams.versionStartType?.toLowerCase(), + versionEnd: rangeParams.versionEnd, + versionEndType: rangeParams.versionEndType?.toLowerCase() + }); + }); + test('buildCpeCandidateSearchBody limits source fields for semantic post-filtering', () => { const output = queryBuilder.buildSearchRequest({ keywordSearch: 'kernel', @@ -608,6 +700,15 @@ describe('searchQueryBuilder getFilterFields() function', () => { expect(JSON.stringify(output)).not.toContain('.keyword":{"gt"'); }); + test.each([ + ['cpeName', { cpeName: 'not-a-cpe-name' }], + ['virtualMatchString', { virtualMatchString: 'not-a-cpe-match-string' }] + ])('getFilterFields() fails closed when invalid %s reaches query construction', (param, params) => { + expect(() => queryBuilder.getFilterFields(params)).toThrow( + `Invalid ${param} value reached query construction.` + ); + }); + test('getFilterFields() builds combined cvssV3 severity filters across v3.0 and v3.1.', () => { const output = queryBuilder.getFilterFields({ cvssV3Severity: 'critical' From 7f9ae6c88dcef38157bd5cb258eeaf64cb1daa7f Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 11:04:15 -0400 Subject: [PATCH 21/33] Bound semantic CPE evaluation work --- README.md | 4 +- config/default.jsonc | 2 +- utils/cpe.js | 37 -------- utils/cpe.test.js | 121 ------------------------- utils/searchCandidateProcessor.js | 115 ++++++++++++++--------- utils/searchCandidateProcessor.test.js | 101 +++++++++++++++++++++ 6 files changed, 176 insertions(+), 204 deletions(-) diff --git a/README.md b/README.md index 5ddf165..063debe 100644 --- a/README.md +++ b/README.md @@ -382,7 +382,7 @@ The cache is intentionally small and short-lived: - Cold-search queue depth: up to 500 unique cold searches can wait per process behind the concurrency limit - Cold-search queue wait: up to 10 seconds before a waiting search returns HTTP 429 - Shared operation waiters: up to 500 callers can wait on one candidate or page-fetch operation per process -- Post-filter evaluation chunk: 250 candidates between cancellation checks and event-loop yields +- Post-filter evaluation chunk: 250 CPE Match Criteria entries between cancellation checks and event-loop yields - OpenSearch request timeout: 20 seconds for each operation - Overall `/search` deadline: 30 seconds - Eviction: oldest entries are removed when the cache exceeds its max size @@ -391,7 +391,7 @@ The admission queue applies only to cold CPE candidate searches. Standard `/sear When a cold search waits in the queue for more than 10 seconds, it fails closed with HTTP 429 and a `SEARCH_QUEUE_TIMEOUT` response. OpenSearch operations use one attempt and return a sanitized HTTP 504 `SEARCH_TIMEOUT` response when they exceed 20 seconds. Standard searches use the shorter of that timeout and the request's remaining overall deadline. Shared candidate and page-fetch operations retain the independent OpenSearch timeout so one caller's remaining time does not shorten work still needed by other callers. Every caller retains its own 30-second overall deadline. -Semantic filtering runs in bounded chunks and yields to the Node.js event loop between chunks. This lets disconnect and deadline cancellation signals interrupt a large candidate evaluation instead of waiting for the full set to finish synchronously. +Semantic filtering validates and evaluates each direct CPE Match Criteria entry in one pass. Once a candidate has satisfied every requested post-filter, later entries are still validated before that candidate can be returned or cached, but redundant match comparisons are skipped. Work is bounded by CPE Match Criteria entry count rather than CVE record count, and filtering yields to the Node.js event loop between chunks. This lets disconnect and deadline cancellation signals interrupt a single record containing many entries instead of waiting for the full record to finish synchronously. Healthy dev timing logs showed the impact for repeated identical `virtualMatchString` range searches: diff --git a/config/default.jsonc b/config/default.jsonc index 0c4c0b8..b04bc01 100644 --- a/config/default.jsonc +++ b/config/default.jsonc @@ -29,7 +29,7 @@ "postFilterColdSearchMaxWaitMs": "10000", // Limits callers sharing one candidate or page-fetch operation per Node.js process. "inFlightMaxWaitersPerOperation": "500", - // Allows cancellation checks between batches of semantic CPE candidate evaluations. + // Allows cancellation checks between batches of semantic CPE Match Criteria entry evaluations. "postFilterEvaluationChunkSize": "250", // Limits each request from the API process to OpenSearch. "openSearchRequestTimeoutMs": "20000", diff --git a/utils/cpe.js b/utils/cpe.js index 58e6741..dd75191 100644 --- a/utils/cpe.js +++ b/utils/cpe.js @@ -535,39 +535,6 @@ function cpeMatchMatchesVirtualMatchString(cpeMatch, virtualMatchString, context return true; } -function getCpeMatchesFromHit(hit) { - const cpeApplicability = hit?._source?.containers?.cna?.cpeApplicability; - const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; - const matches = []; - - applicabilityItems.forEach((applicability) => { - const rootNodes = Array.isArray(applicability?.nodes) ? applicability.nodes : []; - rootNodes.forEach((node) => { - if (Array.isArray(node?.cpeMatch)) { - matches.push(...node.cpeMatch); - } - }); - }); - - return matches; -} - -function hitAppliesToCpeName(hit, cpeName, context, options) { - return getCpeMatchesFromHit(hit).some((match) => cpeMatchAppliesToCpeName(match, cpeName, context, options)); -} - -function hitOverlapsVirtualMatchStringRange(hit, virtualMatchStringRange, context) { - return getCpeMatchesFromHit(hit).some((match) => - cpeMatchOverlapsVirtualMatchStringRange(match, virtualMatchStringRange, context) - ); -} - -function hitMatchesVirtualMatchString(hit, virtualMatchString, context) { - return getCpeMatchesFromHit(hit).some((match) => - cpeMatchMatchesVirtualMatchString(match, virtualMatchString, context) - ); -} - module.exports = { buildCpeCriteriaRegexp, buildCompatibleCriteriaPrefixes, @@ -578,10 +545,6 @@ module.exports = { cpeMatchMatchesVirtualMatchString, cpeMatchOverlapsVirtualMatchStringRange, escapeLuceneRegexpLiteral, - getCpeMatchesFromHit, - hitAppliesToCpeName, - hitMatchesVirtualMatchString, - hitOverlapsVirtualMatchStringRange, hasConcreteComponentsAfter, hasConcreteLeadingComponents, isValidCpeMatch, diff --git a/utils/cpe.test.js b/utils/cpe.test.js index 6068cac..b8317a2 100644 --- a/utils/cpe.test.js +++ b/utils/cpe.test.js @@ -633,125 +633,4 @@ describe('CPE helpers', () => { }) ).toBe(true); }); - - test('hitAppliesToCpeName evaluates direct entries independently of node operator and negate values', () => { - const hit = { - _source: { - containers: { - cna: { - cpeApplicability: { - nodes: [ - { - operator: 'AND', - negate: true, - cpeMatch: [ - { - criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', - vulnerable: true, - versionStartIncluding: '2.9', - versionEndIncluding: '2.10' - }, - { - criteria: 'cpe:2.3:a:other:component:*:*:*:*:*:*:*:*', - vulnerable: true - } - ] - } - ] - } - } - } - } - }; - - expect( - cpe.hitAppliesToCpeName(hit, 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*', undefined, { - requireVulnerable: true - }) - ).toBe(true); - expect(cpe.hitAppliesToCpeName(hit, 'cpe:2.3:a:vendor:product:2.11:*:*:*:*:*:*:*')).toBe(false); - }); - - test('hitOverlapsVirtualMatchStringRange evaluates direct entries independently of node metadata', () => { - const hit = { - _source: { - containers: { - cna: { - cpeApplicability: { - nodes: [ - { - operator: 'OR', - negate: true, - cpeMatch: [ - { - criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', - vulnerable: true, - versionStartIncluding: '2.9', - versionEndIncluding: '2.10' - }, - { - criteria: 'cpe:2.3:a:other:component:*:*:*:*:*:*:*:*', - vulnerable: true - } - ] - } - ] - } - } - } - } - }; - - expect( - cpe.hitOverlapsVirtualMatchStringRange(hit, { - matchString: cpe.parseVirtualMatchString('cpe:2.3:a:vendor:product'), - versionStart: '2.10', - versionStartType: 'including', - versionEnd: '2.11', - versionEndType: 'excluding' - }) - ).toBe(true); - expect( - cpe.hitOverlapsVirtualMatchStringRange(hit, { - matchString: cpe.parseVirtualMatchString('cpe:2.3:a:vendor:product'), - versionStart: '2.11', - versionStartType: 'including', - versionEnd: '2.12', - versionEndType: 'excluding' - }) - ).toBe(false); - }); - - test('getCpeMatchesFromHit ignores nested-only applicability entries', () => { - const hit = { - _source: { - containers: { - cna: { - cpeApplicability: { - nodes: [ - { - operator: 'OR', - negate: false, - nodes: [ - { - cpeMatch: [ - { - criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', - vulnerable: true - } - ] - } - ] - } - ] - } - } - } - } - }; - - expect(cpe.getCpeMatchesFromHit(hit)).toStrictEqual([]); - expect(cpe.hitAppliesToCpeName(hit, 'cpe:2.3:a:vendor:product:2.10:*:*:*:*:*:*:*')).toBe(false); - expect(cpe.hitMatchesVirtualMatchString(hit, 'cpe:2.3:a:vendor:product')).toBe(false); - }); }); diff --git a/utils/searchCandidateProcessor.js b/utils/searchCandidateProcessor.js index 5e408c1..e48a53a 100644 --- a/utils/searchCandidateProcessor.js +++ b/utils/searchCandidateProcessor.js @@ -8,55 +8,53 @@ function isObject(value) { return Boolean(value) && typeof value === 'object' && !Array.isArray(value); } -function isValidCpeCandidateHit(hit, cpeEvaluationContext) { - if (!isObject(hit) || !isObject(hit._source)) return false; +function getCandidateApplicabilityItems(hit) { + if (!isObject(hit) || !isObject(hit._source)) return null; const candidateCveId = hit._source.cveMetadata?.cveId; - if (!cveId.isValidCveId(candidateCveId)) return false; + if (!cveId.isValidCveId(candidateCveId)) return null; const cpeApplicability = hit._source.containers?.cna?.cpeApplicability; const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; - if (!applicabilityItems.length || applicabilityItems.some((item) => !isObject(item))) return false; + if (!applicabilityItems.length || applicabilityItems.some((item) => !isObject(item))) return null; - let matchCount = 0; - for (const applicability of applicabilityItems) { - if (!Array.isArray(applicability.nodes)) return false; + return applicabilityItems; +} - for (const node of applicability.nodes) { - if (!isObject(node) || !Array.isArray(node.cpeMatch)) return false; - if (!node.cpeMatch.every((match) => cpe.isValidCpeMatch(match, cpeEvaluationContext))) return false; - matchCount += node.cpeMatch.length; - } - } +function createPostFilterMatchState(postFilters) { + return { + cpeName: !postFilters?.cpeName, + virtualMatchString: !postFilters?.virtualMatchString, + virtualMatchStringRange: !postFilters?.virtualMatchStringRange + }; +} - return matchCount > 0; +function allPostFiltersMatch(matchState) { + return matchState.cpeName && matchState.virtualMatchString && matchState.virtualMatchStringRange; } -function hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext) { - if ( - postFilters?.cpeName && - !cpe.hitAppliesToCpeName(hit, postFilters.cpeName, cpeEvaluationContext, { +function updatePostFilterMatchState(cpeMatch, postFilters, cpeEvaluationContext, matchState) { + if (!matchState.cpeName) { + matchState.cpeName = cpe.cpeMatchAppliesToCpeName(cpeMatch, postFilters.cpeName, cpeEvaluationContext, { requireVulnerable: postFilters.isVulnerable === true - }) - ) { - return false; + }); } - if ( - postFilters?.virtualMatchString && - !cpe.hitMatchesVirtualMatchString(hit, postFilters.virtualMatchString, cpeEvaluationContext) - ) { - return false; + if (!matchState.virtualMatchString) { + matchState.virtualMatchString = cpe.cpeMatchMatchesVirtualMatchString( + cpeMatch, + postFilters.virtualMatchString, + cpeEvaluationContext + ); } - if ( - postFilters?.virtualMatchStringRange && - !cpe.hitOverlapsVirtualMatchStringRange(hit, postFilters.virtualMatchStringRange, cpeEvaluationContext) - ) { - return false; + if (!matchState.virtualMatchStringRange) { + matchState.virtualMatchStringRange = cpe.cpeMatchOverlapsVirtualMatchStringRange( + cpeMatch, + postFilters.virtualMatchStringRange, + cpeEvaluationContext + ); } - - return true; } function yieldToEventLoop() { @@ -72,23 +70,54 @@ async function filterSearchHitsInChunks(resp, postFilters, signal, chunkSize) { const effectiveChunkSize = Math.max(1, chunkSize ?? constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE); const cpeEvaluationContext = cpe.createEvaluationContext(); const filteredHits = []; + let matchesSinceYield = 0; + + searchDeadline.throwIfAborted(signal); + for (const hit of hitsContainer.hits) { + const applicabilityItems = getCandidateApplicabilityItems(hit); + if (!applicabilityItems) { + throw new OpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_CANDIDATE); + } - for (let start = 0; start < hitsContainer.hits.length; start += effectiveChunkSize) { - searchDeadline.throwIfAborted(signal); - const end = Math.min(start + effectiveChunkSize, hitsContainer.hits.length); + const matchState = createPostFilterMatchState(postFilters); + let matchCount = 0; - for (let index = start; index < end; index += 1) { - const hit = hitsContainer.hits[index]; - if (!isValidCpeCandidateHit(hit, cpeEvaluationContext)) { + for (const applicability of applicabilityItems) { + if (!Array.isArray(applicability.nodes)) { throw new OpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_CANDIDATE); } - if (hitMatchesPostFilters(hit, postFilters, cpeEvaluationContext)) { - filteredHits.push(hit); + + for (const node of applicability.nodes) { + if (!isObject(node) || !Array.isArray(node.cpeMatch)) { + throw new OpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_CANDIDATE); + } + + for (const cpeMatch of node.cpeMatch) { + if (matchesSinceYield >= effectiveChunkSize) { + searchDeadline.throwIfAborted(signal); + await yieldToEventLoop(); + searchDeadline.throwIfAborted(signal); + matchesSinceYield = 0; + } + + if (!cpe.isValidCpeMatch(cpeMatch, cpeEvaluationContext)) { + throw new OpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_CANDIDATE); + } + + matchCount += 1; + matchesSinceYield += 1; + if (!allPostFiltersMatch(matchState)) { + updatePostFilterMatchState(cpeMatch, postFilters, cpeEvaluationContext, matchState); + } + } } } - if (end < hitsContainer.hits.length) { - await yieldToEventLoop(); + if (matchCount === 0) { + throw new OpenSearchResponseError(OPEN_SEARCH_RESPONSE_FAILURE_TYPES.MALFORMED_CANDIDATE); + } + if (allPostFiltersMatch(matchState)) { + filteredHits.push(hit); } } diff --git a/utils/searchCandidateProcessor.test.js b/utils/searchCandidateProcessor.test.js index 6048b35..9e277f5 100644 --- a/utils/searchCandidateProcessor.test.js +++ b/utils/searchCandidateProcessor.test.js @@ -1,4 +1,5 @@ const constants = require('./constants'); +const cpe = require('./cpe'); const searchCandidateProcessor = require('./searchCandidateProcessor'); const searchDeadline = require('./searchDeadline'); @@ -64,6 +65,10 @@ function nestedOnlyCpeCandidateHit() { } describe('searchCandidateProcessor', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + test('validates and filters candidates in the same processing pass', async () => { const matching = cpeCandidateHit({ versionStartIncluding: '2.9', @@ -194,6 +199,102 @@ describe('searchCandidateProcessor', () => { }); }); + test('continues validating CPE entries after the post-filter matches', async () => { + const candidate = cpeCandidateHit(); + candidate._source.containers.cna.cpeApplicability[0].nodes[0].cpeMatch.push({ + criteria: 'not-a-cpe-name', + vulnerable: true + }); + const matcher = jest.spyOn(cpe, 'cpeMatchAppliesToCpeName'); + + await expect( + searchCandidateProcessor.filterSearchHitsInChunks( + candidateResponse([candidate]), + { + cpeName: 'cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*', + isVulnerable: true + }, + undefined, + 10 + ) + ).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_candidate' + }); + expect(matcher).toHaveBeenCalledTimes(1); + }); + + test('preserves candidate order while filtering individual direct CPE entries', async () => { + const first = cpeCandidateHit(); + const excluded = cpeCandidateHit({ + criteria: 'cpe:2.3:a:other:product:*:*:*:*:*:*:*:*' + }); + const third = cpeCandidateHit(); + excluded._source.cveMetadata.cveId = 'CVE-2024-0002'; + third._source.cveMetadata.cveId = 'CVE-2024-0003'; + first._source.containers.cna.cpeApplicability[0].nodes[0].operator = 'AND'; + first._source.containers.cna.cpeApplicability[0].nodes[0].negate = true; + + await expect( + searchCandidateProcessor.filterSearchHitsInChunks( + candidateResponse([first, excluded, third]), + { + virtualMatchString: 'cpe:2.3:a:vendor:product' + }, + undefined, + 1 + ) + ).resolves.toMatchObject({ + candidateCount: 3, + filteredHits: [first, third] + }); + }); + + test('preserves independent entry matching when multiple post-filters are supplied', async () => { + const candidate = cpeCandidateHit(); + candidate._source.containers.cna.cpeApplicability[0].nodes[0].cpeMatch.push({ + criteria: 'cpe:2.3:a:other:component:*:*:*:*:*:*:*:*', + vulnerable: true + }); + + await expect( + searchCandidateProcessor.filterSearchHitsInChunks( + candidateResponse([candidate]), + { + cpeName: 'cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*', + virtualMatchString: 'cpe:2.3:a:other:component' + }, + undefined, + 10 + ) + ).resolves.toMatchObject({ + candidateCount: 1, + filteredHits: [candidate] + }); + }); + + test('honors cancellation between CPE entries within one candidate', async () => { + const candidate = cpeCandidateHit(); + const matches = candidate._source.containers.cna.cpeApplicability[0].nodes[0].cpeMatch; + matches.push(...Array.from({ length: 10 }, () => ({ ...matches[0] }))); + const abortController = new AbortController(); + const matcher = jest.spyOn(cpe, 'cpeMatchMatchesVirtualMatchString'); + const filtering = searchCandidateProcessor.filterSearchHitsInChunks( + candidateResponse([candidate]), + { + virtualMatchString: 'cpe:2.3:a:other:product' + }, + abortController.signal, + 1 + ); + + setImmediate(() => abortController.abort(new Error('cancelled'))); + + await expect(filtering).rejects.toThrow('cancelled'); + expect(matcher.mock.calls.length).toBeGreaterThan(0); + expect(matcher.mock.calls.length).toBeLessThan(matches.length); + }); + test('honors cancellation before candidate validation begins', async () => { const abortController = new AbortController(); abortController.abort(new searchDeadline.SearchClientDisconnectedError()); From 7c51b03bfd56ce557fd21990628c7a6d754d6395 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 11:12:23 -0400 Subject: [PATCH 22/33] Handle asynchronous startup failures --- README.md | 2 ++ server.js | 26 +++++++++++++++-- server.test.js | 76 +++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 063debe..a2e59f7 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,8 @@ npm start The Search API will be served at `localhost:3000`. +Successful port binding emits a sanitized `runtime.lifecycle` event with the `started` outcome. Synchronous configuration failures and asynchronous listener failures such as an unavailable or unauthorized port emit `startup_failed`, set a nonzero process exit code, and do not emit `started`. Startup failure logs contain only a stable error code and, for configuration errors, the relevant configuration path; exception messages, stacks, addresses, and port details are omitted. + ## Health Checks `GET /api/health-check` is a process liveness check. It returns HTTP 200 when the Node.js process is running and deliberately does not contact OpenSearch. Use it where a dependency outage should not restart otherwise healthy API tasks. diff --git a/server.js b/server.js index 820510a..6dd8bd4 100644 --- a/server.js +++ b/server.js @@ -150,15 +150,35 @@ function resolveListenPort(port, defaultPort) { return parsed; } -function startServer(port) { +function startServer( + port, + { + log = console.log, + startupErrorLog = console.error, + setExitCode = (code) => { + process.exitCode = code; + } + } = {} +) { const config = runtimeConfig.validateRuntimeConfig(); const effectivePort = resolveListenPort(port, config.server.port); const app = createApp(); const server = configureHttpServer(http.createServer(app), config.runtime.http); + let startupFailed = false; + const handleServerStartupError = (err) => { + startupFailed = true; + handleStartupError(err, startupErrorLog); + setExitCode(1); + }; + + server.once('error', handleServerStartupError); return server.listen(effectivePort, () => { + server.removeListener('error', handleServerStartupError); + if (startupFailed) return; + const sanitizedConfig = runtimeConfig.sanitizeRuntimeConfig(config); - logRuntimeEvent(console.log, { + logRuntimeEvent(log, { event: 'runtime.lifecycle', outcome: 'started', ...sanitizedConfig, @@ -321,7 +341,7 @@ function installShutdownHandlers(server, { processRef = process, shutdownOptions if (require.main === module) { try { const server = startServer(); - installShutdownHandlers(server); + server.once('listening', () => installShutdownHandlers(server)); } catch (err) { handleStartupError(err); process.exitCode = 1; diff --git a/server.test.js b/server.test.js index ba27bfe..debf205 100644 --- a/server.test.js +++ b/server.test.js @@ -61,6 +61,12 @@ function restoreEnv(name, value) { process.env[name] = value; } +function createFakeServer() { + const server = new EventEmitter(); + server.listen = jest.fn(() => server); + return server; +} + describe('server application bootstrap', () => { const originalBaseUrl = process.env.BASE_URL; const originalNodeEnv = process.env.NODE_ENV; @@ -346,8 +352,7 @@ describe('server application bootstrap', () => { test('startServer listens on the default port when PORT is not set', () => { delete process.env.PORT; - const fakeServer = {}; - fakeServer.listen = jest.fn(() => fakeServer); + const fakeServer = createFakeServer(); const createServer = jest.spyOn(http, 'createServer').mockReturnValue(fakeServer); expect(startServer()).toBe(fakeServer); @@ -378,14 +383,77 @@ describe('server application bootstrap', () => { }); test('startServer passes through an explicit port', () => { - const fakeServer = {}; - fakeServer.listen = jest.fn(() => fakeServer); + const fakeServer = createFakeServer(); jest.spyOn(http, 'createServer').mockReturnValue(fakeServer); expect(startServer('4567')).toBe(fakeServer); expect(fakeServer.listen).toHaveBeenCalledWith(4567, expect.any(Function)); }); + test.each(['EADDRINUSE', 'EACCES'])( + 'startServer handles an asynchronous %s failure through sanitized startup logging', + (code) => { + const fakeServer = createFakeServer(); + let listeningCallback; + fakeServer.listen.mockImplementation((_port, callback) => { + listeningCallback = callback; + return fakeServer; + }); + jest.spyOn(http, 'createServer').mockReturnValue(fakeServer); + const log = jest.fn(); + const startupErrorLog = jest.fn(); + const setExitCode = jest.fn(); + const thrownError = Object.assign(new Error('sensitive startup details'), { + code, + address: '0.0.0.0', + port: 3000 + }); + + expect(startServer(undefined, { log, setExitCode, startupErrorLog })).toBe(fakeServer); + expect(fakeServer.listenerCount('error')).toBe(1); + + fakeServer.emit('error', thrownError); + + expect(setExitCode).toHaveBeenCalledWith(1); + expect(startupErrorLog).toHaveBeenCalledWith( + JSON.stringify({ + event: 'runtime.lifecycle', + outcome: 'startup_failed', + errorCode: 'STARTUP_ERROR' + }) + ); + expect(JSON.stringify(startupErrorLog.mock.calls)).not.toContain('sensitive startup details'); + expect(JSON.stringify(startupErrorLog.mock.calls)).not.toContain('0.0.0.0'); + expect(fakeServer.listenerCount('error')).toBe(0); + + listeningCallback(); + expect(log).not.toHaveBeenCalled(); + } + ); + + test('startServer removes its startup error listener after listening succeeds', () => { + const fakeServer = createFakeServer(); + let listeningCallback; + fakeServer.listen.mockImplementation((_port, callback) => { + listeningCallback = callback; + return fakeServer; + }); + jest.spyOn(http, 'createServer').mockReturnValue(fakeServer); + const log = jest.fn(); + const startupErrorLog = jest.fn(); + const setExitCode = jest.fn(); + + startServer(undefined, { log, setExitCode, startupErrorLog }); + expect(fakeServer.listenerCount('error')).toBe(1); + + listeningCallback(); + + expect(fakeServer.listenerCount('error')).toBe(0); + expect(startupErrorLog).not.toHaveBeenCalled(); + expect(setExitCode).not.toHaveBeenCalled(); + expect(log).toHaveBeenCalledWith(expect.stringContaining('"outcome":"started"')); + }); + test('startServer rejects invalid configuration before creating an HTTP server', () => { process.env.PORT = 'invalid'; runtimeConfig.resetForTests(); From bcc0b5faa463dfda65b925e64e16bee8a736a264 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 11:21:01 -0400 Subject: [PATCH 23/33] Report rolling event loop telemetry --- README.md | 4 +- utils/searchPerformance.js | 98 ++++++++++++++++++----- utils/searchPerformance.test.js | 134 ++++++++++++++++++++++++++++++-- 3 files changed, 211 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index a2e59f7..1b3dac7 100644 --- a/README.md +++ b/README.md @@ -351,7 +351,9 @@ Access logs omit query strings and request bodies. By default, the API retains 1 The API always emits a sanitized JSON `search.performance` event for `/search` queue saturation, shared-operation waiter saturation, queue wait timeout, search timeout, client disconnect, OpenSearch connection errors, and upstream search errors. These events include the request ID, outcome, HTTP status, strategy, cache state, total elapsed time, cancellation reason, and bounded queue or waiter state when applicable. Search terms, CPE values, request bodies, credentials, and upstream error details are never included. -Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, shared waiter counts, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, response bytes, JSON serialization time, process heap usage and per-request heap delta, cache entry and cached CVE ID counts, process-wide event-loop delay, and total handler elapsed time. Leave it disabled during normal operation unless the deployment logging volume is acceptable. +Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, shared waiter counts, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, response bytes, JSON serialization time, process heap usage and per-request heap delta, cache entry and cached CVE ID counts, recent process-wide event-loop delay, and total handler elapsed time. Leave it disabled during normal operation unless the deployment logging volume is acceptable. + +Event-loop delay uses one low-overhead process monitor sampled in rolling 10-second windows. Request events report the most recently completed window through `processEventLoopDelayMeanMs`, `processEventLoopDelayP99Ms`, `processEventLoopDelayMaxMs`, `processEventLoopDelayWindowMs`, and `processEventLoopDelaySampleAgeMs`. These values describe process-wide event-loop saturation near the request, not delay caused by that individual request. They are omitted until the first complete window is available, and concurrent requests read the same completed snapshot without resetting one another's measurements. Heap deltas and event-loop delay are process-level diagnostics rather than isolated per-request measurements because concurrent requests share the same Node.js process. Use them to correlate latency with memory pressure or event-loop stalls across a test window, not to attribute resource use precisely to one request. diff --git a/utils/searchPerformance.js b/utils/searchPerformance.js index 95caf0b..3e365ae 100644 --- a/utils/searchPerformance.js +++ b/utils/searchPerformance.js @@ -2,8 +2,9 @@ const { monitorEventLoopDelay } = require('perf_hooks'); const searchProvider = require('./searchProvider'); const EVENT_LOOP_DELAY_RESOLUTION_MS = 20; +const EVENT_LOOP_DELAY_WINDOW_MS = 10000; const RESPONSE_JSON_INSTRUMENTED = Symbol('responseJsonInstrumented'); -let eventLoopDelayMonitor; +let eventLoopDelaySampler; function markTime() { return process.hrtime.bigint(); @@ -29,15 +30,69 @@ function nanosecondsToMs(value) { return Number.isFinite(value) ? value / 1e6 : undefined; } -function getEventLoopDelayMonitor() { - if (!eventLoopDelayMonitor) { - eventLoopDelayMonitor = monitorEventLoopDelay({ - resolution: EVENT_LOOP_DELAY_RESOLUTION_MS - }); - eventLoopDelayMonitor.enable(); +function createEventLoopDelaySampler({ + cancel = clearInterval, + createMonitor = () => monitorEventLoopDelay({ resolution: EVENT_LOOP_DELAY_RESOLUTION_MS }), + now = Date.now, + schedule = setInterval, + windowMs = EVENT_LOOP_DELAY_WINDOW_MS +} = {}) { + let latestSnapshot; + let monitor; + let timer; + let windowStartedAtMs; + + function captureWindow() { + if (!monitor) return; + + const sampledAtMs = now(); + latestSnapshot = { + meanMs: nanosecondsToMs(monitor.mean), + p99Ms: nanosecondsToMs(monitor.percentile(99)), + maxMs: nanosecondsToMs(monitor.max), + windowMs: Math.max(0, sampledAtMs - windowStartedAtMs), + sampledAtMs + }; + monitor.reset(); + windowStartedAtMs = sampledAtMs; + } + + return { + getSnapshot() { + if (!latestSnapshot) return undefined; + + return { + ...latestSnapshot, + sampleAgeMs: Math.max(0, now() - latestSnapshot.sampledAtMs) + }; + }, + start() { + if (monitor) return; + + monitor = createMonitor(); + monitor.enable(); + windowStartedAtMs = now(); + timer = schedule(captureWindow, windowMs); + timer?.unref?.(); + }, + stop() { + if (timer) cancel(timer); + monitor?.disable(); + latestSnapshot = undefined; + monitor = undefined; + timer = undefined; + windowStartedAtMs = undefined; + } + }; +} + +function getEventLoopDelaySampler() { + if (!eventLoopDelaySampler) { + eventLoopDelaySampler = createEventLoopDelaySampler(); + eventLoopDelaySampler.start(); } - return eventLoopDelayMonitor; + return eventLoopDelaySampler; } function instrumentJsonResponse(res, metrics) { @@ -63,7 +118,7 @@ function beginSearchRequest(metrics, res) { if (!searchProvider.shouldLogSearchPerformance()) return false; metrics.heapUsedStartBytes = process.memoryUsage().heapUsed; - getEventLoopDelayMonitor(); + getEventLoopDelaySampler(); instrumentJsonResponse(res, metrics); return true; } @@ -72,12 +127,16 @@ function completeSearchRequest(metrics) { if (metrics.heapUsedStartBytes === undefined) return; const heapUsedBytes = process.memoryUsage().heapUsed; - const monitor = getEventLoopDelayMonitor(); + const eventLoopDelaySnapshot = getEventLoopDelaySampler().getSnapshot(); metrics.heapUsedBytes = heapUsedBytes; metrics.heapUsedDeltaBytes = heapUsedBytes - metrics.heapUsedStartBytes; - metrics.eventLoopDelayMeanMs = nanosecondsToMs(monitor.mean); - metrics.eventLoopDelayP99Ms = nanosecondsToMs(monitor.percentile(99)); - metrics.eventLoopDelayMaxMs = nanosecondsToMs(monitor.max); + if (eventLoopDelaySnapshot) { + metrics.processEventLoopDelayMeanMs = eventLoopDelaySnapshot.meanMs; + metrics.processEventLoopDelayP99Ms = eventLoopDelaySnapshot.p99Ms; + metrics.processEventLoopDelayMaxMs = eventLoopDelaySnapshot.maxMs; + metrics.processEventLoopDelayWindowMs = eventLoopDelaySnapshot.windowMs; + metrics.processEventLoopDelaySampleAgeMs = eventLoopDelaySnapshot.sampleAgeMs; + } } function logSearchPerformance(metrics, searchOptions, responseOptions, postFilters) { @@ -123,9 +182,11 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte responseSerializationElapsedMs: roundMs(metrics.responseSerializationElapsedMs), heapUsedBytes: metrics.heapUsedBytes, heapUsedDeltaBytes: metrics.heapUsedDeltaBytes, - eventLoopDelayMeanMs: roundMs(metrics.eventLoopDelayMeanMs), - eventLoopDelayP99Ms: roundMs(metrics.eventLoopDelayP99Ms), - eventLoopDelayMaxMs: roundMs(metrics.eventLoopDelayMaxMs), + processEventLoopDelayMeanMs: roundMs(metrics.processEventLoopDelayMeanMs), + processEventLoopDelayP99Ms: roundMs(metrics.processEventLoopDelayP99Ms), + processEventLoopDelayMaxMs: roundMs(metrics.processEventLoopDelayMaxMs), + processEventLoopDelayWindowMs: roundMs(metrics.processEventLoopDelayWindowMs), + processEventLoopDelaySampleAgeMs: roundMs(metrics.processEventLoopDelaySampleAgeMs), openSearchTookMs: metrics.openSearchTookMs, pageFetchTookMs: metrics.pageFetchTookMs, openSearchElapsedMs: roundMs(metrics.openSearchElapsedMs), @@ -142,13 +203,14 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte } function resetForTests() { - eventLoopDelayMonitor?.disable(); - eventLoopDelayMonitor = undefined; + eventLoopDelaySampler?.stop(); + eventLoopDelaySampler = undefined; } module.exports = { beginSearchRequest, completeSearchRequest, + createEventLoopDelaySampler, elapsedMs, getCacheStatus, logSearchPerformance, diff --git a/utils/searchPerformance.test.js b/utils/searchPerformance.test.js index 52a7036..8ea90c3 100644 --- a/utils/searchPerformance.test.js +++ b/utils/searchPerformance.test.js @@ -17,6 +17,122 @@ describe('searchPerformance', () => { jest.restoreAllMocks(); }); + test('samples event-loop delay in completed rolling windows', () => { + let captureWindow; + let nowMs = 1000; + const timer = { + unref: jest.fn() + }; + const monitor = { + disable: jest.fn(), + enable: jest.fn(), + max: 12_345_000, + mean: 2_345_000, + percentile: jest.fn(() => 8_765_000), + reset: jest.fn() + }; + const cancel = jest.fn(); + const createMonitor = jest.fn(() => monitor); + const schedule = jest.fn((callback) => { + captureWindow = callback; + return timer; + }); + const sampler = searchPerformance.createEventLoopDelaySampler({ + cancel, + createMonitor, + now: () => nowMs, + schedule, + windowMs: 10000 + }); + + sampler.start(); + + expect(createMonitor).toHaveBeenCalledTimes(1); + expect(monitor.enable).toHaveBeenCalledTimes(1); + expect(schedule).toHaveBeenCalledWith(expect.any(Function), 10000); + expect(timer.unref).toHaveBeenCalledTimes(1); + expect(sampler.getSnapshot()).toBeUndefined(); + + nowMs = 11100; + captureWindow(); + + expect(monitor.percentile).toHaveBeenCalledWith(99); + expect(monitor.reset).toHaveBeenCalledTimes(1); + expect(sampler.getSnapshot()).toStrictEqual({ + meanMs: 2.345, + p99Ms: 8.765, + maxMs: 12.345, + windowMs: 10100, + sampledAtMs: 11100, + sampleAgeMs: 0 + }); + + nowMs = 11350; + expect(sampler.getSnapshot()).toMatchObject({ + meanMs: 2.345, + sampleAgeMs: 250 + }); + + sampler.start(); + expect(createMonitor).toHaveBeenCalledTimes(1); + expect(schedule).toHaveBeenCalledTimes(1); + + sampler.stop(); + expect(cancel).toHaveBeenCalledWith(timer); + expect(monitor.disable).toHaveBeenCalledTimes(1); + expect(sampler.getSnapshot()).toBeUndefined(); + }); + + test('publishes one immutable snapshot to concurrent readers until the next window completes', () => { + let captureWindow; + let nowMs = 0; + const monitor = { + disable: jest.fn(), + enable: jest.fn(), + max: 3_000_000, + mean: 1_000_000, + percentile: jest.fn(() => 2_000_000), + reset: jest.fn() + }; + const sampler = searchPerformance.createEventLoopDelaySampler({ + cancel: jest.fn(), + createMonitor: () => monitor, + now: () => nowMs, + schedule: (callback) => { + captureWindow = callback; + return { unref: jest.fn() }; + }, + windowMs: 10000 + }); + + sampler.start(); + nowMs = 10000; + captureWindow(); + const firstReader = sampler.getSnapshot(); + const secondReader = sampler.getSnapshot(); + + expect(firstReader).toStrictEqual(secondReader); + expect(firstReader).not.toBe(secondReader); + expect(monitor.reset).toHaveBeenCalledTimes(1); + + monitor.mean = 4_000_000; + monitor.max = 6_000_000; + monitor.percentile.mockReturnValue(5_000_000); + nowMs = 20500; + captureWindow(); + + expect(sampler.getSnapshot()).toMatchObject({ + meanMs: 4, + p99Ms: 5, + maxMs: 6, + windowMs: 10500, + sampleAgeMs: 0 + }); + expect(monitor.reset).toHaveBeenCalledTimes(2); + + sampler.stop(); + }); + test('logSearchPerformance emits redacted search timing metadata when enabled', () => { mockShouldLogSearchPerformance.mockReturnValue(true); @@ -43,9 +159,11 @@ describe('searchPerformance', () => { responseSerializationElapsedMs: 1.234, heapUsedBytes: 2000000, heapUsedDeltaBytes: 50000, - eventLoopDelayMeanMs: 2.345, - eventLoopDelayP99Ms: 8.765, - eventLoopDelayMaxMs: 12.345, + processEventLoopDelayMeanMs: 2.345, + processEventLoopDelayP99Ms: 8.765, + processEventLoopDelayMaxMs: 12.345, + processEventLoopDelayWindowMs: 10000.123, + processEventLoopDelaySampleAgeMs: 250.456, openSearchTookMs: 97, pageFetchTookMs: 8, openSearchElapsedMs: 123.456, @@ -106,9 +224,11 @@ describe('searchPerformance', () => { responseSerializationElapsedMs: 1.23, heapUsedBytes: 2000000, heapUsedDeltaBytes: 50000, - eventLoopDelayMeanMs: 2.35, - eventLoopDelayP99Ms: 8.77, - eventLoopDelayMaxMs: 12.35, + processEventLoopDelayMeanMs: 2.35, + processEventLoopDelayP99Ms: 8.77, + processEventLoopDelayMaxMs: 12.35, + processEventLoopDelayWindowMs: 10000.12, + processEventLoopDelaySampleAgeMs: 250.46, openSearchTookMs: 97, pageFetchTookMs: 8, openSearchElapsedMs: 123.46, @@ -175,6 +295,8 @@ describe('searchPerformance', () => { heapUsedDeltaBytes: expect.any(Number) }); expect(metrics.responseSerializationElapsedMs).toBeGreaterThanOrEqual(0); + expect(metrics.processEventLoopDelayMeanMs).toBeUndefined(); + expect(metrics.processEventLoopDelayWindowMs).toBeUndefined(); }); test('logSearchPerformance always emits sanitized failure outcomes', () => { From b844b14e839a908a1904c81a249822b92a558e9b Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 12:19:31 -0400 Subject: [PATCH 24/33] refactor: remove unused internal exports --- controllers/searchController.js | 16 ++---- routes/search.integration.test.js | 3 + server.js | 11 ---- server.test.js | 48 +--------------- utils/searchProvider.js | 3 - utils/searchQueryBuilder.js | 5 +- utils/searchQueryBuilder.test.js | 92 ++++++++++++++++--------------- 7 files changed, 61 insertions(+), 117 deletions(-) diff --git a/controllers/searchController.js b/controllers/searchController.js index 8853867..16dc69d 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -82,16 +82,16 @@ function recordUpstreamFailure(metrics, err, failureType) { function isOpenSearchUnavailableError(err) { return ( - searchProvider.isSearchProviderContractError?.(err) === true || - searchProvider.isSearchProviderConfigurationError?.(err) === true || + searchProvider.isSearchProviderContractError(err) === true || + searchProvider.isSearchProviderConfigurationError(err) === true || openSearchUnavailableErrorNames.has(err?.name) || openSearchUnavailableStatusCodes.has(getOpenSearchErrorStatusCode(err)) ); } function getOpenSearchUnavailableFailureType(err) { - if (searchProvider.isSearchProviderContractError?.(err)) return 'provider_contract'; - if (searchProvider.isSearchProviderConfigurationError?.(err)) return 'provider_configuration'; + if (searchProvider.isSearchProviderContractError(err)) return 'provider_contract'; + if (searchProvider.isSearchProviderConfigurationError(err)) return 'provider_configuration'; return 'unavailable'; } @@ -128,12 +128,8 @@ function buildWebSearchSort(sort) { return undefined; } -function getHitCveId(hit) { - return hit?._source?.cveMetadata?.cveId ?? hit?._id; -} - function getHitCveIds(hits) { - return hits.map(getHitCveId).filter(Boolean); + return hits.map((hit) => hit._source.cveMetadata.cveId); } function validateAndOrderPageHits(hits, cveIds) { @@ -435,7 +431,7 @@ async function searchHandler(req, res, next) { return res.status(429).json(error.searchInFlightWaiterLimitError()); } - if (searchProvider.isOpenSearchResponseError?.(err)) { + if (searchProvider.isOpenSearchResponseError(err)) { const failureType = err?.failureType ?? 'malformed_response'; recordUpstreamFailure(metrics, err, failureType); diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index 9aab62a..bc79a6d 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -11,6 +11,9 @@ jest.mock('../utils/searchProvider', () => ({ getInFlightMaxWaitersPerOperation: jest.fn(() => 500), getPostFilterEvaluationChunkSize: jest.fn(() => 250), getSearchRequestDeadlineMs: jest.fn(() => 30000), + isOpenSearchResponseError: jest.fn(() => false), + isSearchProviderConfigurationError: jest.fn(() => false), + isSearchProviderContractError: jest.fn(() => false), searchOpenSearch: mockSearchOpenSearch, shouldLogSearchPerformance: jest.fn(() => false) })); diff --git a/server.js b/server.js index 6dd8bd4..9089b9b 100644 --- a/server.js +++ b/server.js @@ -127,15 +127,6 @@ function createApp() { return app; } -function getRuntimeConfig(port) { - const config = runtimeConfig.validateRuntimeConfig(); - const effectivePort = resolveListenPort(port, config.server.port); - - return { - baseURL: config.server.baseURL || `http://127.0.0.1:${effectivePort}` - }; -} - function resolveListenPort(port, defaultPort) { if (port === undefined) return defaultPort; @@ -354,8 +345,6 @@ module.exports = { createRequestLogger, handleUnexpectedError, installShutdownHandlers, - getRedactedRequestPath: requestLogging.getRedactedRequestPath, - getRuntimeConfig, handleStartupError, rejectRequestsDuringShutdown, shutdownServer, diff --git a/server.test.js b/server.test.js index debf205..a558198 100644 --- a/server.test.js +++ b/server.test.js @@ -40,8 +40,6 @@ const { configureHttpServer, createApp, createRequestLogger, - getRedactedRequestPath, - getRuntimeConfig, handleStartupError, handleUnexpectedError, installShutdownHandlers, @@ -49,6 +47,7 @@ const { startServer, swaggerSetupOptions } = require('./server'); +const requestLogging = require('./utils/requestLogging'); const runtimeConfig = require('./utils/runtimeConfig'); const runtimeLifecycle = require('./utils/runtimeLifecycle'); @@ -68,7 +67,6 @@ function createFakeServer() { } describe('server application bootstrap', () => { - const originalBaseUrl = process.env.BASE_URL; const originalNodeEnv = process.env.NODE_ENV; const originalPort = process.env.PORT; @@ -77,7 +75,6 @@ describe('server application bootstrap', () => { }); afterEach(() => { - restoreEnv('BASE_URL', originalBaseUrl); restoreEnv('NODE_ENV', originalNodeEnv); restoreEnv('PORT', originalPort); runtimeLifecycle.resetForTests(); @@ -284,47 +281,6 @@ describe('server application bootstrap', () => { expect(mockSearch).not.toHaveBeenCalled(); }); - test('getRuntimeConfig preserves baseURL behavior when PORT is set without BASE_URL', () => { - delete process.env.BASE_URL; - process.env.PORT = '4567'; - - expect(getRuntimeConfig('4567')).toStrictEqual({ - baseURL: 'http://127.0.0.1:4567' - }); - }); - - test('getRuntimeConfig uses the effective default port when PORT is not configured', () => { - delete process.env.BASE_URL; - delete process.env.PORT; - - expect(getRuntimeConfig(3000)).toStrictEqual({ - baseURL: 'http://127.0.0.1:3000' - }); - }); - - test('getRuntimeConfig uses BASE_URL when it is already configured', () => { - process.env.BASE_URL = 'https://search.example'; - process.env.PORT = '4567'; - - expect(getRuntimeConfig('4567')).toStrictEqual({ - baseURL: 'https://search.example' - }); - }); - - test('getRedactedRequestPath omits query strings from logged URLs', () => { - expect( - getRedactedRequestPath({ - path: '/search', - originalUrl: '/search?keywordSearch=apache' - }) - ).toBe('/search'); - expect( - getRedactedRequestPath({ - originalUrl: '/webSearch?query=apache' - }) - ).toBe('/webSearch'); - }); - test('createRequestLogger disables request logging in test environments', () => { process.env.NODE_ENV = 'test'; const next = jest.fn(); @@ -340,7 +296,7 @@ describe('server application bootstrap', () => { createRequestLogger(); - expect(mockMorgan.token).toHaveBeenCalledWith('redacted-url', getRedactedRequestPath); + expect(mockMorgan.token).toHaveBeenCalledWith('redacted-url', requestLogging.getRedactedRequestPath); expect(mockMorgan.token).toHaveBeenCalledWith('request-id', expect.any(Function)); expect(mockMorgan).toHaveBeenCalledWith( ':method :redacted-url :status :res[content-length] - :response-time ms request-id=:request-id', diff --git a/utils/searchProvider.js b/utils/searchProvider.js index 8a92326..e7ffe6a 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -359,10 +359,7 @@ module.exports = { resetSearchProviderForTests, searchOpenSearch, shouldLogSearchPerformance, - validateOpenSearchResponse: searchResponseValidator.validateOpenSearchResponse, validateSearchReader, - OPEN_SEARCH_RESPONSE_FAILURE_TYPES: searchResponseValidator.OPEN_SEARCH_RESPONSE_FAILURE_TYPES, - OpenSearchResponseError: searchResponseValidator.OpenSearchResponseError, SearchProviderConfigurationError, SearchProviderContractError }; diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index 2a1f62b..f30edc2 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -616,8 +616,5 @@ module.exports = { buildCpeCandidateSearchBody, buildCveIdsSearchBody, buildOpenSearchBody, - buildQueryStrings, - buildSearchRequest, - getFilterFields, - processParams + buildSearchRequest }; diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index 71e580b..79f9c2c 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -46,6 +46,14 @@ function anyOf(clauses) { }; } +function buildQueryStrings(params) { + return queryBuilder.buildSearchRequest(params).queryStringObjects; +} + +function getFilterFields(params) { + return queryBuilder.buildSearchRequest(params).searchOptions.filters; +} + describe('searchQueryBuilder buildSearchRequest()', () => { test('buildSearchRequest builds filters, query strings, date ranges, and pagination options', () => { const output = queryBuilder.buildSearchRequest({ @@ -493,8 +501,8 @@ describe('searchQueryBuilder buildSearchRequest()', () => { }); }); -describe('searchQueryBuilder buildQueryStrings() function', () => { - test('buildQueryStrings() function correctly builds array of one simple_query_string object when keywordSearch is passed.', () => { +describe('searchQueryBuilder buildSearchRequest() query strings', () => { + test('buildSearchRequest() builds one simple_query_string object when keywordSearch is passed.', () => { const testParams = { keywordSearch: 'test* params*' }; @@ -502,20 +510,20 @@ describe('searchQueryBuilder buildQueryStrings() function', () => { const testQueryString = [ { simple_query_string: { - query: 'test* params*', + query: 'test\\** params\\**', default_operator: 'AND', fields: ['containers.cna.descriptions.value'] } } ]; - const output = queryBuilder.buildQueryStrings(testParams); + const output = buildQueryStrings(testParams); expect(output).toHaveLength(1); expect(output).toStrictEqual(testQueryString); }); - test('buildQueryStrings() ignores non-keyword fields instead of generating query_string clauses.', () => { - const output = queryBuilder.buildQueryStrings({ + test('buildSearchRequest() ignores non-keyword fields instead of generating query string clauses.', () => { + const output = buildQueryStrings({ cweId: 'CWE-79', virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809' }); @@ -524,8 +532,8 @@ describe('searchQueryBuilder buildQueryStrings() function', () => { }); }); -describe('searchQueryBuilder getFilterFields() function', () => { - test('getFilterFields() function correctly builds array of one matchPhrase object when one parameter is passed.', () => { +describe('searchQueryBuilder buildSearchRequest() filters', () => { + test('buildSearchRequest() builds one match phrase when one filter parameter is passed.', () => { const testParams = { cweId: 'CWE-79' }; @@ -538,12 +546,12 @@ describe('searchQueryBuilder getFilterFields() function', () => { } ]; - const output = queryBuilder.getFilterFields(testParams); + const output = getFilterFields(testParams); expect(output).toHaveLength(1); expect(output).toStrictEqual(testMatchPhrase); }); - test('getFilterFields() function correctly builds default and severity filter objects when multiple parameters are passed.', () => { + test('buildSearchRequest() builds default and severity filters when multiple parameters are passed.', () => { const testParams = { cweId: 'CWE-79', cvssV4Severity: 'HIGH' @@ -562,13 +570,13 @@ describe('searchQueryBuilder getFilterFields() function', () => { } ]; - const output = queryBuilder.getFilterFields(testParams); + const output = getFilterFields(testParams); expect(output).toHaveLength(2); expect(output).toStrictEqual(testMatchPhrase); }); - test('getFilterFields() builds exact terms filter for comma-separated cveIds.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds an exact terms filter for comma-separated cveIds.', () => { + const output = getFilterFields({ cveIds: 'cve-2024-0002, CVE-2024-0001,CVE-2024-0002' }); @@ -581,8 +589,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds rejected-state exclusions for noRejected.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds rejected-state exclusions for noRejected.', () => { + const output = getFilterFields({ noRejected: true }); @@ -610,24 +618,24 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() ignores noRejected when it is false.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() ignores noRejected when it is false.', () => { + const output = getFilterFields({ noRejected: false }); expect(output).toStrictEqual([]); }); - test('getFilterFields() does not retain legacy empty-string noRejected behavior.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() does not retain legacy empty-string noRejected behavior.', () => { + const output = getFilterFields({ noRejected: '' }); expect(output).toStrictEqual([]); }); - test('getFilterFields() builds a cvssV2 severity range filter on baseScore.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds a cvssV2 severity range filter on baseScore.', () => { + const output = getFilterFields({ cvssV2Severity: 'MEDIUM' }); @@ -643,8 +651,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds a same-vector regexp filter for CNA cvssV2 metrics.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds a same-vector regexp filter for CNA cvssV2 metrics.', () => { + const output = getFilterFields({ cvssV2Metrics: 'AV:N/AC:L/Au:N' }); @@ -653,8 +661,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds exact-version and wildcard-version CPE Name candidate filters.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds exact-version and wildcard-version CPE Name candidate filters.', () => { + const output = getFilterFields({ cpeName: 'cpe:2.3:a:artifex:ghostscript:10.05.1:*:*:*:*:*:*:*' }); @@ -687,8 +695,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { expect(output[0].bool.should).toHaveLength(2); }); - test('getFilterFields() leaves CPE version range applicability to semantic post-filtering.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() leaves CPE version range applicability to semantic post-filtering.', () => { + const output = getFilterFields({ cpeName: 'cpe:2.3:a:artifex:ghostscript:10.05.1:*:*:*:*:*:*:*' }); @@ -703,14 +711,12 @@ describe('searchQueryBuilder getFilterFields() function', () => { test.each([ ['cpeName', { cpeName: 'not-a-cpe-name' }], ['virtualMatchString', { virtualMatchString: 'not-a-cpe-match-string' }] - ])('getFilterFields() fails closed when invalid %s reaches query construction', (param, params) => { - expect(() => queryBuilder.getFilterFields(params)).toThrow( - `Invalid ${param} value reached query construction.` - ); + ])('buildSearchRequest() fails closed when invalid %s reaches query construction', (param, params) => { + expect(() => getFilterFields(params)).toThrow(`Invalid ${param} value reached query construction.`); }); - test('getFilterFields() builds combined cvssV3 severity filters across v3.0 and v3.1.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds combined cvssV3 severity filters across v3.0 and v3.1.', () => { + const output = getFilterFields({ cvssV3Severity: 'critical' }); @@ -735,8 +741,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() constrains versioned cvssV3 metrics to the matching CNA field.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() constrains versioned cvssV3 metrics to the matching CNA field.', () => { + const output = getFilterFields({ cvssV3Metrics: 'CVSS:3.1/AC:L/AV:N' }); @@ -745,8 +751,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() searches both v3 versions when cvssV3Metrics omits a header.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() searches both v3 versions when cvssV3Metrics omits a header.', () => { + const output = getFilterFields({ cvssV3Metrics: 'AV:N' }); @@ -758,8 +764,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() escapes regexp metacharacters in CVSS metric filters.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() escapes regexp metacharacters in CVSS metric filters.', () => { + const output = getFilterFields({ cvssV4Metrics: 'CVSS#4.0/AV:N/AC:*?/UI:P' }); @@ -770,16 +776,16 @@ describe('searchQueryBuilder getFilterFields() function', () => { expect(JSON.stringify(output)).not.toContain('containers.adp'); }); - test('getFilterFields() fails closed when invalid CVSS metrics reach query construction.', () => { + test('buildSearchRequest() fails closed when invalid CVSS metrics reach query construction.', () => { expect(() => - queryBuilder.getFilterFields({ + getFilterFields({ cvssV3Metrics: 'CVSS:3.1' }) ).toThrow('Invalid cvssV3Metrics value reached query construction.'); }); - test('getFilterFields() does not build filter objects for incorrect parameters.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() does not build filter objects for incorrect parameters.', () => { + const output = getFilterFields({ test: 'EXTERNAL' }); From 2ad4b7dbfff7a7f9e45a95b06d2cf61ff1c2c391 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 12:23:27 -0400 Subject: [PATCH 25/33] refactor: remove obsolete base URL config --- .env.example | 1 - README.md | 3 +-- utils/runtimeConfig.js | 3 --- utils/runtimeConfig.test.js | 24 ++---------------------- 4 files changed, 3 insertions(+), 28 deletions(-) diff --git a/.env.example b/.env.example index ef0af0e..f2680a2 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,5 @@ NODE_CONFIG_ENV=devel PORT=3000 -BASE_URL= OpenSearchCveIndex=cve-index-local OpenSearchAllowUnknownSslCerts=false OpenSearchDomainEndpoint= diff --git a/README.md b/README.md index 1b3dac7..8a9f7e0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ Rename `.env.example` to `.env` and replace or check the following values. - `NODE_CONFIG_ENV` - node-config profile name; use `devel` locally and `prod` in production images - `PORT` - port used for the search API, defaulted to 3000 for development -- `BASE_URL` - optional absolute public URL used by local runtime helpers; credentials are not permitted - `OpenSearchCveIndex` - existing CVE index name in your OpenSearch deployment - `OpenSearchAllowUnknownSslCerts` - set to `true` only for local development against self-signed certificates - `OpenSearchDomainEndpoint` - absolute URL to an existing OpenSearch deployment. This API does not create or manage the OpenSearch index or mapping. @@ -72,7 +71,7 @@ SearchPostFilterColdSearchMaxWaitMs < SearchRequestDeadlineMs HttpHeadersTimeoutMs <= HttpRequestTimeoutMs ``` -`OpenSearchDomainEndpoint` must be an absolute HTTP or HTTPS URL, and `OpenSearchCveIndex` must be non-empty. `BASE_URL`, when supplied, must also be an absolute HTTP or HTTPS URL and cannot contain credentials. OpenSearch connectivity is not required at startup; `/api/ready-check` remains responsible for dependency readiness. +`OpenSearchDomainEndpoint` must be an absolute HTTP or HTTPS URL, and `OpenSearchCveIndex` must be non-empty. OpenSearch connectivity is not required at startup; `/api/ready-check` remains responsible for dependency readiness. Production images set both `NODE_ENV=production` and `NODE_CONFIG_ENV=prod`, matching `config/prod.jsonc`. Successful startup emits a sanitized `runtime.lifecycle` JSON event with effective safety settings. Startup failures identify only the invalid configuration path and never log configured values, URLs, index names, credentials, or exception messages. diff --git a/utils/runtimeConfig.js b/utils/runtimeConfig.js index 9fc709f..77745c4 100644 --- a/utils/runtimeConfig.js +++ b/utils/runtimeConfig.js @@ -189,12 +189,9 @@ function resolveRuntimeConfig({ readConfigValue(appConfig, 'search.index', defaultSearchProvider.index), { maxLength: 255 } ); - const baseURL = parseOptionalString('BASE_URL', env.BASE_URL, { maxLength: 4096 }); - const config = { server: { port: parseInteger('PORT', env.PORT ?? DEFAULT_PORT, RUNTIME_CONFIG_LIMITS.port), - baseURL: parseHttpUrl('BASE_URL', baseURL, { allowCredentials: false }), nodeEnvironment: String(env.NODE_ENV || 'development'), configEnvironment: String(env.NODE_CONFIG_ENV || env.NODE_ENV || 'default') }, diff --git a/utils/runtimeConfig.test.js b/utils/runtimeConfig.test.js index 850ead3..90777c4 100644 --- a/utils/runtimeConfig.test.js +++ b/utils/runtimeConfig.test.js @@ -37,7 +37,6 @@ describe('runtimeConfig', () => { expect(config).toMatchObject({ server: { port: 3000, - baseURL: '', nodeEnvironment: 'test', configEnvironment: 'test' }, @@ -115,7 +114,6 @@ describe('runtimeConfig', () => { }, { PORT: '4000', - BASE_URL: 'https://api.example', NODE_CONFIG_ENV: 'prod' } ); @@ -123,7 +121,6 @@ describe('runtimeConfig', () => { expect(config).toMatchObject({ server: { port: 4000, - baseURL: 'https://api.example', nodeEnvironment: 'test', configEnvironment: 'prod' }, @@ -244,29 +241,12 @@ describe('runtimeConfig', () => { expect(() => runtimeConfig.validateRuntimeConfig(config)).toThrow(path); }); - test('rejects credential-bearing BASE_URL values', () => { - expect(() => - resolve( - {}, - { - BASE_URL: 'https://admin:secret@api.example' - } - ) - ).toThrow('BASE_URL must not contain credentials.'); - }); - - test('sanitized startup metadata does not expose provider or base URLs', () => { - const config = resolve( - {}, - { - BASE_URL: 'https://api.example' - } - ); + test('sanitized startup metadata does not expose provider configuration', () => { + const config = resolve(); const serialized = JSON.stringify(runtimeConfig.sanitizeRuntimeConfig(config)); expect(serialized).not.toContain('search.example'); expect(serialized).not.toContain('test-index'); - expect(serialized).not.toContain('api.example'); }); }); From 89500a7b2ed9abf029b57df9862f3c060abd27bf Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 12:35:11 -0400 Subject: [PATCH 26/33] refactor: simplify search validation --- controllers/middleware.js | 66 ++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/controllers/middleware.js b/controllers/middleware.js index a822cc3..af2c0ce 100644 --- a/controllers/middleware.js +++ b/controllers/middleware.js @@ -44,9 +44,11 @@ const dateRangePairs = [ ['lastModStartDate', 'lastModEndDate'], ['pubStartDate', 'pubEndDate'] ]; -const versionRangeParams = ['versionStart', 'versionStartType', 'versionEnd', 'versionEndType']; -const versionRangeValueParams = ['versionStart', 'versionEnd']; -const versionRangeTypeParams = ['versionStartType', 'versionEndType']; +const versionRangeBounds = [ + ['versionStart', 'versionStartType', 'versionStartPairError'], + ['versionEnd', 'versionEndType', 'versionEndPairError'] +]; +const versionRangeParams = versionRangeBounds.flatMap(([valueParam, typeParam]) => [valueParam, typeParam]); const nonStringSearchParams = [ 'keywordExactMatch', 'cveIds', @@ -89,20 +91,13 @@ function parseCveIds(value) { return cveIds; } -function getRequestedResultsPerPage(params) { - return hasParam(params, 'resultsPerPage') ? Number(params.resultsPerPage) : constants.RESULTS_PER_PAGE; -} - -function getRequestedPageNumber(params) { - return hasParam(params, 'pageNumber') ? Number(params.pageNumber) : 1; -} - -function exceedsSearchWindow(params) { - const resultsPerPage = getRequestedResultsPerPage(params); - const pageNumber = getRequestedPageNumber(params); - const from = resultsPerPage * (pageNumber - 1); +function getPagination(params) { + const resultsPerPage = hasParam(params, 'resultsPerPage') + ? Number(params.resultsPerPage) + : constants.RESULTS_PER_PAGE; + const pageNumber = hasParam(params, 'pageNumber') ? Number(params.pageNumber) : 1; - return from + resultsPerPage > constants.MAX_SEARCH_WINDOW; + return { resultsPerPage, pageNumber, from: resultsPerPage * (pageNumber - 1) }; } function isValidDateTime(value) { @@ -161,26 +156,28 @@ function validateParams(req, res, next) { return res.status(400).json(error.noRejectedValueError()); } + const parsedVirtualMatchString = params.virtualMatchString + ? cpe.parseVirtualMatchString(params.virtualMatchString) + : null; + if (hasAnyParam(params, versionRangeParams)) { if (!params.virtualMatchString) { return res.status(400).json(error.versionRangeVirtualMatchStringRequiredError()); } - if (hasParam(params, 'versionStart') !== hasParam(params, 'versionStartType')) { - return res.status(400).json(error.versionStartPairError()); - } - - if (hasParam(params, 'versionEnd') !== hasParam(params, 'versionEndType')) { - return res.status(400).json(error.versionEndPairError()); + for (const [valueParam, typeParam, pairError] of versionRangeBounds) { + if (hasParam(params, valueParam) !== hasParam(params, typeParam)) { + return res.status(400).json(error[pairError]()); + } } - for (const param of versionRangeValueParams) { + for (const [param] of versionRangeBounds) { if (hasParam(params, param) && !isSingleNonEmptyString(params[param])) { return res.status(400).json(error.versionRangeValueError(param)); } } - for (const param of versionRangeTypeParams) { + for (const [, param] of versionRangeBounds) { if ( hasParam(params, param) && (!isSingleNonEmptyString(params[param]) || @@ -190,12 +187,11 @@ function validateParams(req, res, next) { } } - const virtualMatchString = cpe.parseVirtualMatchString(params.virtualMatchString); - if (!virtualMatchString.valid) { + if (!parsedVirtualMatchString.valid) { return res.status(400).json(error.virtualMatchStringVersionRangeFormatError()); } - if (virtualMatchString.hasVersionComponent) { + if (parsedVirtualMatchString.hasVersionComponent) { return res.status(400).json(error.virtualMatchStringVersionComponentError()); } @@ -230,8 +226,7 @@ function validateParams(req, res, next) { } if (params?.virtualMatchString) { - const virtualMatchString = cpe.parseVirtualMatchString(params.virtualMatchString); - if (!virtualMatchString.valid) { + if (!parsedVirtualMatchString.valid) { return res.status(400).json(error.virtualMatchStringFormatError()); } } @@ -246,29 +241,28 @@ function validateParams(req, res, next) { return res.status(404).json(error.invalidCpeNameError()); } + const pagination = getPagination(params); + //Ensures that resultsPerPage is a positive integer if ( hasParam(params, 'resultsPerPage') && - (!Number.isInteger(Number(params.resultsPerPage)) || Number(params.resultsPerPage) < 1) + (!Number.isInteger(pagination.resultsPerPage) || pagination.resultsPerPage < 1) ) { return res.status(400).json(error.resultsPerPageError()); } //Ensures that resultsPerPage is not above the OpenSearch page size cap - if (hasParam(params, 'resultsPerPage') && Number(params.resultsPerPage) > constants.MAX_RESULTS_LIMIT) { + if (hasParam(params, 'resultsPerPage') && pagination.resultsPerPage > constants.MAX_RESULTS_LIMIT) { return res.status(400).json(error.maxResultsError()); } //Ensures that pageNumber is a positive integer - if ( - hasParam(params, 'pageNumber') && - (!Number.isInteger(Number(params.pageNumber)) || Number(params.pageNumber) < 1) - ) { + if (hasParam(params, 'pageNumber') && (!Number.isInteger(pagination.pageNumber) || pagination.pageNumber < 1)) { return res.status(400).json(error.pageNumberError()); } //Ensures shallow pagination stays within the OpenSearch from/size result window - if (exceedsSearchWindow(params)) { + if (pagination.from + pagination.resultsPerPage > constants.MAX_SEARCH_WINDOW) { return res.status(400).json(error.searchWindowError()); } From aad8976ad37bb82d8cce5dd42d8a42ad96968c07 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 12:43:46 -0400 Subject: [PATCH 27/33] test: consolidate search validation coverage --- controllers/middleware.test.js | 175 +++++------------------------- routes/cpeName.test.js | 62 ----------- routes/search.integration.test.js | 95 ---------------- 3 files changed, 26 insertions(+), 306 deletions(-) delete mode 100644 routes/cpeName.test.js diff --git a/controllers/middleware.test.js b/controllers/middleware.test.js index 6f82844..6bfe8d4 100644 --- a/controllers/middleware.test.js +++ b/controllers/middleware.test.js @@ -592,132 +592,42 @@ describe('middleware validation function', () => { expectValidationSuccess(); }); - test('ValidateParams returns an error when a non enumerator value is passed for cvssV2Severity.', () => { - validate({ - cvssV2Severity: 'critical' - }); - - expectValidationError( - 'INVALID_CVSSV2_SEVERITY', - 'critical is not an allowed input for the cvssV2Severity. Please use LOW, MEDIUM, or HIGH.' - ); - }); - - test('ValidateParams successfully accepts enumerator value medium for cvssV2Severity.', () => { - validate({ - cvssV2Severity: 'medium' - }); - - expectValidationSuccess(); - }); - - test('ValidateParams returns an error when a non enumerator value is passed for cvssV3Severity.', () => { - validate({ - cvssV3Severity: 'Test' - }); - - expectValidationError( - 'INVALID_CVSSV3_SEVERITY', - 'Test is not an allowed input for the cvssV3Severity. Please use LOW, MEDIUM, HIGH, or CRITICAL.' - ); - }); - - test('ValidateParams successfully accepts enumerator value critical for cvssV3Severity.', () => { - validate({ - cvssV3Severity: 'critical' - }); - - expectValidationSuccess(); - }); - - test('ValidateParams returns an error when a non enumerator value is passed for cvssV3_0Severity.', () => { - validate({ - cvssV3_0Severity: 'Test' - }); - - expectValidationError( - 'INVALID_CVSSV3_0_SEVERITY', - 'Test is not an allowed input for the cvssV3_0Severity. Please use LOW, MEDIUM, HIGH, or CRITICAL.' - ); - }); - - test('ValidateParams successfully accepts enumerator value high for cvssV3_0Severity.', () => { - validate({ - cvssV3_0Severity: 'high' - }); - - expectValidationSuccess(); - }); - - test('ValidateParams returns an error when a non enumerator value is passed for cvssV3_1Severity.', () => { - validate({ - cvssV3_1Severity: 'Test' - }); - - expectValidationError( - 'INVALID_CVSSV3_1_SEVERITY', - 'Test is not an allowed input for the cvssV3_1Severity. Please use LOW, MEDIUM, HIGH, or CRITICAL.' - ); - }); - - test('ValidateParams successfully accepts enumerator value critical for cvssV3_1Severity.', () => { - validate({ - cvssV3_1Severity: 'critical' - }); - - expectValidationSuccess(); - }); - - test('ValidateParams returns an error when a non enumerator value is passed for cvssV4Severity.', () => { - validate({ - cvssV4Severity: 'Test' - }); + test.each([ + ['cvssV2Severity', 'critical', 'INVALID_CVSSV2_SEVERITY', 'LOW, MEDIUM, or HIGH'], + ['cvssV3Severity', 'Test', 'INVALID_CVSSV3_SEVERITY', 'LOW, MEDIUM, HIGH, or CRITICAL'], + ['cvssV3_0Severity', 'Test', 'INVALID_CVSSV3_0_SEVERITY', 'LOW, MEDIUM, HIGH, or CRITICAL'], + ['cvssV3_1Severity', 'Test', 'INVALID_CVSSV3_1_SEVERITY', 'LOW, MEDIUM, HIGH, or CRITICAL'], + ['cvssV4Severity', 'Test', 'INVALID_CVSSV4_SEVERITY', 'LOW, MEDIUM, HIGH, or CRITICAL'] + ])('ValidateParams rejects an invalid %s value.', (param, value, errorCode, allowedValues) => { + validate({ [param]: value }); expectValidationError( - 'INVALID_CVSSV4_SEVERITY', - 'Test is not an allowed input for the cvssV4Severity. Please use LOW, MEDIUM, HIGH, or CRITICAL.' + errorCode, + `${value} is not an allowed input for the ${param}. Please use ${allowedValues}.` ); }); - test('ValidateParams successfully accepts enumerator value high for cvssV4Severity.', () => { - validate({ - cvssV4Severity: 'high' - }); + test.each([ + ['cvssV2Severity', 'medium'], + ['cvssV3Severity', 'critical'], + ['cvssV3_0Severity', 'high'], + ['cvssV3_1Severity', 'critical'], + ['cvssV4Severity', 'high'] + ])('ValidateParams accepts a valid %s value.', (param, value) => { + validate({ [param]: value }); expectValidationSuccess(); }); - test('ValidateParams returns an error when a non integer value is passed for resultsPerPage.', () => { - validate({ - resultsPerPage: 'Test' - }); - - expectValidationError( - 'RESULTS_PER_PAGE_INT', - 'The resultsPerPage parameter must be an integer with a value greater than 0.' - ); - }); - - test('ValidateParams returns an error when a non-positive integer value is passed for resultsPerPage.', () => { - validate({ - resultsPerPage: -1 - }); - - expectValidationError( - 'RESULTS_PER_PAGE_INT', - 'The resultsPerPage parameter must be an integer with a value greater than 0.' - ); - }); - - test('ValidateParams returns an error when resultsPerPage is zero.', () => { - validate({ - resultsPerPage: 0 - }); + test.each([ + ['resultsPerPage', 'non-integer', 'Test', 'RESULTS_PER_PAGE_INT'], + ['resultsPerPage', 'zero', 0, 'RESULTS_PER_PAGE_INT'], + ['pageNumber', 'non-integer', 'Test', 'PAGE_NUMBER_INT'], + ['pageNumber', 'zero', 0, 'PAGE_NUMBER_INT'] + ])('ValidateParams rejects %s with a %s value.', (param, _caseName, value, errorCode) => { + validate({ [param]: value }); - expectValidationError( - 'RESULTS_PER_PAGE_INT', - 'The resultsPerPage parameter must be an integer with a value greater than 0.' - ); + expectValidationError(errorCode, `The ${param} parameter must be an integer with a value greater than 0.`); }); test('ValidateParams returns an error when resultsPerPage exceeds the max result size.', () => { @@ -728,39 +638,6 @@ describe('middleware validation function', () => { expectValidationError('MAX_RESULTS_LIMIT', 'The resultsPerPage must be an integer less than or equal to 500.'); }); - test('ValidateParams returns an error when a non integer value is passed for pageNumber.', () => { - validate({ - pageNumber: 'Test' - }); - - expectValidationError( - 'PAGE_NUMBER_INT', - 'The pageNumber parameter must be an integer with a value greater than 0.' - ); - }); - - test('ValidateParams returns an error when a non-positive integer value is passed for pageNumber.', () => { - validate({ - pageNumber: -1 - }); - - expectValidationError( - 'PAGE_NUMBER_INT', - 'The pageNumber parameter must be an integer with a value greater than 0.' - ); - }); - - test('ValidateParams returns an error when pageNumber is zero.', () => { - validate({ - pageNumber: 0 - }); - - expectValidationError( - 'PAGE_NUMBER_INT', - 'The pageNumber parameter must be an integer with a value greater than 0.' - ); - }); - test('ValidateParams accepts pagination at the maximum search window boundary.', () => { validate({ resultsPerPage: 500, diff --git a/routes/cpeName.test.js b/routes/cpeName.test.js deleted file mode 100644 index 4e0d82e..0000000 --- a/routes/cpeName.test.js +++ /dev/null @@ -1,62 +0,0 @@ -const { createRouterApp, request } = require('../test/http'); - -jest.mock('../controllers/searchController', () => ({ - SEARCH: jest.fn((req, res) => - res.status(200).json({ - route: 'search', - params: req.searchParams - }) - ), - WEB_SEARCH: jest.fn() -})); - -const controller = require('../controllers/searchController'); -const router = require('./index'); - -describe('POST /search cpeName route behavior', () => { - let app; - - beforeEach(() => { - app = createRouterApp(router); - jest.clearAllMocks(); - }); - - test('rejects cpeName with wildcard version before the controller runs', async () => { - const cpeName = 'cpe:2.3:a:artifex:ghostscript:*:*:*:*:*:*:*:*'; - const response = await request(app, { - method: 'POST', - path: '/search', - body: { - cpeName - } - }); - - expect(response.status).toBe(404); - expect(response.body).toStrictEqual({ - error: 'INVALID_CPE_NAME', - message: - 'The cpeName parameter must be a CPE 2.3 name with part a, h, or o and non-wildcard vendor, product, and version components.' - }); - expect(controller.SEARCH).not.toHaveBeenCalled(); - }); - - test('allows valid cpeName through to the controller', async () => { - const cpeName = 'cpe:2.3:a:artifex:ghostscript:10.05.1:*:*:*:*:*:*:*'; - const response = await request(app, { - method: 'POST', - path: '/search', - body: { - cpeName - } - }); - - expect(response.status).toBe(200); - expect(controller.SEARCH).toHaveBeenCalledTimes(1); - expect(response.body).toStrictEqual({ - route: 'search', - params: { - cpeName - } - }); - }); -}); diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index bc79a6d..68ab08c 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -1,5 +1,4 @@ const { request } = require('../test/http'); -const constants = require('../utils/constants'); const mockSearchOpenSearch = jest.fn(); const mockBasicSearch = jest.fn(); @@ -152,59 +151,6 @@ describe('search route integration', () => { expect(requestErrorLog.mock.calls[0][0]).not.toContain('sensitive'); }); - test('POST /search rejects invalid dates before invoking OpenSearch', async () => { - const response = await request(createApp(), { - method: 'POST', - path: '/search', - body: { - pubStartDate: '2024-02-30T00:00:00.000Z' - } - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'DATE_TIME_FORMAT', - message: "The pubStartDate parameter must match the following format: yyyy-MM-ddTHH:mm:ss.SSS'Z'" - }); - expect(mockSearchOpenSearch).not.toHaveBeenCalled(); - }); - - test('POST /search rejects reversed date ranges before invoking OpenSearch', async () => { - const response = await request(createApp(), { - method: 'POST', - path: '/search', - body: { - pubStartDate: '2024-02-01T00:00:00.000Z', - pubEndDate: '2024-01-01T00:00:00.000Z' - } - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'DATE_RANGE_ORDER', - message: 'The pubStartDate parameter must be less than or equal to pubEndDate.' - }); - expect(mockSearchOpenSearch).not.toHaveBeenCalled(); - }); - - test('POST /search rejects CVSS headers without metric tokens before invoking OpenSearch', async () => { - const response = await request(createApp(), { - method: 'POST', - path: '/search', - body: { - cvssV3Metrics: 'CVSS:3.1' - } - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'CVSS_METRICS_FORMAT', - message: - 'The cvssV3Metrics parameter must contain at least one unique KEY:VALUE metric token and use a compatible CVSS version header when a header is provided.' - }); - expect(mockSearchOpenSearch).not.toHaveBeenCalled(); - }); - test('POST /search rejects unsupported CPE parts before invoking OpenSearch', async () => { const response = await request(createApp(), { method: 'POST', @@ -223,47 +169,6 @@ describe('search route integration', () => { expect(mockSearchOpenSearch).not.toHaveBeenCalled(); }); - test('POST /search rejects excessive keyword terms before invoking OpenSearch', async () => { - const response = await request(createApp(), { - method: 'POST', - path: '/search', - body: { - keywordSearch: Array(constants.MAX_KEYWORD_SEARCH_TOKENS + 1) - .fill('term') - .join(' ') - } - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'SEARCH_FIELD_TOKEN_LIMIT', - message: `The keywordSearch field must contain ${constants.MAX_KEYWORD_SEARCH_TOKENS} or fewer terms.` - }); - expect(mockSearchOpenSearch).not.toHaveBeenCalled(); - }); - - test('POST /search rejects an inverted virtualMatchString version range before invoking OpenSearch', async () => { - const response = await request(createApp(), { - method: 'POST', - path: '/search', - body: { - virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', - versionStart: '3.0', - versionStartType: 'including', - versionEnd: '2.0', - versionEndType: 'including' - } - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'VERSION_RANGE_ORDER', - message: - 'The versionStart parameter must be less than versionEnd. Equal values require both bounds to be including.' - }); - expect(mockSearchOpenSearch).not.toHaveBeenCalled(); - }); - test('POST /webSearch normalizes body fields and formats BasicSearchManager responses', async () => { mockBasicSearch.mockResolvedValue(basicResponse({ total: 1, hits: [{ _id: 'CVE-2024-0002' }] })); From bf78e7f64e0ba81487953a527bce215a8bdbc718 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 12:54:54 -0400 Subject: [PATCH 28/33] docs: clarify semantic CPE result limits --- README.md | 4 +++- api-docs/openapi.json | 6 +++--- api-docs/openapi.test.js | 17 +++++++++++++++++ routes/swagger.js | 6 +++--- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8a9f7e0..5946ff0 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ Publication and last-modified date filters support one-sided ranges. Values must `pageSize` reports the number of CVE records actually returned in the current response. `page` is the requested one-based page, except a search with no matches reports page `0`. An accepted page beyond the available matches returns an empty `hits` array and page size `0`. -Pagination is limited to complete pages inside the first 10,000 results: `from + resultsPerPage` must not exceed 10,000. `totalPages` reports the smaller of the pages implied by the match count and the complete pages accessible inside that window. For example, a page size of 30 allows at most 333 pages. The match total itself is not reduced by this pagination calculation. Point-in-time search and deep pagination remain deferred. +Pagination is limited to complete pages inside the first 10,000 results: `from + resultsPerPage` must not exceed 10,000. `totalPages` reports the smaller of the pages implied by the match count and the complete pages accessible inside that window. For example, a page size of 30 allows at most 333 pages. Standard-search totals are exact and are not reduced by this pagination calculation. Semantic CPE searches have a separate 10,000-candidate evaluation limit described below, so their totals can be incomplete. Point-in-time search and deep pagination remain deferred. Use `cveIds` to request up to 100 CVE records by comma-separated CVE ID. Each ID must contain the `CVE` prefix, a four-digit year, and a sequence number containing 4 to 19 digits. The prefix is accepted without regard to case and normalized to uppercase for the OpenSearch query. @@ -370,6 +370,8 @@ This is a process-wide safety limit, not a precise distributed quota. Each ECS t Searches that use `cpeName`, `virtualMatchString`, or `virtualMatchString` with version range parameters use a semantic CPE post-filter. The API first asks OpenSearch for up to 10,000 lightweight candidate records, filters those candidates in Node.js, then fetches the requested page of full CVE records by CVE ID. +The 10,000-candidate fetch is a hard evaluation boundary, not only a pagination limit. If a broad semantic CPE query produces more than 10,000 OpenSearch candidates, candidates beyond that boundary are not evaluated. The response can therefore omit later applicable CVE records, and its total reports only matches found within the evaluated candidate set. Add other filters or use a more specific CPE value where possible. Evaluating the complete candidate set requires a future point-in-time and `search_after` implementation. + The post-filter cache stores only the filtered CVE ID list and candidate count for that expensive candidate query. It does not cache full CVE records. On a cache hit, the API skips the 10,000-candidate OpenSearch query and semantic post-filtering, slices the cached CVE IDs for the requested page, and fetches that page of full CVE records by ID. Candidate retrieval asks OpenSearch for exact-version or wildcard-version criteria and leaves range applicability decisions to the semantic post-filter. The full-record page fetch uses an exact CVE ID set without total-hit counting or OpenSearch sorting; the API requires one unique full record for every requested CVE ID and restores the requested CVE ID order locally. Missing, duplicate, malformed, or unexpected page records fail closed with a sanitized HTTP 502 response and evict the associated semantic candidate cache entry so the next request must repeat candidate discovery. Standard searches retain exact total counting and deterministic sorting. diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 8a0e900..2bb9200 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -499,7 +499,7 @@ "type": "string", "maxLength": 2048, "example": "cpe:2.3:a:freetype:freetype:2.8.1:*:*:*:*:*:*:*", - "description": "Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches." + "description": "Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Semantic evaluation is limited to the first 10000 OpenSearch candidates, so broad searches may omit later matches and report incomplete totals. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches." }, "isVulnerable": { "type": "boolean", @@ -581,7 +581,7 @@ "type": "string", "maxLength": 2048, "example": "cpe:2.3:o:microsoft:windows_10_1809", - "description": "Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component." + "description": "Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component. Semantic evaluation, including version range searches, is limited to the first 10000 OpenSearch candidates, so broad searches may omit later matches and report incomplete totals." } } }, @@ -594,7 +594,7 @@ "value": { "type": "integer", "minimum": 0, - "description": "Number of records matched by the active search strategy." + "description": "Number of records matched by the active search strategy. Standard-search totals are exact. For semantic cpeName and virtualMatchString searches, this value counts matches found within the first 10000 OpenSearch candidates and may be incomplete." }, "relation": { "type": "string", diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index e51e77a..66115a7 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -184,6 +184,23 @@ describe('OpenAPI contract', () => { expect(openapi.paths['/search'].post.responses['429'].description).toContain('waiter limit'); }); + test('documents the bounded semantic CPE candidate window', () => { + const searchProperties = openapi.components.schemas.SearchRequest.properties; + const totalDescription = openapi.components.schemas.SearchHitTotal.properties.value.description; + + ['cpeName', 'virtualMatchString'].forEach((field) => { + expect(searchProperties[field].description).toContain( + `first ${constants.MAX_SEARCH_WINDOW} OpenSearch candidates` + ); + expect(searchProperties[field].description).toContain('may omit later matches'); + expect(searchProperties[field].description).toContain('incomplete totals'); + }); + expect(searchProperties.virtualMatchString.description).toContain('including version range searches'); + expect(totalDescription).toContain('Standard-search totals are exact'); + expect(totalDescription).toContain(`first ${constants.MAX_SEARCH_WINDOW} OpenSearch candidates`); + expect(totalDescription).toContain('may be incomplete'); + }); + test('describes POST /search as a paginated result operation', () => { const operation = openapi.paths['/search'].post; diff --git a/routes/swagger.js b/routes/swagger.js index aba5b68..2982644 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -126,7 +126,7 @@ const searchRequestProperties = { } ), cpeName: searchRequestProperty( - "Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches.", + `Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Semantic evaluation is limited to the first ${constants.MAX_SEARCH_WINDOW} OpenSearch candidates, so broad searches may omit later matches and report incomplete totals. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches.`, { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cpeName, @@ -226,7 +226,7 @@ const searchRequestProperties = { } ), virtualMatchString: searchRequestProperty( - "Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component.", + `Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component. Semantic evaluation, including version range searches, is limited to the first ${constants.MAX_SEARCH_WINDOW} OpenSearch candidates, so broad searches may omit later matches and report incomplete totals.`, { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.virtualMatchString, @@ -243,7 +243,7 @@ const searchResponseSchemas = { value: { type: 'integer', minimum: 0, - description: 'Number of records matched by the active search strategy.' + description: `Number of records matched by the active search strategy. Standard-search totals are exact. For semantic cpeName and virtualMatchString searches, this value counts matches found within the first ${constants.MAX_SEARCH_WINDOW} OpenSearch candidates and may be incomplete.` }, relation: { type: 'string', From a43949acc93e72b1195badfc9d48a08ac9859d04 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 13:37:01 -0400 Subject: [PATCH 29/33] test: streamline search controller coverage --- controllers/searchController.test.js | 777 +------------------------ utils/searchCandidateProcessor.test.js | 9 + 2 files changed, 35 insertions(+), 751 deletions(-) diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index b5c4ec7..8b68cd0 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -57,7 +57,6 @@ jest.mock('../utils/searchProvider', () => ({ })); const controller = require('./searchController.js'); -const searchCandidateProcessor = require('../utils/searchCandidateProcessor'); const searchInFlightRegistry = require('../utils/searchInFlightRegistry'); const searchPerformance = require('../utils/searchPerformance'); const searchPostFilterCache = require('../utils/searchPostFilterCache'); @@ -88,23 +87,6 @@ function rawResponse(totalValue, hits = []) { return advancedResponse(totalValue, hits).data; } -function cvssRegexp(field, value) { - return { - regexp: { - [`${field}.keyword`]: { - value, - case_insensitive: true, - flags: 'INTERSECTION|ANYSTRING', - rewrite: 'constant_score' - } - } - }; -} - -function cvssValue(...tokens) { - return tokens.map((token) => `(@/)?${token}(/@)?`).join('&'); -} - function basicResponse({ ok = true, status = 'ok', total = 0, hits = [], notes = [], errors = [] } = {}) { return { status, @@ -231,19 +213,13 @@ describe('Search controller SEARCH handler', () => { jest.restoreAllMocks(); }); - test('SEARCH builds filters, query strings, date ranges, and pagination options', async () => { + test('SEARCH executes a standard search and formats pagination metadata', async () => { const resp = advancedResponse(25, [{ _id: 'CVE-2024-0001' }]); mockApiSearch.mockResolvedValue(resp); await controller.SEARCH( { searchParams: { - keywordSearch: 'remote code', - cveIds: 'CVE-2024-0001', - pubStartDate: '2024-01-01T00:00:00.000Z', - pubEndDate: '2024-01-31T00:00:00.000Z', - lastModStartDate: '2024-02-01T00:00:00.000Z', - lastModEndDate: '2024-02-29T00:00:00.000Z', resultsPerPage: 10, pageNumber: '3' } @@ -252,60 +228,11 @@ describe('Search controller SEARCH handler', () => { next ); - expect(mockApiSearch).toHaveBeenCalledWith( - { - query: { - bool: { - must: [ - { - simple_query_string: { - query: 'remote* code*', - default_operator: 'AND', - fields: ['containers.cna.descriptions.value'] - } - } - ], - filter: [ - { - terms: { - 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] - } - }, - { - range: { - 'cveMetadata.datePublished': { - gte: '2024-01-01T00:00:00.000Z', - lte: '2024-01-31T00:00:00.000Z' - } - } - }, - { - range: { - 'cveMetadata.dateUpdated': { - gte: '2024-02-01T00:00:00.000Z', - lte: '2024-02-29T00:00:00.000Z' - } - } - } - ] - } - }, - sort: [ - { - _score: { - order: 'desc' - } - }, - { - 'cveMetadata.cveId.keyword': { - order: 'asc' - } - } - ], - track_total_hits: true, + expect(mockSearchOpenSearch).toHaveBeenCalledWith( + expect.objectContaining({ size: 10, from: 20 - }, + }), expect.objectContaining({ operation: 'standard', signal: expect.any(AbortSignal), @@ -322,227 +249,6 @@ describe('Search controller SEARCH handler', () => { }); }); - test('SEARCH sends keywordSearch through match_phrase when keywordExactMatch is true', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(1)); - - await controller.SEARCH( - { - searchParams: { - keywordSearch: 'remote code', - keywordExactMatch: 'true' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool).toStrictEqual({ - must: [], - filter: [ - { - match_phrase: { - 'containers.cna.descriptions.value': 'remote code' - } - } - ] - }); - }); - - test('SEARCH builds an exact terms filter for cveIds', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(2)); - - await controller.SEARCH( - { - searchParams: { - cveIds: 'CVE-2024-0001,CVE-2024-0002' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - terms: { - 'cveMetadata.cveId.keyword': ['CVE-2024-0001', 'CVE-2024-0002'] - } - } - ]); - expect(mockApiSearch.mock.calls[0][0].query.bool.must).toStrictEqual([]); - }); - - test('SEARCH builds rejected-state exclusions for noRejected', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(2)); - - await controller.SEARCH( - { - searchParams: { - noRejected: true - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - bool: { - must_not: [ - { - terms: { - 'cveMetadata.state.keyword': ['REJECT', 'REJECTED', 'Rejected'] - } - }, - { - terms: { - 'containers.cna.x_legacyV4Record.CVE_data_meta.STATE.keyword': [ - 'REJECT', - 'REJECTED', - 'Rejected' - ] - } - } - ] - } - } - ]); - expect(mockApiSearch.mock.calls[0][0].query.bool.must).toStrictEqual([]); - }); - - test('SEARCH processes virtualMatchString into a structured CPE criteria filter', async () => { - mockSearchRaw.mockResolvedValue(rawResponse(0)); - - await controller.SEARCH( - { - searchParams: { - virtualMatchString: 'cpe:2.3:a:microsoft:windows:*:*:*:*:*:*:*:*' - } - }, - res, - next - ); - - expect(mockSearchRaw.mock.calls[0][0]._source).toStrictEqual([ - 'cveMetadata.cveId', - 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria', - 'containers.cna.cpeApplicability.nodes.cpeMatch.vulnerable', - 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartIncluding', - 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartExcluding', - 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndIncluding', - 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndExcluding' - ]); - expect(mockSearchRaw.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - bool: { - should: [ - 'cpe:2.3:a:microsoft:windows:', - 'cpe:2.3:a:microsoft:*:', - 'cpe:2.3:a:*:windows:', - 'cpe:2.3:a:*:*:', - 'cpe:2.3:*:microsoft:windows:', - 'cpe:2.3:*:microsoft:*:', - 'cpe:2.3:*:*:windows:', - 'cpe:2.3:*:*:*:' - ].map((value) => ({ - prefix: { - 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword': { - value, - case_insensitive: true - } - } - })), - minimum_should_match: 1 - } - } - ]); - expect(mockApiSearch).not.toHaveBeenCalled(); - }); - - test('SEARCH builds filters for CWE, CPE Name criteria, and vulnerable CPE parameters', async () => { - mockSearchRaw.mockResolvedValue(rawResponse(0)); - - await controller.SEARCH( - { - searchParams: { - cweId: 'CWE-79', - cpeName: 'cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*', - isVulnerable: 'true' - } - }, - res, - next - ); - - const body = mockSearchRaw.mock.calls[0][0]; - const filters = body.query.bool.filter; - - expect(filters).toHaveLength(3); - expect(filters[0]).toStrictEqual({ - match_phrase: { - 'containers.cna.problemTypes.descriptions.cweId': 'CWE-79' - } - }); - expect(filters[1].bool.minimum_should_match).toBe(1); - expect(filters[1].bool.should[0]).toStrictEqual({ - bool: { - should: [ - 'cpe:2.3:a:vendor:product:1.0:', - 'cpe:2.3:a:vendor:*:1.0:', - 'cpe:2.3:a:*:product:1.0:', - 'cpe:2.3:a:*:*:1.0:', - 'cpe:2.3:*:vendor:product:1.0:', - 'cpe:2.3:*:vendor:*:1.0:', - 'cpe:2.3:*:*:product:1.0:', - 'cpe:2.3:*:*:*:1.0:' - ].map((value) => ({ - prefix: { - 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword': { - value, - case_insensitive: true - } - } - })), - minimum_should_match: 1 - } - }); - expect(filters[2]).toStrictEqual({ - term: { - 'containers.cna.cpeApplicability.nodes.cpeMatch.vulnerable': true - } - }); - expect(body.size).toBe(10000); - expect(body.from).toBe(0); - expect(mockApiSearch).not.toHaveBeenCalled(); - }); - - test('SEARCH requires the matching CPE entry itself to be vulnerable', async () => { - const matchingNotVulnerable = cpeHit('CVE-2024-0001', { - criteria: 'cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*', - vulnerable: false - }); - matchingNotVulnerable._source.containers.cna.cpeApplicability.nodes[0].cpeMatch.push({ - criteria: 'cpe:2.3:a:other:product:1.0:*:*:*:*:*:*:*', - vulnerable: true - }); - mockSearchRaw.mockResolvedValue(rawResponse(1, [matchingNotVulnerable])); - - await controller.SEARCH( - { - searchParams: { - cpeName: 'cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*', - isVulnerable: true - } - }, - res, - next - ); - - expect(res.status).toHaveBeenCalledWith(200); - expect(res.json.mock.calls[0][0].data.hits.total.value).toBe(0); - expect(res.json.mock.calls[0][0].data.hits.hits).toStrictEqual([]); - expect(mockApiSearch).not.toHaveBeenCalled(); - }); - test('SEARCH applies semantic CPE version filtering before building response metadata', async () => { const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', @@ -582,29 +288,32 @@ describe('Search controller SEARCH handler', () => { next ); - expect(mockApiSearch).toHaveBeenCalledWith( - { - query: { - bool: { - must: [], - filter: [ - { - terms: { - 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] - } - } - ] - } - }, - track_total_hits: false, - size: 1, + expect(mockSearchOpenSearch).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + size: 10000, from: 0 - }, + }), + expect.objectContaining({ + operation: 'cpe-candidate', + signal: expect.any(AbortSignal) + }) + ); + expect(mockSearchOpenSearch).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + size: 1, + from: 0, + track_total_hits: false + }), expect.objectContaining({ operation: 'page-fetch', signal: expect.any(AbortSignal) }) ); + expect(mockApiSearch.mock.calls[0][0].query.bool.filter[0].terms).toStrictEqual({ + 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] + }); expect(res.status).toHaveBeenCalledWith(200); expect(res.json).toHaveBeenCalledWith({ page: 1, @@ -622,169 +331,6 @@ describe('Search controller SEARCH handler', () => { }); }); - test('chunked filtering rejects malformed candidate hit collections', async () => { - await expect( - searchCandidateProcessor.filterSearchHitsInChunks( - { - data: { - hits: {} - } - }, - {}, - undefined, - 100 - ) - ).rejects.toMatchObject({ - code: 'OPEN_SEARCH_RESPONSE_INVALID', - failureType: 'malformed_response' - }); - }); - - test('chunked filtering does not combine platform and version matches from different CPE entries', async () => { - const candidate = cpeHit('CVE-2026-0250', { - criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:macos:*:*:*:*:*', - vulnerable: true, - versionStartIncluding: '6.0', - versionEndExcluding: '6.1' - }); - candidate._source.containers.cna.cpeApplicability.nodes[0].cpeMatch.push( - { - criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:macos:*:*:*:*:*', - vulnerable: true, - versionStartIncluding: '6.2', - versionEndExcluding: '6.2.7' - }, - { - criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:android:*:*:*:*:*', - vulnerable: true, - versionStartIncluding: '6.1', - versionEndExcluding: '6.1.13' - }, - { - criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:chrome_os:*:*:*:*:*', - vulnerable: true, - versionStartIncluding: '6.1', - versionEndExcluding: '6.1.13' - } - ); - - const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(1, [candidate]), - { - cpeName: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:6.1.1:*:macos:*:*:*:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - - expect(filteredHits).toStrictEqual([]); - }); - - test('chunked filtering matches CPE string literals without lexical case', async () => { - const candidate = cpeHit('CVE-2026-0251', { - criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:*:*:*:macOS:*:*', - vulnerable: true, - versionStartIncluding: '6.0', - versionEndExcluding: '6.1' - }); - - const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(1, [candidate]), - { - cpeName: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:6.0.1:*:*:*:*:macos:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - - expect(filteredHits).toStrictEqual([candidate]); - }); - - test('chunked filtering preserves exact version padding while normalizing numeric range segments', async () => { - const exactCandidate = cpeHit('CVE-2025-9392', { - criteria: 'cpe:2.3:o:linksys:re6250_firmware:1.0.04.001:*:*:*:*:*:*:*', - vulnerable: true - }); - const rangeCandidate = cpeHit('CVE-2026-33280', { - criteria: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:*:*:*:*:*:*:*:*', - vulnerable: true, - versionEndExcluding: '6.02' - }); - - const exactResult = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(1, [exactCandidate]), - { - cpeName: 'cpe:2.3:o:linksys:re6250_firmware:1.0.4.1:*:*:*:*:*:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - const includedRangeResult = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(1, [rangeCandidate]), - { - cpeName: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:6.1:*:*:*:*:*:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - const excludedRangeResult = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(1, [rangeCandidate]), - { - cpeName: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:6.2:*:*:*:*:*:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - - expect(exactResult.filteredHits).toStrictEqual([]); - expect(includedRangeResult.filteredHits).toStrictEqual([rangeCandidate]); - expect(excludedRangeResult.filteredHits).toStrictEqual([]); - }); - - test('chunked filtering preserves CPE NA matches across numeric version ranges', async () => { - const criteria = 'cpe:2.3:a:microsoft:edge:*:*:*:*:*:android:*:*'; - const candidates = [ - cpeHit('CVE-2020-17153', { - criteria, - vulnerable: true, - versionStartIncluding: '-' - }), - cpeHit('CVE-2024-38208', { - criteria, - vulnerable: true, - versionStartIncluding: '1.0.0', - versionEndExcluding: '128.0.2739.42' - }), - cpeHit('CVE-2026-35429', { - criteria, - vulnerable: true, - versionStartIncluding: '1.0.0', - versionEndExcluding: '148.0.3967.55' - }) - ]; - - const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(candidates.length, candidates), - { - cpeName: 'cpe:2.3:a:microsoft:edge:-:*:*:*:*:android:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - - expect(filteredHits.map((hit) => hit._id)).toStrictEqual([ - 'CVE-2020-17153', - 'CVE-2024-38208', - 'CVE-2026-35429' - ]); - }); - test('SEARCH applies semantic virtualMatchString version range filtering before building response metadata', async () => { const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' @@ -825,21 +371,9 @@ describe('Search controller SEARCH handler', () => { next ); - expect(mockSearchRaw.mock.calls[0][0].size).toBe(10000); - expect(mockSearchRaw.mock.calls[0][0].from).toBe(0); - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - terms: { - 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] - } - } - ]); - expect(mockApiSearch.mock.calls[0][0]).toMatchObject({ - track_total_hits: false, - size: 1, - from: 0 + expect(mockApiSearch.mock.calls[0][0].query.bool.filter[0].terms).toStrictEqual({ + 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] }); - expect(mockApiSearch.mock.calls[0][0].sort).toBeUndefined(); expect(mockApiSearch.mock.calls[0][1]).toEqual( expect.objectContaining({ operation: 'page-fetch', @@ -1144,85 +678,6 @@ describe('Search controller SEARCH handler', () => { expect(secondRes.status).toHaveBeenCalledWith(200); }); - test('SEARCH applies semantic virtualMatchString filtering before building response metadata', async () => { - const applicable = cpeHit('CVE-2024-0001', { - criteria: 'cpe:2.3:o:microsoft:windows_10_1809:-:*:*:*:*:*:x86:*' - }); - const notApplicable = cpeHit('CVE-2024-0002', { - criteria: 'cpe:2.3:o:microsoft:windows_11_23h2:-:*:*:*:*:*:x86:*' - }); - const fullApplicable = { - _id: 'CVE-2024-0001', - _source: { - cveMetadata: { - cveId: 'CVE-2024-0001' - }, - containers: { - cna: { - title: 'full record' - } - } - } - }; - - mockSearchRaw.mockResolvedValue(rawResponse(2, [applicable, notApplicable])); - mockApiSearch.mockResolvedValue(advancedResponse(1, [fullApplicable])); - - await controller.SEARCH( - { - searchParams: { - virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809', - resultsPerPage: '1', - pageNumber: '1' - } - }, - res, - next - ); - - expect(mockSearchRaw.mock.calls[0][0].size).toBe(10000); - expect(mockSearchRaw.mock.calls[0][0].from).toBe(0); - expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ - page: 1, - pageSize: 1, - totalPages: 1, - status: 'ok', - data: { - hits: { - total: { - value: 1 - }, - hits: [fullApplicable] - } - } - }); - }); - - test('chunked semantic filtering stops after cancellation between batches', async () => { - const hits = [ - cpeHit('CVE-2024-0001', { - criteria: 'cpe:2.3:o:microsoft:windows_10_1809:-:*:*:*:*:*:x86:*' - }), - cpeHit('CVE-2024-0002', { - criteria: 'cpe:2.3:o:microsoft:windows_10_1809:-:*:*:*:*:*:x86:*' - }) - ]; - const abortController = new AbortController(); - const filtering = searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(2, hits), - { - virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809' - }, - abortController.signal, - 1 - ); - - abortController.abort(new Error('cancelled')); - - await expect(filtering).rejects.toThrow('cancelled'); - }); - test('SEARCH returns 429 when the cold semantic post-filter queue is full', async () => { const candidateSearch = deferred(); const firstRes = createResponse(); @@ -1525,186 +980,6 @@ describe('Search controller SEARCH handler', () => { expect(console.log.mock.calls[0][0]).not.toContain('remote code'); }); - test('SEARCH builds version-specific CVSS v3 severity filters', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(1)); - - await controller.SEARCH( - { - searchParams: { - cvssV3_0Severity: 'high', - cvssV3_1Severity: 'critical' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - term: { - 'containers.cna.metrics.cvssV3_0.baseSeverity.keyword': 'HIGH' - } - }, - { - term: { - 'containers.cna.metrics.cvssV3_1.baseSeverity.keyword': 'CRITICAL' - } - } - ]); - }); - - test('SEARCH builds cvssV2 metric filters', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(1)); - - await controller.SEARCH( - { - searchParams: { - cvssV2Metrics: 'AV:N/AC:L/Au:N' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - cvssRegexp('containers.cna.metrics.cvssV2_0.vectorString', cvssValue('AC:L', 'Au:N', 'AV:N')) - ]); - }); - - test('SEARCH builds version-specific CVSS v3 metric filters', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(1)); - - await controller.SEARCH( - { - searchParams: { - cvssV3_0Metrics: 'CVSS:3.0/AV:N/AC:L', - cvssV3_1Metrics: 'CVSS:3.1/PR:N/UI:R' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - cvssRegexp('containers.cna.metrics.cvssV3_0.vectorString', cvssValue('AC:L', 'AV:N')), - cvssRegexp('containers.cna.metrics.cvssV3_1.vectorString', cvssValue('PR:N', 'UI:R')) - ]); - }); - - test('SEARCH escapes Lucene reserved characters in keywordSearch terms', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(1)); - - await controller.SEARCH( - { - searchParams: { - keywordSearch: 'c++ path:/tmp (test)' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.must).toStrictEqual([ - { - simple_query_string: { - query: 'c\\+\\+* path\\:\\/tmp* \\(test\\)*', - default_operator: 'AND', - fields: ['containers.cna.descriptions.value'] - } - } - ]); - }); - - test('SEARCH keeps whitespace-only keywordSearch as an empty simple query string', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(0)); - - await controller.SEARCH( - { - searchParams: { - keywordSearch: ' ' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.must).toStrictEqual([ - { - simple_query_string: { - query: '', - default_operator: 'AND', - fields: ['containers.cna.descriptions.value'] - } - } - ]); - }); - - test('SEARCH preserves escaped colons inside virtualMatchString segments', async () => { - mockSearchRaw.mockResolvedValue(rawResponse(0)); - - await controller.SEARCH( - { - searchParams: { - virtualMatchString: 'cpe:2.3:a:vendor:product\\:edition:*:*:*:*:*:*:*:*' - } - }, - res, - next - ); - - expect(mockSearchRaw.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - bool: { - should: [ - 'cpe:2.3:a:vendor:product\\:edition:', - 'cpe:2.3:a:vendor:*:', - 'cpe:2.3:a:*:product\\:edition:', - 'cpe:2.3:a:*:*:', - 'cpe:2.3:*:vendor:product\\:edition:', - 'cpe:2.3:*:vendor:*:', - 'cpe:2.3:*:*:product\\:edition:', - 'cpe:2.3:*:*:*:' - ].map((value) => ({ - prefix: { - 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword': { - value, - case_insensitive: true - } - } - })), - minimum_should_match: 1 - } - } - ]); - expect(mockApiSearch).not.toHaveBeenCalled(); - }); - - test('SEARCH builds a broad CPE filter for wildcard-only virtualMatchString segments', async () => { - mockSearchRaw.mockResolvedValue(rawResponse(0)); - - await controller.SEARCH( - { - searchParams: { - virtualMatchString: 'cpe:2.3:*:*:*:*:*:*:*:*:*:*:*' - } - }, - res, - next - ); - - expect(mockSearchRaw.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - regexp: { - 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword': { - value: 'cpe:2\\.3:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*', - case_insensitive: true - } - } - } - ]); - expect(mockApiSearch).not.toHaveBeenCalled(); - }); - test('SEARCH returns zeroed pagination metadata when OpenSearch returns zero hits', async () => { const resp = advancedResponse(0); mockApiSearch.mockResolvedValue(resp); diff --git a/utils/searchCandidateProcessor.test.js b/utils/searchCandidateProcessor.test.js index 9e277f5..d3bd3d3 100644 --- a/utils/searchCandidateProcessor.test.js +++ b/utils/searchCandidateProcessor.test.js @@ -69,6 +69,15 @@ describe('searchCandidateProcessor', () => { jest.restoreAllMocks(); }); + test('rejects a malformed OpenSearch hits response', async () => { + await expect( + searchCandidateProcessor.filterSearchHitsInChunks({ data: { hits: {} } }, {}) + ).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_response' + }); + }); + test('validates and filters candidates in the same processing pass', async () => { const matching = cpeCandidateHit({ versionStartIncluding: '2.9', From 757f826b5b0e9b53cb9a4d465506f59d480417f9 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 13:56:45 -0400 Subject: [PATCH 30/33] test: consolidate delivery policy coverage --- config/runtime-config.test.js | 40 ----------------------------------- test/delivery-policy.test.js | 40 ++++++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index 7f24e84..77486cf 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -79,46 +79,6 @@ describe('runtime container and compose configuration', () => { expect(dockerignore).toContain('**/reports'); }); - test('CI builds, scans, and smoke tests the production runtime image', () => { - const workflow = readProjectFile('.github', 'workflows', 'quality.yml'); - - expect(workflow).toContain('pull_request:'); - expect(workflow).toContain('push:'); - expect(workflow).toContain('merge_group:'); - expect(workflow).toContain('name: Dependency Review'); - expect(workflow).toContain('actions/dependency-review-action@'); - expect(workflow).toContain('name: Runtime Container'); - expect(workflow).toContain('docker build --tag cve-search-api:pr .'); - expect(workflow).toContain('aquasecurity/trivy-action@'); - expect(workflow).toContain('trivyignores: .trivyignore'); - expect(workflow).toContain('version: v0.69.3'); - expect(workflow).toContain('http://127.0.0.1:4100/api/health-check'); - expect(workflow).toContain('http://127.0.0.1:4100/api-docs/openapi.json'); - expect(workflow).toContain(`--format '{{.Config.User}}'`); - expect(workflow).toContain(`--format '{{json .Config.Cmd}}'`); - expect(workflow).not.toContain('/api/ready-check'); - }); - - test('manual OpenSearch contract workflow uses a protected environment and secrets', () => { - const workflow = readProjectFile('.github', 'workflows', 'opensearch-contract.yml'); - - expect(workflow).toContain('workflow_dispatch:'); - expect(workflow).toContain('type: environment'); - expect(workflow).toContain('environment: ${{ inputs.environment }}'); - expect(workflow).toContain('${{ secrets.OPENSEARCH_DOMAIN_ENDPOINT }}'); - expect(workflow).toContain('${{ secrets.OPENSEARCH_CVE_INDEX }}'); - expect(workflow).toContain('npm run test:opensearch-contract'); - expect(workflow).not.toMatch(/https?:\/\//); - }); - - test('Dependabot tracks npm, Docker, and GitHub Actions dependencies', () => { - const dependabot = readProjectFile('.github', 'dependabot.yml'); - - expect(dependabot).toContain('package-ecosystem: npm'); - expect(dependabot).toContain('package-ecosystem: docker'); - expect(dependabot).toContain('package-ecosystem: github-actions'); - }); - test('server-only compose file joins the existing OpenSearch 3.5 network', () => { const compose = readProjectFile('docker-compose.yaml'); diff --git a/test/delivery-policy.test.js b/test/delivery-policy.test.js index 65af699..3986ed6 100644 --- a/test/delivery-policy.test.js +++ b/test/delivery-policy.test.js @@ -25,22 +25,54 @@ describe('delivery security policy', () => { }); test('exposes OpenSearch secrets only to the trusted contract execution step', () => { - const workflow = readWorkflow('.github/workflows/opensearch-contract.yml'); + const workflowPath = '.github/workflows/opensearch-contract.yml'; + const workflowSource = readRepositoryFile(workflowPath); + const workflow = readWorkflow(workflowPath); const job = workflow.jobs.contract; const installStep = job.steps.find(({ name }) => name === 'Install production dependencies'); const contractStep = job.steps.find(({ name }) => name === 'Run read-only contract checks'); + expect(workflow.on.workflow_dispatch.inputs.environment).toMatchObject({ + required: true, + type: 'environment' + }); expect(job.if).toBe("github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main'"); + expect(job.environment).toBe('${{ inputs.environment }}'); expect(job.env).toStrictEqual({ NODE_ENV: 'production', NODE_CONFIG_ENV: 'prod' }); expect(installStep.env).toBeUndefined(); + expect(contractStep.run).toBe('npm run test:opensearch-contract'); expect(contractStep.env).toStrictEqual({ OpenSearchDomainEndpoint: '${{ secrets.OPENSEARCH_DOMAIN_ENDPOINT }}', OpenSearchCveIndex: '${{ secrets.OPENSEARCH_CVE_INDEX }}', OpenSearchAllowUnknownSslCerts: "${{ vars.OPENSEARCH_ALLOW_UNKNOWN_SSL_CERTS || 'false' }}" }); + expect(workflowSource).not.toMatch(/https?:\/\//); + }); + + test('runs repository quality and production image validation for supported events', () => { + const workflow = readWorkflow('.github/workflows/quality.yml'); + const dependencyReviewStep = workflow.jobs['dependency-review'].steps.find( + ({ name }) => name === 'Review dependency changes' + ); + const qualityStep = workflow.jobs.quality.steps.find(({ name }) => name === 'Run quality checks'); + const containerJob = workflow.jobs.container; + const buildStep = containerJob.steps.find(({ name }) => name === 'Build production image'); + const validationStep = containerJob.steps.find(({ name }) => name === 'Validate runtime image'); + + expect(Object.keys(workflow.on)).toStrictEqual(['pull_request', 'push', 'merge_group']); + expect(workflow.on.push.branches).toStrictEqual(['dev', 'main']); + expect(dependencyReviewStep.uses).toMatch(/^actions\/dependency-review-action@[a-f0-9]{40}$/); + expect(qualityStep.run).toBe('npm run quality'); + expect(containerJob.name).toBe('Runtime Container'); + expect(buildStep.run).toBe('docker build --tag cve-search-api:pr .'); + expect(validationStep.run).toContain('http://127.0.0.1:4100/api/health-check'); + expect(validationStep.run).toContain('http://127.0.0.1:4100/api-docs/openapi.json'); + expect(validationStep.run).toContain(`--format '{{.Config.User}}'`); + expect(validationStep.run).toContain(`--format '{{json .Config.Cmd}}'`); + expect(validationStep.run).not.toContain('/api/ready-check'); }); test('reports all severe image findings before enforcing the fixable vulnerability gate', () => { @@ -61,13 +93,15 @@ describe('delivery security policy', () => { 'exit-code': 0, 'ignore-unfixed': false, severity: 'CRITICAL,HIGH', - trivyignores: '/dev/null' + trivyignores: '/dev/null', + version: 'v0.69.3' } }); expect(steps[gateIndex].with).toMatchObject({ 'exit-code': 1, 'ignore-unfixed': true, - trivyignores: '.trivyignore' + trivyignores: '.trivyignore', + version: 'v0.69.3' }); }); From beee7c06dee2f68d767e74364e6d6c3690bbe2cc Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 14:00:38 -0400 Subject: [PATCH 31/33] test: organize OpenAPI contract coverage --- api-docs/openapi.test.js | 68 +++++++++++++++------------------------- 1 file changed, 26 insertions(+), 42 deletions(-) diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 66115a7..8ed149b 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -6,9 +6,7 @@ const packageInfo = require('../package.json'); const constants = require('../utils/constants'); const { generateOpenApi } = require('../routes/swagger'); -function jsonRequestBodySchema(path, method) { - const schema = openapi.paths[path][method].requestBody.content['application/json'].schema; - +function resolveSchema(schema) { if (!schema.$ref) return schema; return schema.$ref @@ -17,13 +15,6 @@ function jsonRequestBodySchema(path, method) { .reduce((value, key) => value[key], openapi); } -function resolveSchemaRef(ref) { - return ref - .replace('#/', '') - .split('/') - .reduce((value, key) => value[key], openapi); -} - describe('OpenAPI contract', () => { test('documents the supported route surface', () => { expect(openapi.paths['/search'].post.operationId).toBe('search'); @@ -62,8 +53,8 @@ describe('OpenAPI contract', () => { test('POST /search documents all JSON body fields accepted by the router and controller', () => { const operation = openapi.paths['/search'].post; - const schema = jsonRequestBodySchema('/search', 'post'); const mediaType = operation.requestBody.content['application/json']; + const schema = resolveSchema(mediaType.schema); expect(operation.parameters || []).toStrictEqual([]); expect(operation.requestBody.description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`); @@ -89,18 +80,6 @@ describe('OpenAPI contract', () => { }); expect(schema.properties.noRejected.type).toBe('boolean'); expect(schema.properties.noRejected.description).toContain('When true'); - expect(schema.properties.isVulnerable.description).toContain('same CPE Match Criteria entry'); - expect(schema.properties.isVulnerable.description).toContain('False or omitted'); - expect(schema.properties.cpeName.description).toContain('Maven-style ordering'); - expect(schema.properties.cpeName.description).toContain('part must be a, h, or o'); - expect(schema.properties.cpeName.description).toContain( - 'Every concrete non-version component must be compatible with the same CPE Match Criteria entry' - ); - expect(schema.properties.cpeName.description).toContain('optional * components remain broad matches'); - expect(schema.properties.cpeName.description).toContain('insensitive to lexical case'); - expect(schema.properties.cpeName.description).toContain('exact CPE Match Criteria preserve leading zeros'); - expect(schema.properties.cpeName.description).toContain('entries independently'); - expect(schema.properties.cpeName.description).toContain("does not evaluate a node's operator or negate value"); expect(schema.properties.versionStartType.enum).toStrictEqual(constants.CPE_VERSION_RANGE_TYPES); expect(schema.properties.versionEndType.enum).toStrictEqual(constants.CPE_VERSION_RANGE_TYPES); expect(schema.properties.resultsPerPage.maximum).toBe(constants.MAX_RESULTS_LIMIT); @@ -115,19 +94,6 @@ describe('OpenAPI contract', () => { expect(schema.properties.versionEnd.description).toContain( 'prerelease qualifiers before an unqualified release' ); - expect(schema.properties.virtualMatchString.description).toContain('CPE Match Criteria'); - expect(schema.properties.virtualMatchString.description).toContain('compatible'); - expect(schema.properties.virtualMatchString.description).toContain('insensitive to lexical case'); - expect(schema.properties.virtualMatchString.description).toContain('without a version component'); - expect(schema.properties.virtualMatchString.description).toContain('part must be a, h, o, *, or -'); - expect(schema.properties.virtualMatchString.description).toContain( - 'explicit empty colon-delimited components are invalid' - ); - expect(schema.properties.virtualMatchString.description).toContain('entries independently'); - expect(schema.properties.virtualMatchString.description).toContain( - "does not evaluate a node's operator or negate value" - ); - expect(schema.properties.virtualMatchString.example).toBe('cpe:2.3:o:microsoft:windows_10_1809'); expect(schema.properties.cweId.description).toContain('CNA data'); expect(schema.properties.cweId.description).toContain('ADP-provided CWE data is not searched'); expect(schema.properties.cvssV3Metrics.description).toContain('same stored vector string'); @@ -166,22 +132,37 @@ describe('OpenAPI contract', () => { expect(operation.responses['410'].description).toContain('use POST /search'); }); - test('documents cpeName as a CPE Match Criteria filter with required components', () => { - const cpeName = openapi.components.schemas.SearchRequest.properties.cpeName; + test('documents semantic CPE filter behavior', () => { + const { cpeName, isVulnerable, virtualMatchString } = openapi.components.schemas.SearchRequest.properties; + expect(isVulnerable.description).toContain('same CPE Match Criteria entry'); + expect(isVulnerable.description).toContain('False or omitted'); expect(cpeName.description).toContain('CPE Match Criteria'); expect(cpeName.description).toContain('part must be a, h, or o'); + expect(cpeName.description).toContain( + 'Every concrete non-version component must be compatible with the same CPE Match Criteria entry' + ); + expect(cpeName.description).toContain('optional * components remain broad matches'); expect(cpeName.description).toContain('version range fields'); expect(cpeName.description).toContain('insensitive to lexical case'); expect(cpeName.description).toContain('exact CPE Match Criteria preserve leading zeros'); + expect(cpeName.description).toContain('Maven-style ordering'); expect(cpeName.description).toContain('logical value NA'); expect(cpeName.description).toContain('numeric range boundaries'); + expect(cpeName.description).toContain('entries independently'); + expect(cpeName.description).toContain("does not evaluate a node's operator or negate value"); expect(cpeName.description).toContain('HTTP 404'); expect(cpeName.description).toContain('virtualMatchString'); expect(cpeName.example).toBe('cpe:2.3:a:freetype:freetype:2.8.1:*:*:*:*:*:*:*'); - expect(openapi.paths['/search'].post.responses['404'].description).toBe('Invalid cpeName value'); - expect(openapi.paths['/search'].post.responses['429'].description).toContain('rate limit'); - expect(openapi.paths['/search'].post.responses['429'].description).toContain('waiter limit'); + expect(virtualMatchString.description).toContain('CPE Match Criteria'); + expect(virtualMatchString.description).toContain('compatible'); + expect(virtualMatchString.description).toContain('insensitive to lexical case'); + expect(virtualMatchString.description).toContain('without a version component'); + expect(virtualMatchString.description).toContain('part must be a, h, o, *, or -'); + expect(virtualMatchString.description).toContain('explicit empty colon-delimited components are invalid'); + expect(virtualMatchString.description).toContain('entries independently'); + expect(virtualMatchString.description).toContain("does not evaluate a node's operator or negate value"); + expect(virtualMatchString.example).toBe('cpe:2.3:o:microsoft:windows_10_1809'); }); test('documents the bounded semantic CPE candidate window', () => { @@ -216,7 +197,10 @@ describe('OpenAPI contract', () => { expect(responses['400'].description).toContain('Malformed JSON'); expect(responses['400'].description).toContain('CVSS vector'); expect(responses['400'].description).toContain('date range'); + expect(responses['404'].description).toBe('Invalid cpeName value'); expect(responses['415'].description).toContain('application/json'); + expect(responses['429'].description).toContain('rate limit'); + expect(responses['429'].description).toContain('waiter limit'); expect(responses['502'].description).toContain('invalid response'); expect(responses['503'].description).toContain('OpenSearch'); expect(responses['503'].headers['Retry-After']).toStrictEqual({ @@ -230,7 +214,7 @@ describe('OpenAPI contract', () => { test('documents the complete POST /search success and error response shapes', () => { const responses = openapi.paths['/search'].post.responses; const successRef = responses['200'].content['application/json'].schema.$ref; - const successSchema = resolveSchemaRef(successRef); + const successSchema = resolveSchema(responses['200'].content['application/json'].schema); expect(successRef).toBe('#/components/schemas/SearchResponse'); expect(successSchema.required).toStrictEqual(['page', 'pageSize', 'totalPages', 'status', 'data']); From 0b499e7aac0f3eeac8dc738945f7d222a5f24e5d Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 14:44:11 -0400 Subject: [PATCH 32/33] docs: align documentation with current API behavior --- README.md | 20 +-- api-docs/openapi.json | 307 +++++++++++++++++++++++++++++++++++++-- api-docs/openapi.test.js | 67 +++++++-- config/default.jsonc | 2 +- routes/index.js | 242 ++++++++++++++++++++++++++++-- routes/swagger.js | 53 +++++++ utils/constants.js | 14 +- 7 files changed, 659 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 5946ff0..4514cda 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Running the API locally -Rename `.env.example` to `.env` and replace or check the following values. +Copy `.env.example` to `.env` and replace or check the following values. - `NODE_CONFIG_ENV` - node-config profile name; use `devel` locally and `prod` in production images - `PORT` - port used for the search API, defaulted to 3000 for development @@ -33,7 +33,7 @@ Rename `.env.example` to `.env` and replace or check the following values. - `NPM_CA_CERT` - optional absolute host path to a corporate or self-signed CA certificate bundle used only during local Docker Compose builds ```bash -mv .env.example .env +cp .env.example .env ``` ### Runtime Configuration Validation @@ -101,9 +101,9 @@ docker compose -f docker-compose.yaml up --build The compose file runs only the Search API with `NODE_ENV=production` and `NODE_CONFIG_ENV=prod`. -## Deploying with npm +## Running with npm -**NOTE: The current locked `cve-core` dependency requires Node.js `>= 16.20.2 < 22`.** +Use Node.js 24.14.1 to match the repository's CI jobs and Docker build/runtime images. The current locked `cve-core` package metadata still declares Node.js `>= 16.20.2 < 22`, so npm can emit an engine warning during installation. Updating that upstream engine declaration belongs in the separately owned `cve-core` repository; this repository validates its current application and container behavior on Node.js 24.14.1. 1. Set `OpenSearchDomainEndpoint` in `.env` to an existing OpenSearch deployment: @@ -114,7 +114,7 @@ OpenSearchDomainEndpoint=https://search.example.com 2. Install dependencies: ```bash -npm install +npm ci ``` 3. Run the CVE Search API in development mode: @@ -129,7 +129,7 @@ npm run dev npm start ``` -The Search API will be served at `localhost:3000`. +The Search API will be served at `http://localhost:3000` by default, or at the port configured with `PORT`. Successful port binding emits a sanitized `runtime.lifecycle` event with the `started` outcome. Synchronous configuration failures and asynchronous listener failures such as an unavailable or unauthorized port emit `startup_failed`, set a nonzero process exit code, and do not emit `started`. Startup failure logs contain only a stable error code and, for configuration errors, the relevant configuration path; exception messages, stacks, addresses, and port details are omitted. @@ -292,11 +292,11 @@ The runtime container and Docker Compose serve the committed `api-docs/openapi.j npm run swagger-autogen ``` -The Swagger API docs will be served at `localhost:3000/api-docs` while the CVE Search API is running via Docker or npm. +The Swagger API docs will be served at `http://localhost:3000/api-docs` while the CVE Search API is running via Docker or npm. The Swagger UI is intentionally read-only: interactive request submission and the "Try it out" controls are disabled. Use an HTTP client or the documented curl examples to call the API. ## Testing -Unit, route, OpenAPI, runtime config, and error helper tests can be run with npm: +Unit, integration-style route, OpenAPI contract, runtime configuration, delivery policy, and utility tests can be run with npm: ```bash npm test @@ -313,7 +313,7 @@ npm run quality Pull requests, merge-queue validation, and pushes to `dev` or `main` run the quality gate and independently build the production `Dockerfile`. The container job reports all high and critical image vulnerabilities without blocking, then rejects fixed high and critical vulnerabilities that are not covered by an approved exception. It starts the image, checks process liveness and the packaged OpenAPI document, and verifies the runtime user and command. It intentionally does not call `/api/ready-check` or `/search`, because the standard CI workflow does not own or provision an OpenSearch index. -Pull requests also run GitHub dependency review and reject newly introduced high or critical vulnerabilities in runtime or development dependencies. The blocking production image scan excludes only `CVE-2026-39244`, an `adm-zip` advisory inherited from the separately owned `cve-core` dependency. The exception records its package owner, fixed version, and review deadline in `.trivyignore`; repository tests fail after that deadline or if the dependency provenance changes. The non-blocking scan still reports the exception and other unfixed severe findings. New advisories are not covered by the exception. +Pull requests also run GitHub dependency review and reject newly introduced high or critical vulnerabilities in runtime or development dependencies. Because this repository is private, its dependency graph and GitHub Code Security must remain enabled for that job to run. The blocking production image scan excludes only `CVE-2026-39244`, an `adm-zip` advisory inherited from the separately owned `cve-core` dependency. The exception records its package owner, fixed version, and review deadline in `.trivyignore`; repository tests fail after that deadline or if the dependency provenance changes. The non-blocking scan still reports the exception and other unfixed severe findings. New advisories are not covered by the exception. An opt-in, read-only OpenSearch contract check is available for an approved environment that can reach an existing index: @@ -390,7 +390,7 @@ The cache is intentionally small and short-lived: - Post-filter evaluation chunk: 250 CPE Match Criteria entries between cancellation checks and event-loop yields - OpenSearch request timeout: 20 seconds for each operation - Overall `/search` deadline: 30 seconds -- Eviction: oldest entries are removed when the cache exceeds its max size +- Eviction: least-recently-used entries are removed when the cache exceeds its max size The admission queue applies only to cold CPE candidate searches. Standard `/search` requests and page fetches are not placed in this queue. When the cold-search queue is full, new unique CPE post-filter searches fail closed with HTTP 429 and a `SEARCH_QUEUE_FULL` response. Identical requests for a search that is already in flight still wait on the shared in-flight result and do not consume additional queue slots, up to the per-operation waiter limit. Additional callers fail closed with HTTP 429 and `SEARCH_IN_FLIGHT_WAITER_LIMIT`. diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 2bb9200..c2ec8e4 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -44,6 +44,14 @@ }, "400": { "description": "Malformed JSON or invalid request body, parameter, CVSS vector, date range, or version range", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -61,6 +69,14 @@ }, "404": { "description": "Invalid cpeName value", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -71,6 +87,14 @@ }, "413": { "description": "The JSON request body exceeded the 100 KB limit", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -81,6 +105,14 @@ }, "415": { "description": "A non-empty request body did not use the application/json media type with UTF-8 encoding", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -109,6 +141,12 @@ "schema": { "type": "string" } + }, + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } } }, "content": { @@ -121,6 +159,14 @@ }, "500": { "description": "An unexpected internal server error occurred", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -131,6 +177,14 @@ }, "502": { "description": "OpenSearch returned an invalid response or an unexpected upstream error occurred", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -147,6 +201,12 @@ "schema": { "type": "integer" } + }, + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } } }, "content": { @@ -159,6 +219,14 @@ }, "504": { "description": "Search exceeded its OpenSearch or overall processing deadline", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -293,6 +361,13 @@ "type": "string" } } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } } } } @@ -304,11 +379,11 @@ "CVE Record" ], "summary": "Deprecated raw query search", - "description": "

Expected Behavior

Retrieves all matching CVE records after applying the JSON request body. This endpoint is deprecated and can be disabled by the service operator. Use POST /search instead.

", + "description": "

Expected Behavior

Retrieves one bounded window of matching CVE records after applying the JSON request body. The resultsTotal field reports the total match count independently from the returned data window. This endpoint is deprecated and can be disabled by the service operator. Use POST /search instead.

", "operationId": "webSearch", "responses": { "200": { - "description": "A list of matching CVE records", + "description": "A bounded window of matching CVE records", "headers": { "X-Request-ID": { "description": "Request correlation ID", @@ -316,20 +391,158 @@ "type": "string" } } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebSearchResponse" + } + } } }, "400": { - "description": "Bad Request" + "description": "Malformed JSON or invalid raw-query request fields", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ErrorResponse" + }, + { + "$ref": "#/components/schemas/ValidationErrorList" + } + ] + } + } + } }, "410": { - "description": "The deprecated endpoint is disabled; use POST /search" + "description": "The deprecated endpoint is disabled; use POST /search", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "413": { + "description": "The JSON request body exceeded the 100 KB limit", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "500": { + "description": "An unexpected internal server error occurred", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "502": { + "description": "OpenSearch returned an invalid response or an unexpected upstream error occurred", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebSearchResponse" + } + } + } + }, + "503": { + "description": "The service is shutting down", + "headers": { + "Retry-After": { + "description": "Seconds before retrying the request", + "schema": { + "type": "integer" + } + }, + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } }, "504": { - "description": "Search exceeded its OpenSearch request deadline" + "description": "Search exceeded its OpenSearch request deadline", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebSearchResponse" + } + } + } } }, "requestBody": { "required": true, + "description": "JSON raw-query search and optional bounded window controls. The JSON request body cannot exceed 100 KB.", "content": { "application/json": { "schema": { @@ -341,36 +554,46 @@ "properties": { "query": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "Non-empty raw search query interpreted by the installed cve-core BasicSearchManager." }, "sort": { "type": "object", "additionalProperties": false, + "required": [ + "property" + ], + "description": "Optional sort selection. sort.property is required when sort is provided; sort.order defaults to desc when omitted.", "properties": { "property": { "type": "string", "enum": [ "cveId", "relevancy" - ] + ], + "description": "Sort by CVE ID or OpenSearch relevance score." }, "order": { "type": "string", "enum": [ "asc", "desc" - ] + ], + "default": "desc", + "description": "Sort direction." } } }, "from": { "type": "integer", - "minimum": 0 + "minimum": 0, + "description": "Zero-based offset into matching records. from plus size cannot exceed 10000." }, "size": { "type": "integer", "minimum": 1, - "maximum": 500 + "maximum": 500, + "description": "Maximum number of records returned in this response. from plus size cannot exceed 10000." } } } @@ -715,6 +938,70 @@ } } }, + "WebSearchMetadata": { + "type": "object", + "required": [ + "searchStatus", + "version", + "cveCore", + "cveMitreCore", + "notes", + "errors" + ], + "additionalProperties": false, + "properties": { + "searchStatus": { + "type": "string" + }, + "version": { + "type": "string" + }, + "cveCore": { + "type": "string" + }, + "cveMitreCore": { + "type": "string" + }, + "queryTime_ms": { + "type": "number", + "minimum": 0 + }, + "notes": { + "type": "array", + "items": {} + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "WebSearchResponse": { + "type": "object", + "required": [ + "searchMetadata", + "resultsTotal", + "data" + ], + "additionalProperties": false, + "properties": { + "searchMetadata": { + "$ref": "#/components/schemas/WebSearchMetadata" + }, + "resultsTotal": { + "type": "integer", + "minimum": 0 + }, + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SearchHit" + } + } + } + }, "ErrorResponse": { "type": "object", "required": [ diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 8ed149b..166024b 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -121,15 +121,55 @@ describe('OpenAPI contract', () => { expect(operation.deprecated).toBe(true); expect(operation.description).toContain('Use POST /search instead'); + expect(operation.description).toContain('one bounded window'); + expect(operation.description).toContain('resultsTotal'); expect(operation.parameters || []).toStrictEqual([]); expect(operation.requestBody.required).toBe(true); + expect(operation.requestBody.description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`); expect(schema.required).toStrictEqual(['query']); expect(schema.additionalProperties).toBe(false); expect(schema.properties.query.type).toBe('string'); expect(schema.properties.sort.additionalProperties).toBe(false); + expect(schema.properties.sort.required).toStrictEqual(['property']); expect(schema.properties.sort.properties.property.enum).toStrictEqual(['cveId', 'relevancy']); + expect(schema.properties.sort.properties.order.default).toBe('desc'); expect(schema.properties.size.maximum).toBe(constants.MAX_RESULTS_LIMIT); + expect(operation.responses['200'].content['application/json'].schema).toStrictEqual({ + $ref: '#/components/schemas/WebSearchResponse' + }); + expect(operation.responses['400'].content['application/json'].schema.oneOf).toStrictEqual([ + { + $ref: '#/components/schemas/ErrorResponse' + }, + { + $ref: '#/components/schemas/ValidationErrorList' + } + ]); expect(operation.responses['410'].description).toContain('use POST /search'); + ['410', '413', '500', '503'].forEach((status) => { + expect(operation.responses[status].content['application/json'].schema).toStrictEqual({ + $ref: '#/components/schemas/ErrorResponse' + }); + }); + ['502', '504'].forEach((status) => { + expect(operation.responses[status].content['application/json'].schema).toStrictEqual({ + $ref: '#/components/schemas/WebSearchResponse' + }); + }); + + expect(openapi.components.schemas.WebSearchResponse.required).toStrictEqual([ + 'searchMetadata', + 'resultsTotal', + 'data' + ]); + expect(openapi.components.schemas.WebSearchMetadata.required).toStrictEqual([ + 'searchStatus', + 'version', + 'cveCore', + 'cveMitreCore', + 'notes', + 'errors' + ]); }); test('documents semantic CPE filter behavior', () => { @@ -232,7 +272,7 @@ describe('OpenAPI contract', () => { } ]); - ['404', '415', '429', '500', '502', '503', '504'].forEach((status) => { + ['404', '413', '415', '429', '500', '502', '503', '504'].forEach((status) => { expect(responses[status].content['application/json'].schema).toStrictEqual({ $ref: '#/components/schemas/ErrorResponse' }); @@ -242,17 +282,26 @@ describe('OpenAPI contract', () => { }); test('documents request correlation headers', () => { - expect(openapi.paths['/search'].post.responses['200'].headers['X-Request-ID']).toStrictEqual({ - description: 'Request correlation ID', - schema: { - type: 'string' - } + Object.values(openapi.paths).forEach((pathItem) => { + Object.values(pathItem).forEach((operation) => { + Object.values(operation.responses).forEach((response) => { + expect(response.headers['X-Request-ID']).toStrictEqual({ + description: 'Request correlation ID', + schema: { + type: 'string' + } + }); + }); + }); }); - expect(openapi.paths['/api/health-check'].get.responses['200'].headers['X-Request-ID']).toBeDefined(); - expect(openapi.paths['/api/ready-check'].get.responses['200'].headers['X-Request-ID']).toBeDefined(); + expect(openapi.paths['/api/ready-check'].get.responses['503'].headers['Retry-After']).toBeDefined(); expect(openapi.paths['/api/ready-check'].get.responses['503'].description).toContain('shutting down'); - expect(openapi.paths['/webSearch'].post.responses['200'].headers['X-Request-ID']).toBeDefined(); + expect(openapi.paths['/api/ready-check'].get.responses['503'].content['application/json'].schema).toStrictEqual( + { + $ref: '#/components/schemas/ErrorResponse' + } + ); }); test('committed OpenAPI artifact matches generated route documentation', async () => { diff --git a/config/default.jsonc b/config/default.jsonc index b04bc01..7f05ddc 100644 --- a/config/default.jsonc +++ b/config/default.jsonc @@ -2,7 +2,7 @@ // Default configuration // These values are overridable using other *.jsonc (e.g., prod.jsonc) // as well as using environment variables (e.g., in `.env`) - // - Each configuration is mapped using custom-environment-variables.jsonc to enable environment varialbe overrides. + // - Each configuration is mapped using custom-environment-variables.jsonc to enable environment variable overrides. // - For more information, see cve-core/src/adapters/config/AppConfig.ts // NOTE for consistency, all values need to be strings for proper type when using .env overrides // Explicit invalid values fail application startup. See README.md for accepted ranges and diff --git a/routes/index.js b/routes/index.js index e76a818..ce80d39 100644 --- a/routes/index.js +++ b/routes/index.js @@ -266,6 +266,14 @@ content: { } #swagger.responses[400] = { description: 'Malformed JSON or invalid request body, parameter, CVSS vector, date range, or version range', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -283,6 +291,14 @@ content: { } #swagger.responses[404] = { description: 'Invalid cpeName value', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -293,6 +309,14 @@ content: { } #swagger.responses[415] = { description: 'A non-empty request body did not use the application/json media type with UTF-8 encoding', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -303,6 +327,14 @@ content: { } #swagger.responses[413] = { description: 'The JSON request body exceeded the 100 KB limit', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -331,6 +363,12 @@ headers: { schema: { type: "string" } + }, + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } } }, content: { @@ -343,6 +381,14 @@ content: { } #swagger.responses[502] = { description: 'OpenSearch returned an invalid response or an unexpected upstream error occurred', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -359,6 +405,12 @@ headers: { schema: { type: "integer" } + }, + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } } }, content: { @@ -371,6 +423,14 @@ content: { } #swagger.responses[504] = { description: 'Search exceeded its OpenSearch or overall processing deadline', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -381,6 +441,14 @@ content: { } #swagger.responses[500] = { description: 'An unexpected internal server error occurred', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -477,6 +545,13 @@ headers: { type: "string" } } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } } } */ @@ -492,9 +567,10 @@ router.post( #swagger.summary = "Deprecated raw query search" #swagger.description = "

Expected Behavior

-

Retrieves all matching CVE records after applying the JSON request body. This endpoint is deprecated and can be disabled by the service operator. Use POST /search instead.

+

Retrieves one bounded window of matching CVE records after applying the JSON request body. The resultsTotal field reports the total match count independently from the returned data window. This endpoint is deprecated and can be disabled by the service operator. Use POST /search instead.

#swagger.requestBody = { required: true, + description: "JSON raw-query search and optional bounded window controls. The JSON request body cannot exceed 100 KB.", content: { "application/json": { schema: { @@ -504,30 +580,38 @@ router.post( properties: { query: { type: "string", - minLength: 1 + minLength: 1, + description: "Non-empty raw search query interpreted by the installed cve-core BasicSearchManager." }, sort: { type: "object", additionalProperties: false, + required: ["property"], + description: "Optional sort selection. sort.property is required when sort is provided; sort.order defaults to desc when omitted.", properties: { property: { type: "string", - enum: ["cveId", "relevancy"] + enum: ["cveId", "relevancy"], + description: "Sort by CVE ID or OpenSearch relevance score." }, order: { type: "string", - enum: ["asc", "desc"] + enum: ["asc", "desc"], + default: "desc", + description: "Sort direction." } } }, from: { type: "integer", - minimum: 0 + minimum: 0, + description: "Zero-based offset into matching records. from plus size cannot exceed 10000." }, size: { type: "integer", minimum: 1, - maximum: 500 + maximum: 500, + description: "Maximum number of records returned in this response. from plus size cannot exceed 10000." } } } @@ -535,7 +619,7 @@ router.post( } } #swagger.responses[200] = { -description: 'A list of matching CVE records', +description: 'A bounded window of matching CVE records', headers: { "X-Request-ID": { description: "Request correlation ID", @@ -543,13 +627,153 @@ headers: { type: "string" } } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/WebSearchResponse" + } + } } } -#swagger.responses[504] = { -description: 'Search exceeded its OpenSearch request deadline', +#swagger.responses[400] = { +description: 'Malformed JSON or invalid raw-query request fields', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + oneOf: [ + { + $ref: "#/components/schemas/ErrorResponse" + }, + { + $ref: "#/components/schemas/ValidationErrorList" + } + ] + } + } +} } #swagger.responses[410] = { description: 'The deprecated endpoint is disabled; use POST /search', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} +#swagger.responses[413] = { +description: 'The JSON request body exceeded the 100 KB limit', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} +#swagger.responses[500] = { +description: 'An unexpected internal server error occurred', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} +#swagger.responses[502] = { +description: 'OpenSearch returned an invalid response or an unexpected upstream error occurred', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/WebSearchResponse" + } + } +} +} +#swagger.responses[503] = { +description: 'The service is shutting down', +headers: { + "Retry-After": { + description: "Seconds before retrying the request", + schema: { + type: "integer" + } + }, + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} +#swagger.responses[504] = { +description: 'Search exceeded its OpenSearch request deadline', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/WebSearchResponse" + } + } +} } */ requireWebSearchEnabled, diff --git a/routes/swagger.js b/routes/swagger.js index 2982644..0629922 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -348,6 +348,59 @@ const searchResponseSchemas = { } } }, + WebSearchMetadata: { + type: 'object', + required: ['searchStatus', 'version', 'cveCore', 'cveMitreCore', 'notes', 'errors'], + additionalProperties: false, + properties: { + searchStatus: { + type: 'string' + }, + version: { + type: 'string' + }, + cveCore: { + type: 'string' + }, + cveMitreCore: { + type: 'string' + }, + queryTime_ms: { + type: 'number', + minimum: 0 + }, + notes: { + type: 'array', + items: {} + }, + errors: { + type: 'array', + items: { + $ref: '#/components/schemas/ErrorResponse' + } + } + } + }, + WebSearchResponse: { + type: 'object', + required: ['searchMetadata', 'resultsTotal', 'data'], + additionalProperties: false, + properties: { + searchMetadata: { + $ref: '#/components/schemas/WebSearchMetadata' + }, + resultsTotal: { + type: 'integer', + minimum: 0 + }, + data: { + type: 'array', + items: { + $ref: '#/components/schemas/SearchHit' + } + } + } + }, ErrorResponse: { type: 'object', required: ['error', 'message'], diff --git a/utils/constants.js b/utils/constants.js index 483a1b2..8a1b1c7 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -59,15 +59,15 @@ const cpeMatchFields = { // Dictionary of fields to paths /** * keywordSearch: list of terms - * keywordExactMatch: list of terms, must be an exact match to terms in a description + * keywordExactMatch: boolean that requires keywordSearch terms to appear as an exact phrase * cveIds: comma-separated list of CVE-year-sequence identifiers - * noRejected: no-value flag that excludes rejected CVE records + * noRejected: boolean that excludes rejected CVE records when true * cweId: CWE-XXX - * cvssV2Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL - * cvssV3Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL - * cvssV3_0Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL - * cvssV3_1Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL - * cvssV4Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL + * cvssV2Severity: LOW, MEDIUM, HIGH + * cvssV3Severity: LOW, MEDIUM, HIGH, CRITICAL + * cvssV3_0Severity: LOW, MEDIUM, HIGH, CRITICAL + * cvssV3_1Severity: LOW, MEDIUM, HIGH, CRITICAL + * cvssV4Severity: LOW, MEDIUM, HIGH, CRITICAL * cvssV2Metrics: CVSSv2 vector string * cvssV3Metrics: CVSSv3_0 or CVSSv3_1 vector string * cvssV3_0Metrics: CVSSv3_0 vector string From 832a67cc76f388c410afb16ab5c6ad224cbfbe21 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 16:33:19 -0400 Subject: [PATCH 33/33] Harden search validation and runtime safeguards --- .env.example | 1 + README.md | 20 +- api-docs/openapi.json | 24 ++- api-docs/openapi.test.js | 13 +- config/custom-environment-variables.jsonc | 3 +- config/default.jsonc | 4 +- config/runtime-config.test.js | 1 + controllers/middleware.js | 5 + controllers/middleware.test.js | 30 ++- controllers/searchController.js | 110 +++++++---- controllers/searchController.test.js | 219 +++++++++++++++++----- package-lock.json | 6 +- package.json | 2 +- routes/index.js | 18 ++ routes/search.integration.test.js | 61 ++++++ routes/swagger.js | 4 +- test/contract/opensearch.js | 90 ++++++--- test/contract/opensearch.test.js | 188 +++++++++++++++---- utils/constants.js | 4 + utils/cpe.js | 3 + utils/cpe.test.js | 13 ++ utils/cweId.js | 20 ++ utils/cweId.test.js | 33 ++++ utils/errors.js | 14 ++ utils/errors.test.js | 16 ++ utils/runtimeConfig.js | 9 +- utils/runtimeConfig.test.js | 8 +- utils/searchCandidateProcessor.test.js | 4 + utils/searchPerformance.js | 88 ++++++++- utils/searchPerformance.test.js | 62 +++++- utils/searchProvider.js | 5 + utils/searchProvider.test.js | 3 + utils/searchQueryBuilder.js | 8 + utils/searchQueryBuilder.test.js | 3 +- utils/searchResponseLimiter.js | 61 ++++++ utils/searchResponseLimiter.test.js | 92 +++++++++ 36 files changed, 1058 insertions(+), 187 deletions(-) create mode 100644 utils/cweId.js create mode 100644 utils/cweId.test.js create mode 100644 utils/searchResponseLimiter.js create mode 100644 utils/searchResponseLimiter.test.js diff --git a/.env.example b/.env.example index f2680a2..19fb5bd 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,7 @@ SearchPostFilterEvaluationChunkSize=250 SearchOpenSearchRequestTimeoutMs=20000 SearchRequestDeadlineMs=30000 SearchReadinessTimeoutMs=5000 +SearchMaxResponseBytes=10485760 RequestLogSuccessSampleRate=0.1 RequestLogHealthChecks=false ShutdownTimeoutMs=35000 diff --git a/README.md b/README.md index 4514cda..1735ec9 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Copy `.env.example` to `.env` and replace or check the following values. - `SearchOpenSearchRequestTimeoutMs` - optional timeout for each OpenSearch request, default `20000` - `SearchRequestDeadlineMs` - optional overall `/search` processing deadline, default `30000` - `SearchReadinessTimeoutMs` - optional timeout for the OpenSearch index readiness check, default `5000` +- `SearchMaxResponseBytes` - optional maximum serialized successful `/search` response size in bytes, default `10485760` (10 MiB) - `RequestLogSuccessSampleRate` - optional fraction of successful non-health-check access logs to retain, default `0.1` - `RequestLogHealthChecks` - optional flag to retain successful health-check access logs, default `false` - `ShutdownTimeoutMs` - optional graceful HTTP shutdown deadline, default `35000` @@ -55,6 +56,7 @@ The following numeric bounds are enforced: | `SearchOpenSearchRequestTimeoutMs` | 1 through 300,000 | | `SearchRequestDeadlineMs` | 1 through 600,000 | | `SearchReadinessTimeoutMs` | 1 through 60,000 | +| `SearchMaxResponseBytes` | 65,536 through 104,857,600 | | `RequestLogSuccessSampleRate` | 0 through 1 | | `ShutdownTimeoutMs` | 1 through 900,000 | | `HttpHeadersTimeoutMs` | 1 through 60,000 | @@ -159,7 +161,7 @@ String inputs are bounded before query construction: | ---------------------------- | ---------------------------------------------------------------------: | | `keywordSearch` | 512 characters and 64 whitespace-delimited terms | | `cveIds` | 4,096 characters overall, 100 IDs, and 4-19 sequence digits per CVE ID | -| `cweId` | 32 characters | +| `cweId` | 32 characters and `CWE-` format | | `cpeName` | 2,048 characters | | `virtualMatchString` | 2,048 characters | | `versionStart`, `versionEnd` | 256 characters each | @@ -169,6 +171,8 @@ Requests that exceed a limit return HTTP 400 before contacting OpenSearch. These CVSS vector filters accept full or partial vectors containing at least one unique `KEY:VALUE` metric token. An optional vector header must identify a version supported by the selected parameter. Requested metric tokens can appear in any order, but all tokens must occur in the same stored vector string. CVSS and CWE filters search CNA-provided data only; ADP-provided enrichment is not currently included. +The `cweId` filter accepts a positive decimal CWE identifier such as `CWE-79`. The prefix is case-insensitive, surrounding whitespace is trimmed, and valid input is normalized to uppercase before query construction. Zero, leading-zero identifiers such as `CWE-079`, missing or non-numeric identifiers, and suffixes return HTTP 400 without contacting OpenSearch. + Publication and last-modified date filters support one-sided ranges. Values must use the exact UTC format `yyyy-MM-ddTHH:mm:ss.SSSZ`, including three fractional-second digits and the trailing `Z`. When both values are supplied, `pubStartDate` must be less than or equal to `pubEndDate`, and `lastModStartDate` must be less than or equal to `lastModEndDate`. ### Search Ordering and Pagination @@ -179,6 +183,8 @@ Publication and last-modified date filters support one-sided ranges. Values must Pagination is limited to complete pages inside the first 10,000 results: `from + resultsPerPage` must not exceed 10,000. `totalPages` reports the smaller of the pages implied by the match count and the complete pages accessible inside that window. For example, a page size of 30 allows at most 333 pages. Standard-search totals are exact and are not reduced by this pagination calculation. Semantic CPE searches have a separate 10,000-candidate evaluation limit described below, so their totals can be incomplete. Point-in-time search and deep pagination remain deferred. +Successful `/search` responses are also limited by serialized UTF-8 size. The default limit is 10 MiB and can be changed with `SearchMaxResponseBytes`. The API serializes the complete response and rechecks the overall search deadline before sending any bytes. If the successful payload exceeds the configured limit, the API returns HTTP 422 with `SEARCH_RESPONSE_TOO_LARGE`; reduce `resultsPerPage` and retry. Error responses bypass this success-payload limit so clients receive a complete error body. Match totals and the 500-record page-size limit are unchanged. + Use `cveIds` to request up to 100 CVE records by comma-separated CVE ID. Each ID must contain the `CVE` prefix, a four-digit year, and a sequence number containing 4 to 19 digits. The prefix is accepted without regard to case and normalized to uppercase for the OpenSearch query. ```bash @@ -321,7 +327,7 @@ An opt-in, read-only OpenSearch contract check is available for an approved envi npm run test:opensearch-contract ``` -Set `OpenSearchDomainEndpoint` and `OpenSearchCveIndex` in the environment or local `.env` before running it. `OpenSearchAllowUnknownSslCerts` remains optional and should be enabled only for local development against a self-signed certificate. The contract check sends a minimal readiness query, validates representative field capabilities, requires a concrete CVE record, and uses that record for a positive CVE-ID search before running the remaining bounded query shapes. It closes its OpenSearch client and emits sanitized JSON output. It never creates, updates, maps, or deletes indexes. +Set `OpenSearchDomainEndpoint` and `OpenSearchCveIndex` in the environment or local `.env` before running it. `OpenSearchAllowUnknownSslCerts` remains optional and should be enabled only for local development against a self-signed certificate. The contract check sends a minimal readiness query, validates representative field capabilities, and requires a concrete CVE record with a structurally valid direct CNA CPE applicability entry. It uses that record for a positive CVE-ID search and one of its validated CPE criteria values for a positive candidate search before running the remaining bounded query shapes. It closes its OpenSearch client and emits sanitized JSON output. It never creates, updates, maps, or deletes indexes. The contract check is not part of public pull-request CI. The `OpenSearch Contract` GitHub Actions workflow runs it only through a manual dispatch against a selected protected GitHub environment. Configure that environment with: @@ -331,13 +337,13 @@ The contract check is not part of public pull-request CI. The `OpenSearch Contra The workflow can run only from the trusted `dev` or `main` branch. The selected GitHub environment should additionally require reviewers and restrict deployment branches. OpenSearch secrets are exposed only to the final contract execution step, after checkout, Node setup, and dependency installation have completed. -The selected runner must have network access to the endpoint. Change the workflow to an approved self-hosted runner before use when the index is not reachable from GitHub-hosted runners. The workflow performs readiness, field-capability, positive source-projection, exact-total, CVE ID keyword-subfield, keyword, publication-date, CVSS, and CPE candidate checks. Every search is read-only and returns at most one record. Connection values and credentials remain in environment secrets rather than source files or workflow text. +The selected runner must have network access to the endpoint. Change the workflow to an approved self-hosted runner before use when the index is not reachable from GitHub-hosted runners. The workflow performs readiness, field-capability, positive CPE source-projection, exact-total, CVE ID keyword-subfield, keyword, publication-date, CVSS, and positive CPE candidate checks. Candidate responses are validated with the same CVE ID, applicability, node, CPE match, vulnerability, criteria, and version-range requirements used by application searches. Every search is read-only and returns at most one record. Connection values and credentials remain in environment secrets rather than source files or workflow text. The validation tiers have separate purposes: - Jest and `npm run quality` verify application behavior, API contracts, and regression coverage with controlled test doubles. - Pull-request container validation verifies production-image packaging, non-root execution, startup, liveness, and packaged API documentation. -- The optional OpenSearch contract check verifies that the installed client can communicate with a non-empty externally managed index whose representative searchable fields satisfy the application contract. +- The optional OpenSearch contract check verifies that the installed client can communicate with a non-empty externally managed index whose representative searchable fields and direct CNA CPE applicability records satisfy the application contract. - Load and performance testing remain in the separate automation performance-testing repository. Dependabot checks npm packages, Docker base-image digests, and GitHub Actions weekly. Version-update pull requests target the `dev` integration branch; GitHub security updates continue to follow the repository default-branch behavior. Both Dockerfiles pin the Node build image and Distroless runtime image to immutable multi-architecture digests; Dependabot updates those references through reviewable pull requests. @@ -350,7 +356,9 @@ Access logs omit query strings and request bodies. By default, the API retains 1 The API always emits a sanitized JSON `search.performance` event for `/search` queue saturation, shared-operation waiter saturation, queue wait timeout, search timeout, client disconnect, OpenSearch connection errors, and upstream search errors. These events include the request ID, outcome, HTTP status, strategy, cache state, total elapsed time, cancellation reason, and bounded queue or waiter state when applicable. Search terms, CPE values, request bodies, credentials, and upstream error details are never included. -Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, shared waiter counts, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, response bytes, JSON serialization time, process heap usage and per-request heap delta, cache entry and cached CVE ID counts, recent process-wide event-loop delay, and total handler elapsed time. Leave it disabled during normal operation unless the deployment logging volume is acceptable. +Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, shared waiter counts, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, response bytes, JSON serialization time, process heap usage and per-request heap delta, cache entry and cached CVE ID counts, recent process-wide event-loop delay, and total handler elapsed time. Oversized response failures are logged even when detailed timing is disabled and include the attempted byte count, configured maximum, and attempted serialization time. Leave detailed logging disabled during normal operation unless the deployment logging volume is acceptable. + +Candidate-discovery and page-fetch timings describe the underlying operation, which can be shared by multiple active requests. Every request that successfully waits for that operation reports the same operation timing while retaining its own cache status, in-flight status, waiter count, wait duration, deadline, response, and outcome. If the initiating client disconnects but another waiter remains, the surviving request still receives the completed operation metrics. Cache entries contain only candidate results, so a later cache hit does not replay historical candidate, post-filter, or queue timings; it reports only work performed for the current request, such as its page fetch. Event-loop delay uses one low-overhead process monitor sampled in rolling 10-second windows. Request events report the most recently completed window through `processEventLoopDelayMeanMs`, `processEventLoopDelayP99Ms`, `processEventLoopDelayMaxMs`, `processEventLoopDelayWindowMs`, and `processEventLoopDelaySampleAgeMs`. These values describe process-wide event-loop saturation near the request, not delay caused by that individual request. They are omitted until the first complete window is available, and concurrent requests read the same completed snapshot without resetting one another's measurements. @@ -415,7 +423,7 @@ This helps most when multiple users submit the same expensive CPE search within The cache and cold-search queue are a temporary per-process proof of concept. They do not share state across ECS tasks, so each task warms and admits work independently depending on load balancer routing. Track cache-hit and in-flight-share rates, cold-search queue waits and rejections, cold-versus-warm latency, and OpenSearch time to decide their long-term value. If the benefit is low, remove this layer; if it is material, replace it with an AWS-managed shared cache and admission-control design. -The request lifecycle and in-flight waiter registry are separate from the result cache, so a later shared-cache replacement does not need to change caller cancellation behavior. The per-process TTL, entry limit, cold-search concurrency, queue depth, queue wait, shared waiter limit, evaluation chunk size, OpenSearch timeout, and overall request deadline can be tuned with `SearchPostFilterCacheTtlMs`, `SearchPostFilterCacheMaxEntries`, `SearchPostFilterColdSearchConcurrency`, `SearchPostFilterColdSearchMaxQueueDepth`, `SearchPostFilterColdSearchMaxWaitMs`, `SearchInFlightMaxWaitersPerOperation`, `SearchPostFilterEvaluationChunkSize`, `SearchOpenSearchRequestTimeoutMs`, and `SearchRequestDeadlineMs`. +The request lifecycle and in-flight waiter registry are separate from the result cache, so a later shared-cache replacement does not need to change caller cancellation behavior. The per-process TTL, entry limit, cold-search concurrency, queue depth, queue wait, shared waiter limit, evaluation chunk size, OpenSearch timeout, overall request deadline, and successful response size can be tuned with `SearchPostFilterCacheTtlMs`, `SearchPostFilterCacheMaxEntries`, `SearchPostFilterColdSearchConcurrency`, `SearchPostFilterColdSearchMaxQueueDepth`, `SearchPostFilterColdSearchMaxWaitMs`, `SearchInFlightMaxWaitersPerOperation`, `SearchPostFilterEvaluationChunkSize`, `SearchOpenSearchRequestTimeoutMs`, `SearchRequestDeadlineMs`, and `SearchMaxResponseBytes`. ### HTTP Transport Timeouts diff --git a/api-docs/openapi.json b/api-docs/openapi.json index c2ec8e4..78cf60f 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -121,6 +121,24 @@ } } }, + "422": { + "description": "The serialized successful response exceeded the configured maximum size; reduce resultsPerPage and retry", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, "429": { "description": "Application rate limit exceeded, the cold CPE search queue is full or exceeded its maximum wait, or a shared semantic search operation reached its waiter limit", "headers": { @@ -635,9 +653,11 @@ }, "cweId": { "type": "string", + "minLength": 5, "maxLength": 32, + "pattern": "^\\s*[Cc][Ww][Ee]-[1-9][0-9]*\\s*$", "example": "CWE-79", - "description": "Returns CVE records whose CNA data includes the provided Common Weakness Enumeration ID. ADP-provided CWE data is not searched." + "description": "Returns CVE records whose CNA data includes the provided Common Weakness Enumeration ID. The accepted format is the CWE- prefix followed by a positive integer without leading zeros. Prefix matching is case-insensitive, surrounding whitespace is trimmed, and valid input is normalized to uppercase before search. Invalid values return HTTP 400. ADP-provided CWE data is not searched." }, "cvssV2Severity": { "type": "string", @@ -1029,4 +1049,4 @@ } } } -} \ No newline at end of file +} diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 166024b..192a6da 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -60,6 +60,10 @@ describe('OpenAPI contract', () => { expect(operation.requestBody.description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`); expect(mediaType.schema).toStrictEqual({ $ref: '#/components/schemas/SearchRequest' }); expect(operation.responses['413'].description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`); + expect(operation.responses['422'].description).toContain('reduce resultsPerPage'); + expect(operation.responses['422'].content['application/json'].schema).toStrictEqual({ + $ref: '#/components/schemas/ErrorResponse' + }); expect(schema.additionalProperties).toBe(false); expect(Object.keys(schema.properties)).toStrictEqual(constants.SEARCH_QUERY_PARAMS); Object.values(schema.properties).forEach((property) => { @@ -95,7 +99,13 @@ describe('OpenAPI contract', () => { 'prerelease qualifiers before an unqualified release' ); expect(schema.properties.cweId.description).toContain('CNA data'); + expect(schema.properties.cweId.description).toContain('CWE- prefix followed by a positive integer'); + expect(schema.properties.cweId.description).toContain('without leading zeros'); + expect(schema.properties.cweId.description).toContain('normalized to uppercase'); + expect(schema.properties.cweId.description).toContain('HTTP 400'); expect(schema.properties.cweId.description).toContain('ADP-provided CWE data is not searched'); + expect(schema.properties.cweId.minLength).toBe(5); + expect(schema.properties.cweId.pattern).toBe(constants.CWE_ID_PATTERN); expect(schema.properties.cvssV3Metrics.description).toContain('same stored vector string'); expect(schema.properties.cvssV3Metrics.description).toContain('CNA data'); expect(schema.properties.cvssV3Metrics.description).toContain('ADP-provided CVSS data is not searched'); @@ -239,6 +249,7 @@ describe('OpenAPI contract', () => { expect(responses['400'].description).toContain('date range'); expect(responses['404'].description).toBe('Invalid cpeName value'); expect(responses['415'].description).toContain('application/json'); + expect(responses['422'].description).toContain('reduce resultsPerPage'); expect(responses['429'].description).toContain('rate limit'); expect(responses['429'].description).toContain('waiter limit'); expect(responses['502'].description).toContain('invalid response'); @@ -272,7 +283,7 @@ describe('OpenAPI contract', () => { } ]); - ['404', '413', '415', '429', '500', '502', '503', '504'].forEach((status) => { + ['404', '413', '415', '422', '429', '500', '502', '503', '504'].forEach((status) => { expect(responses[status].content['application/json'].schema).toStrictEqual({ $ref: '#/components/schemas/ErrorResponse' }); diff --git a/config/custom-environment-variables.jsonc b/config/custom-environment-variables.jsonc index e9a2744..a72b60e 100644 --- a/config/custom-environment-variables.jsonc +++ b/config/custom-environment-variables.jsonc @@ -21,7 +21,8 @@ "postFilterEvaluationChunkSize": "SearchPostFilterEvaluationChunkSize", "openSearchRequestTimeoutMs": "SearchOpenSearchRequestTimeoutMs", "requestDeadlineMs": "SearchRequestDeadlineMs", - "readinessTimeoutMs": "SearchReadinessTimeoutMs" + "readinessTimeoutMs": "SearchReadinessTimeoutMs", + "maxResponseBytes": "SearchMaxResponseBytes" }, "requestLogging": { "successSampleRate": "RequestLogSuccessSampleRate", diff --git a/config/default.jsonc b/config/default.jsonc index 7f05ddc..86df8a5 100644 --- a/config/default.jsonc +++ b/config/default.jsonc @@ -36,7 +36,9 @@ // Limits total processing time for a /search request. "requestDeadlineMs": "30000", // Limits the minimal OpenSearch request used by /api/ready-check. - "readinessTimeoutMs": "5000" + "readinessTimeoutMs": "5000", + // Limits a successfully serialized /search response before any bytes are sent. + "maxResponseBytes": "10485760" }, // Access logging settings. Error responses are always logged. "requestLogging": { diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index 77486cf..6e91493 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -119,6 +119,7 @@ describe('runtime container and compose configuration', () => { ); expect(customEnvironmentVariables).toContain('"requestDeadlineMs": "SearchRequestDeadlineMs"'); expect(customEnvironmentVariables).toContain('"readinessTimeoutMs": "SearchReadinessTimeoutMs"'); + expect(customEnvironmentVariables).toContain('"maxResponseBytes": "SearchMaxResponseBytes"'); expect(customEnvironmentVariables).toContain('"successSampleRate": "RequestLogSuccessSampleRate"'); expect(customEnvironmentVariables).toContain('"healthChecks": "RequestLogHealthChecks"'); expect(customEnvironmentVariables).toContain('"shutdownTimeoutMs": "ShutdownTimeoutMs"'); diff --git a/controllers/middleware.js b/controllers/middleware.js index af2c0ce..f9831f0 100644 --- a/controllers/middleware.js +++ b/controllers/middleware.js @@ -2,6 +2,7 @@ const errors = require('../utils/errors'); const constants = require('../utils/constants'); const cpe = require('../utils/cpe'); const cveId = require('../utils/cveId'); +const cweId = require('../utils/cweId'); const cvss = require('../utils/cvss'); const error = new errors.errors(); @@ -152,6 +153,10 @@ function validateParams(req, res, next) { } } + if (hasParam(params, 'cweId') && !cweId.isValidCweId(params.cweId)) { + return res.status(400).json(error.cweIdFormatError()); + } + if (hasParam(params, 'noRejected') && ![true, false].includes(params.noRejected)) { return res.status(400).json(error.noRejectedValueError()); } diff --git a/controllers/middleware.test.js b/controllers/middleware.test.js index 6bfe8d4..0024e2e 100644 --- a/controllers/middleware.test.js +++ b/controllers/middleware.test.js @@ -97,7 +97,9 @@ describe('middleware validation function', () => { ['wildcard part', 'cpe:2.3:*:vendor:product:1.0:*:*:*:*:*:*:*'], ['wildcard vendor', 'cpe:2.3:a:*:product:1.0:*:*:*:*:*:*:*'], ['wildcard product', 'cpe:2.3:a:vendor:*:1.0:*:*:*:*:*:*:*'], - ['wildcard version', 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*'] + ['wildcard version', 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*'], + ['empty optional component', 'cpe:2.3:a:vendor:product:1.0::*'], + ['empty trailing component', 'cpe:2.3:a:vendor:product:1.0:'] ])('ValidateParams returns an error when cpeName has %s.', (_caseName, cpeName) => { validate({ cpeName @@ -401,6 +403,7 @@ describe('middleware validation function', () => { test.each([ ['keywordSearch', ''], + ['cweId', ''], ['cpeName', ' '], ['virtualMatchString', ''], ['pubStartDate', ''], @@ -485,6 +488,29 @@ describe('middleware validation function', () => { expectValidationError('CVE_IDS_LIMIT', 'The cveIds parameter cannot contain more than 100 CVE IDs.'); }); + test.each([ + ['a missing prefix', '79'], + ['a zero identifier', 'CWE-0'], + ['a leading zero', 'CWE-079'], + ['a missing numeric identifier', 'CWE-'], + ['a non-numeric identifier', 'CWE-noinfo'], + ['embedded whitespace', 'CWE- 79'], + ['a suffix', 'CWE-79-extra'] + ])('ValidateParams returns an error when cweId has %s.', (_caseName, cweId) => { + validate({ cweId }); + + expectValidationError( + 'CWE_ID_FORMAT', + 'The cweId parameter must use the format CWE- without leading zeros.' + ); + }); + + test.each(['CWE-79', 'cwe-79', ' CwE-79 '])('ValidateParams accepts valid cweId value %p.', (cweId) => { + validate({ cweId }); + + expectValidationSuccess(); + }); + test.each(Object.entries(constants.SEARCH_STRING_LENGTH_LIMITS))( 'ValidateParams returns an error when %s exceeds its maximum length.', (param, maxLength) => { @@ -501,7 +527,7 @@ describe('middleware validation function', () => { test.each([ ['keywordSearch', 'a'.repeat(constants.SEARCH_STRING_LENGTH_LIMITS.keywordSearch)], - ['cweId', 'a'.repeat(constants.SEARCH_STRING_LENGTH_LIMITS.cweId)] + ['cweId', `CWE-${'1'.repeat(constants.SEARCH_STRING_LENGTH_LIMITS.cweId - 4)}`] ])('ValidateParams accepts %s at its maximum length.', (param, value) => { validate({ [param]: value diff --git a/controllers/searchController.js b/controllers/searchController.js index 16dc69d..5f45f49 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -8,6 +8,7 @@ const searchInFlightRegistry = require('../utils/searchInFlightRegistry'); const searchPostFilterWorkQueue = require('../utils/searchPostFilterWorkQueue'); const searchProvider = require('../utils/searchProvider'); const searchQueryBuilder = require('../utils/searchQueryBuilder'); +const searchResponseLimiter = require('../utils/searchResponseLimiter'); const { OPEN_SEARCH_RESPONSE_FAILURE_TYPES, OpenSearchResponseError } = require('../utils/searchResponseValidator'); const constants = require('../utils/constants'); const error = new errors.errors(); @@ -197,15 +198,21 @@ async function fetchFullHitsByCveIds(cveIds, metrics, signal) { const waitStart = searchPerformance.markTime(); const inFlight = searchInFlightRegistry.acquire( inFlightKey, - async (operationSignal) => { - const body = searchQueryBuilder.buildCveIdsSearchBody(cveIds); - const pageFetchStart = searchPerformance.markTime(); - const pageData = await executeOpenSearch(body, 'page-fetch', operationSignal, metrics); - metrics.pageFetchElapsedMs = searchPerformance.elapsedMs(pageFetchStart); - metrics.pageFetchTookMs = pageData?.took; - - return validateAndOrderPageHits(pageData.hits.hits, cveIds); - }, + (operationSignal) => + searchPerformance.captureOperationOutcome(async (operationMetrics) => { + const body = searchQueryBuilder.buildCveIdsSearchBody(cveIds); + const pageFetchStart = searchPerformance.markTime(); + let pageData; + + try { + pageData = await executeOpenSearch(body, 'page-fetch', operationSignal, operationMetrics); + } finally { + operationMetrics.pageFetchElapsedMs = searchPerformance.elapsedMs(pageFetchStart); + } + operationMetrics.pageFetchTookMs = pageData?.took; + + return validateAndOrderPageHits(pageData.hits.hits, cveIds); + }), { signal, maxWaiters: searchProvider.getInFlightMaxWaitersPerOperation() @@ -214,31 +221,42 @@ async function fetchFullHitsByCveIds(cveIds, metrics, signal) { metrics.pageFetchInFlightHit = inFlight.shared; metrics.pageFetchInFlightWaiterCount = inFlight.waiterCount; - const result = await inFlight.promise; + const outcome = await inFlight.promise; if (inFlight.shared) { metrics.pageFetchWaitElapsedMs = searchPerformance.elapsedMs(waitStart); } - return result; + return searchPerformance.consumeOperationOutcome(outcome, metrics); } -async function buildPostFilterCandidateResult(candidateBody, postFilters, metrics, signal) { +async function buildPostFilterCandidateResult(candidateBody, postFilters, operationMetrics, signal) { const candidateStart = searchPerformance.markTime(); - const candidateData = await executeOpenSearch(candidateBody, 'cpe-candidate', signal, metrics); - metrics.openSearchElapsedMs = searchPerformance.elapsedMs(candidateStart); - metrics.openSearchTookMs = candidateData?.took; + let candidateData; + + try { + candidateData = await executeOpenSearch(candidateBody, 'cpe-candidate', signal, operationMetrics); + } finally { + operationMetrics.openSearchElapsedMs = searchPerformance.elapsedMs(candidateStart); + } + operationMetrics.openSearchTookMs = candidateData?.took; const candidateResp = { data: candidateData }; const postFilterStart = searchPerformance.markTime(); - const { candidateCount, filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( - candidateResp, - postFilters, - signal, - searchProvider.getPostFilterEvaluationChunkSize() - ); - metrics.postFilterElapsedMs = searchPerformance.elapsedMs(postFilterStart); + let candidateResult; + + try { + candidateResult = await searchCandidateProcessor.filterSearchHitsInChunks( + candidateResp, + postFilters, + signal, + searchProvider.getPostFilterEvaluationChunkSize() + ); + } finally { + operationMetrics.postFilterElapsedMs = searchPerformance.elapsedMs(postFilterStart); + } + const { candidateCount, filteredHits } = candidateResult; return { candidateCount, @@ -257,24 +275,30 @@ async function getPostFilterCandidateResult(cacheKey, candidateBody, postFilters metrics.cacheHit = false; const inFlightKey = searchInFlightRegistry.buildKey('candidate', cacheKey); - const queueStart = searchPerformance.markTime(); const waitStart = searchPerformance.markTime(); const inFlight = searchInFlightRegistry.acquire( inFlightKey, - (operationSignal) => - searchPostFilterWorkQueue - .run( + (operationSignal) => { + const queueStart = searchPerformance.markTime(); + + return searchPerformance.captureOperationOutcome(async (operationMetrics) => { + const result = await searchPostFilterWorkQueue.run( () => { - metrics.coldSearchQueueWaitElapsedMs = searchPerformance.elapsedMs(queueStart); - metrics.coldSearchQueueState = searchPostFilterWorkQueue.getState(); - return buildPostFilterCandidateResult(candidateBody, postFilters, metrics, operationSignal); + operationMetrics.coldSearchQueueWaitElapsedMs = searchPerformance.elapsedMs(queueStart); + operationMetrics.coldSearchQueueState = searchPostFilterWorkQueue.getState(); + return buildPostFilterCandidateResult( + candidateBody, + postFilters, + operationMetrics, + operationSignal + ); }, { signal: operationSignal } - ) - .then((result) => { - searchPostFilterCache.set(cacheKey, result); - return result; - }), + ); + searchPostFilterCache.set(cacheKey, result); + return result; + }); + }, { signal, maxWaiters: searchProvider.getInFlightMaxWaitersPerOperation() @@ -283,12 +307,12 @@ async function getPostFilterCandidateResult(cacheKey, candidateBody, postFilters metrics.inFlightHit = inFlight.shared; metrics.inFlightWaiterCount = inFlight.waiterCount; - const result = await inFlight.promise; + const outcome = await inFlight.promise; if (inFlight.shared) { metrics.inFlightWaitElapsedMs = searchPerformance.elapsedMs(waitStart); } - return result; + return searchPerformance.consumeOperationOutcome(outcome, metrics); } async function executeOptimizedPostFilterSearch( @@ -366,6 +390,11 @@ async function searchHandler(req, res, next) { res, searchProvider.getSearchRequestDeadlineMs() ); + searchResponseLimiter.instrumentSearchResponse(res, { + maxResponseBytes: searchProvider.getSearchMaxResponseBytes(), + metrics, + checkDeadline: lifecycle.getRemainingMs + }); let searchOptions; let postFilters; let responseOptions; @@ -396,6 +425,15 @@ async function searchHandler(req, res, next) { ...resp }); } catch (err) { + if (searchResponseLimiter.isSearchResponseTooLargeError(err)) { + metrics.outcome = 'response_too_large'; + metrics.statusCode = 422; + metrics.attemptedResponseBytes = err.responseBytes; + metrics.maxResponseBytes = err.maxResponseBytes; + + return res.status(422).json(error.searchResponseTooLargeError()); + } + if (searchDeadline.isSearchClientDisconnectedError(err)) { metrics.outcome = 'client_disconnected'; metrics.statusCode = 499; diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 8b68cd0..0db51b4 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -24,6 +24,7 @@ const mockGetPostFilterColdSearchMaxWaitMs = jest.fn(() => 10000); const mockGetInFlightMaxWaitersPerOperation = jest.fn(() => 500); const mockGetPostFilterEvaluationChunkSize = jest.fn(() => 250); const mockGetSearchRequestDeadlineMs = jest.fn(() => 30000); +const mockGetSearchMaxResponseBytes = jest.fn(() => 10 * 1024 * 1024); const mockIsOpenSearchResponseError = jest.fn((err) => err?.code === 'OPEN_SEARCH_RESPONSE_INVALID'); const mockIsSearchProviderConfigurationError = jest.fn((err) => err?.code === 'SEARCH_PROVIDER_CONFIGURATION'); const mockIsSearchProviderContractError = jest.fn((err) => err?.code === 'SEARCH_PROVIDER_CONTRACT'); @@ -48,6 +49,7 @@ jest.mock('../utils/searchProvider', () => ({ getPostFilterColdSearchMaxWaitMs: mockGetPostFilterColdSearchMaxWaitMs, getInFlightMaxWaitersPerOperation: mockGetInFlightMaxWaitersPerOperation, getPostFilterEvaluationChunkSize: mockGetPostFilterEvaluationChunkSize, + getSearchMaxResponseBytes: mockGetSearchMaxResponseBytes, getSearchRequestDeadlineMs: mockGetSearchRequestDeadlineMs, isOpenSearchResponseError: mockIsOpenSearchResponseError, isSearchProviderConfigurationError: mockIsSearchProviderConfigurationError, @@ -64,11 +66,30 @@ const searchPostFilterCache = require('../utils/searchPostFilterCache'); function createResponse() { const res = {}; res.status = jest.fn(() => res); - res.json = jest.fn(() => res); + res.jsonMock = jest.fn(() => res); + res.json = res.jsonMock; res.set = jest.fn(() => res); return res; } +function createSerializableResponse() { + const res = { + statusCode: 200, + originalSend: jest.fn() + }; + res.status = jest.fn((statusCode) => { + res.statusCode = statusCode; + return res; + }); + res.originalSend.mockImplementation(() => res); + res.send = res.originalSend; + res.jsonMock = jest.fn((body) => res.send(JSON.stringify(body))); + res.json = res.jsonMock; + res.set = jest.fn(() => res); + res.getHeader = jest.fn(() => undefined); + return res; +} + function advancedResponse(totalValue, hits = []) { return { status: 'ok', @@ -187,6 +208,7 @@ describe('Search controller SEARCH handler', () => { mockGetPostFilterColdSearchMaxWaitMs.mockClear(); mockGetInFlightMaxWaitersPerOperation.mockClear(); mockGetPostFilterEvaluationChunkSize.mockClear(); + mockGetSearchMaxResponseBytes.mockClear(); mockGetSearchRequestDeadlineMs.mockClear(); mockIsOpenSearchResponseError.mockClear(); mockIsSearchProviderConfigurationError.mockClear(); @@ -198,6 +220,7 @@ describe('Search controller SEARCH handler', () => { mockGetPostFilterColdSearchMaxWaitMs.mockReturnValue(10000); mockGetInFlightMaxWaitersPerOperation.mockReturnValue(500); mockGetPostFilterEvaluationChunkSize.mockReturnValue(250); + mockGetSearchMaxResponseBytes.mockReturnValue(10 * 1024 * 1024); mockGetSearchRequestDeadlineMs.mockReturnValue(30000); searchPostFilterCache.clear(); searchInFlightRegistry.clear(); @@ -241,7 +264,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 3, pageSize: 1, totalPages: 3, @@ -249,6 +272,41 @@ describe('Search controller SEARCH handler', () => { }); }); + test('SEARCH converts an oversized successful payload to a sanitized 422 before sending it', async () => { + res = createSerializableResponse(); + mockGetSearchMaxResponseBytes.mockReturnValue(256); + mockApiSearch.mockResolvedValue( + advancedResponse(1, [ + { + _id: 'CVE-2024-0001', + _source: { + description: 'x'.repeat(512) + } + } + ]) + ); + + await controller.SEARCH( + { + searchParams: { + keywordSearch: 'remote code', + resultsPerPage: 1, + pageNumber: 1 + } + }, + res, + next + ); + + expect(res.status.mock.calls).toStrictEqual([[200], [422]]); + expect(res.originalSend).toHaveBeenCalledTimes(1); + expect(JSON.parse(res.originalSend.mock.calls[0][0])).toStrictEqual({ + error: 'SEARCH_RESPONSE_TOO_LARGE', + message: 'Search response exceeds the maximum response size. Reduce resultsPerPage and try again.' + }); + expect(next).not.toHaveBeenCalled(); + }); + test('SEARCH applies semantic CPE version filtering before building response metadata', async () => { const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', @@ -315,7 +373,7 @@ describe('Search controller SEARCH handler', () => { 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] }); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 1, pageSize: 1, totalPages: 1, @@ -381,7 +439,7 @@ describe('Search controller SEARCH handler', () => { }) ); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 1, pageSize: 1, totalPages: 1, @@ -398,6 +456,7 @@ describe('Search controller SEARCH handler', () => { }); test('SEARCH reuses cached semantic post-filter IDs for repeated virtualMatchString range searches', async () => { + mockShouldLogSearchPerformance.mockReturnValue(true); const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' }); @@ -427,10 +486,10 @@ describe('Search controller SEARCH handler', () => { mockSearchRaw.mockResolvedValue(rawResponse(2, [applicable, notApplicable])); mockApiSearch.mockResolvedValue(advancedResponse(1, [fullApplicable])); - await controller.SEARCH(req, res, next); + await controller.SEARCH({ requestId: 'cache-miss-request', ...req }, res, next); const secondRes = createResponse(); - await controller.SEARCH(req, secondRes, next); + await controller.SEARCH({ requestId: 'cache-hit-request', ...req }, secondRes, next); expect(mockSearchRaw).toHaveBeenCalledTimes(1); expect(mockApiSearch).toHaveBeenCalledTimes(2); @@ -448,7 +507,7 @@ describe('Search controller SEARCH handler', () => { }); expect(mockApiSearch.mock.calls[1][0].sort).toBeUndefined(); expect(secondRes.status).toHaveBeenCalledWith(200); - expect(secondRes.json).toHaveBeenCalledWith({ + expect(secondRes.jsonMock).toHaveBeenCalledWith({ page: 1, pageSize: 1, totalPages: 1, @@ -463,6 +522,24 @@ describe('Search controller SEARCH handler', () => { } } }); + + const performanceLogs = console.log.mock.calls.map(([entry]) => JSON.parse(entry)); + expect(performanceLogs.find(({ requestId }) => requestId === 'cache-miss-request')).toMatchObject({ + cacheStatus: 'miss', + openSearchOperations: ['cpe-candidate', 'page-fetch'], + openSearchElapsedMs: expect.any(Number), + postFilterElapsedMs: expect.any(Number) + }); + expect(performanceLogs.find(({ requestId }) => requestId === 'cache-hit-request')).toMatchObject({ + cacheStatus: 'hit', + openSearchOperations: ['page-fetch'] + }); + expect(performanceLogs.find(({ requestId }) => requestId === 'cache-hit-request')).not.toHaveProperty( + 'openSearchElapsedMs' + ); + expect(performanceLogs.find(({ requestId }) => requestId === 'cache-hit-request')).not.toHaveProperty( + 'postFilterElapsedMs' + ); }); test.each([ @@ -471,6 +548,7 @@ describe('Search controller SEARCH handler', () => { ['malformed', ['CVE-2024-0001'], [{ _id: 'CVE-2024-0001', _source: { cveMetadata: {} } }]], ['unexpected', ['CVE-2024-0001'], [fullCveHit('CVE-2024-9999')]] ])('SEARCH fails closed for a %s full-record page response', async (_caseName, candidateIds, pageHits) => { + mockShouldLogSearchPerformance.mockReturnValue(true); const candidates = candidateIds.map((id) => cpeHit(id, { criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' @@ -493,7 +571,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); @@ -506,7 +584,9 @@ describe('Search controller SEARCH handler', () => { requestId: `${_caseName}-page-fetch-request`, outcome: 'upstream_invalid_response', statusCode: 502, - upstreamFailureType: 'incomplete_page_fetch' + upstreamFailureType: 'incomplete_page_fetch', + openSearchOperations: ['cpe-candidate', 'page-fetch'], + pageFetchElapsedMs: expect.any(Number) }); expect(next).not.toHaveBeenCalled(); }); @@ -528,7 +608,7 @@ describe('Search controller SEARCH handler', () => { expect(mockSearchOpenSearch).not.toHaveBeenCalled(); expect(res.status).not.toHaveBeenCalled(); - expect(res.json).not.toHaveBeenCalled(); + expect(res.jsonMock).not.toHaveBeenCalled(); expect(next).toHaveBeenCalledWith(expect.any(TypeError)); } ); @@ -558,7 +638,7 @@ describe('Search controller SEARCH handler', () => { expect(mockApiSearch).toHaveBeenCalledTimes(2); expect(res.status).toHaveBeenCalledWith(502); expect(secondRes.status).toHaveBeenCalledWith(200); - expect(secondRes.json).toHaveBeenCalledWith( + expect(secondRes.jsonMock).toHaveBeenCalledWith( expect.objectContaining({ page: 1, pageSize: 1, @@ -568,6 +648,7 @@ describe('Search controller SEARCH handler', () => { }); test('SEARCH coalesces concurrent identical cold semantic post-filter searches', async () => { + mockShouldLogSearchPerformance.mockReturnValue(true); const candidateSearch = deferred(); const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' @@ -583,26 +664,26 @@ describe('Search controller SEARCH handler', () => { } } }; - const req = { - searchParams: { - virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', - versionStart: '2.2', - versionStartType: 'including', - versionEnd: '2.6', - versionEndType: 'excluding', - resultsPerPage: '1', - pageNumber: '1' - } + const searchParams = { + virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', + versionStart: '2.2', + versionStartType: 'including', + versionEnd: '2.6', + versionEndType: 'excluding', + resultsPerPage: '1', + pageNumber: '1' }; + const firstReq = { requestId: 'first-shared-request', searchParams }; + const secondReq = { requestId: 'second-shared-request', searchParams }; const firstRes = createResponse(); const secondRes = createResponse(); mockSearchRaw.mockReturnValue(candidateSearch.promise); mockApiSearch.mockResolvedValue(advancedResponse(1, [fullApplicable])); - const firstSearch = controller.SEARCH(req, firstRes, next); + const firstSearch = controller.SEARCH(firstReq, firstRes, next); await Promise.resolve(); - const secondSearch = controller.SEARCH(req, secondRes, next); + const secondSearch = controller.SEARCH(secondReq, secondRes, next); candidateSearch.resolve(rawResponse(2, [applicable, notApplicable])); await Promise.all([firstSearch, secondSearch]); @@ -611,7 +692,7 @@ describe('Search controller SEARCH handler', () => { expect(mockApiSearch).toHaveBeenCalledTimes(1); expect(firstRes.status).toHaveBeenCalledWith(200); expect(secondRes.status).toHaveBeenCalledWith(200); - expect(secondRes.json).toHaveBeenCalledWith({ + expect(secondRes.jsonMock).toHaveBeenCalledWith({ page: 1, pageSize: 1, totalPages: 1, @@ -626,9 +707,31 @@ describe('Search controller SEARCH handler', () => { } } }); + + const performanceLogs = console.log.mock.calls.map(([entry]) => JSON.parse(entry)); + const firstLog = performanceLogs.find(({ requestId }) => requestId === 'first-shared-request'); + const secondLog = performanceLogs.find(({ requestId }) => requestId === 'second-shared-request'); + expect(firstLog).toMatchObject({ + cacheStatus: 'miss', + inFlightHit: false, + openSearchOperations: ['cpe-candidate', 'page-fetch'], + openSearchElapsedMs: expect.any(Number), + postFilterElapsedMs: expect.any(Number), + pageFetchElapsedMs: expect.any(Number) + }); + expect(secondLog).toMatchObject({ + cacheStatus: 'in_flight', + inFlightHit: true, + openSearchOperations: ['cpe-candidate', 'page-fetch'], + openSearchElapsedMs: expect.any(Number), + postFilterElapsedMs: expect.any(Number), + pageFetchElapsedMs: expect.any(Number), + inFlightWaitElapsedMs: expect.any(Number) + }); }); test('SEARCH keeps shared CPE work active when one client disconnects', async () => { + mockShouldLogSearchPerformance.mockReturnValue(true); const candidateSearch = deferred(); const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' @@ -650,6 +753,8 @@ describe('Search controller SEARCH handler', () => { }; firstReq.searchParams = searchParams; secondReq.searchParams = searchParams; + firstReq.requestId = 'disconnected-shared-request'; + secondReq.requestId = 'surviving-shared-request'; const firstRes = createResponse(); const secondRes = createResponse(); let operationSignal; @@ -676,6 +781,18 @@ describe('Search controller SEARCH handler', () => { expect(mockApiSearch).toHaveBeenCalledTimes(1); expect(firstRes.status).not.toHaveBeenCalled(); expect(secondRes.status).toHaveBeenCalledWith(200); + + const performanceLogs = console.log.mock.calls.map(([entry]) => JSON.parse(entry)); + expect(performanceLogs.find(({ requestId }) => requestId === 'disconnected-shared-request')).toMatchObject({ + outcome: 'client_disconnected', + openSearchOperations: [] + }); + expect(performanceLogs.find(({ requestId }) => requestId === 'surviving-shared-request')).toMatchObject({ + outcome: 'success', + openSearchOperations: ['cpe-candidate', 'page-fetch'], + openSearchElapsedMs: expect.any(Number), + postFilterElapsedMs: expect.any(Number) + }); }); test('SEARCH returns 429 when the cold semantic post-filter queue is full', async () => { @@ -715,7 +832,7 @@ describe('Search controller SEARCH handler', () => { expect(secondRes.set).toHaveBeenCalledWith('Retry-After', '5'); expect(secondRes.status).toHaveBeenCalledWith(429); - expect(secondRes.json).toHaveBeenCalledWith({ + expect(secondRes.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_QUEUE_FULL', message: 'Search service is busy. Please retry later.' }); @@ -782,7 +899,7 @@ describe('Search controller SEARCH handler', () => { expect(secondRes.set).toHaveBeenCalledWith('Retry-After', '5'); expect(secondRes.status).toHaveBeenCalledWith(429); - expect(secondRes.json).toHaveBeenCalledWith({ + expect(secondRes.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_QUEUE_TIMEOUT', message: 'Search service is busy. Please retry later.' }); @@ -828,7 +945,7 @@ describe('Search controller SEARCH handler', () => { expect(secondRes.set).toHaveBeenCalledWith('Retry-After', '5'); expect(secondRes.status).toHaveBeenCalledWith(429); - expect(secondRes.json).toHaveBeenCalledWith({ + expect(secondRes.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_IN_FLIGHT_WAITER_LIMIT', message: 'Search service is busy. Please retry later.' }); @@ -848,6 +965,7 @@ describe('Search controller SEARCH handler', () => { }); test('SEARCH returns sanitized upstream errors when optimized post-filter search fails', async () => { + mockShouldLogSearchPerformance.mockReturnValue(true); mockSearchRaw.mockRejectedValue({ name: 'ResponseError', message: 'candidate search failed', @@ -858,6 +976,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { + requestId: 'candidate-failure-request', searchParams: { virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809', resultsPerPage: '1', @@ -870,10 +989,16 @@ describe('Search controller SEARCH handler', () => { expect(mockApiSearch).not.toHaveBeenCalled(); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); + expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ + requestId: 'candidate-failure-request', + outcome: 'upstream_error', + openSearchOperations: ['cpe-candidate'], + openSearchElapsedMs: expect.any(Number) + }); expect(next).not.toHaveBeenCalled(); }); @@ -900,7 +1025,7 @@ describe('Search controller SEARCH handler', () => { expect(mockSearchRaw).toHaveBeenCalledTimes(2); for (const response of [res, secondRes]) { expect(response.status).toHaveBeenCalledWith(502); - expect(response.json).toHaveBeenCalledWith({ + expect(response.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); @@ -932,7 +1057,7 @@ describe('Search controller SEARCH handler', () => { expect(mockSearchRaw).toHaveBeenCalledTimes(2); for (const response of [res, secondRes]) { expect(response.status).toHaveBeenCalledWith(502); - expect(response.json).toHaveBeenCalledWith({ + expect(response.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); @@ -993,7 +1118,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 0, pageSize: 0, totalPages: 0, @@ -1017,7 +1142,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 2, pageSize: 0, totalPages: 1, @@ -1041,7 +1166,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 333, pageSize: 1, totalPages: 333, @@ -1069,7 +1194,7 @@ describe('Search controller SEARCH handler', () => { expect(res.status).toHaveBeenCalledWith(503); expect(res.set).toHaveBeenCalledWith('Retry-After', '5'); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'OPEN_SEARCH_ERROR', message: 'Search backend is temporarily unavailable. Please try again later.' }); @@ -1104,7 +1229,7 @@ describe('Search controller SEARCH handler', () => { expect(res.status).toHaveBeenCalledWith(503); expect(res.set).toHaveBeenCalledWith('Retry-After', '5'); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'OPEN_SEARCH_ERROR', message: 'Search backend is temporarily unavailable. Please try again later.' }); @@ -1137,7 +1262,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(504); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_TIMEOUT', message: 'Search request timed out. Please try again later.' }); @@ -1169,7 +1294,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); @@ -1199,7 +1324,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(504); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_TIMEOUT', message: 'Search request timed out. Please try again later.' }); @@ -1236,7 +1361,7 @@ describe('Search controller SEARCH handler', () => { expect(mockGetSearchRequestDeadlineMs).toHaveBeenCalledTimes(1); expect(res.status).toHaveBeenCalledWith(504); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_TIMEOUT', message: 'Search request timed out. Please try again later.' }); @@ -1267,7 +1392,7 @@ describe('Search controller SEARCH handler', () => { await search; expect(res.status).not.toHaveBeenCalled(); - expect(res.json).not.toHaveBeenCalled(); + expect(res.jsonMock).not.toHaveBeenCalled(); expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ event: 'search.performance', requestId: 'client-disconnect-request', @@ -1299,7 +1424,7 @@ describe('Search controller SEARCH handler', () => { expect(mockApiSearch).not.toHaveBeenCalled(); expect(res.status).not.toHaveBeenCalled(); - expect(res.json).not.toHaveBeenCalled(); + expect(res.jsonMock).not.toHaveBeenCalled(); }); test('SEARCH returns sanitized errors from cve-core', async () => { @@ -1323,7 +1448,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); @@ -1351,7 +1476,7 @@ describe('Search controller SEARCH handler', () => { expect(next).toHaveBeenCalledWith(err); expect(res.status).not.toHaveBeenCalled(); - expect(res.json).not.toHaveBeenCalled(); + expect(res.jsonMock).not.toHaveBeenCalled(); expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ event: 'search.performance', requestId: 'internal-error-request', @@ -1425,7 +1550,7 @@ describe('Search controller WEB_SEARCH handler', () => { size: 10 }); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ searchMetadata: { searchStatus: 'ok', version: '2025-08-26', @@ -1570,7 +1695,7 @@ describe('Search controller WEB_SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json.mock.calls[0][0]).toStrictEqual({ + expect(res.jsonMock.mock.calls[0][0]).toStrictEqual({ searchMetadata: { searchStatus: 'errors', version: '2025-08-26', @@ -1604,7 +1729,7 @@ describe('Search controller WEB_SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json.mock.calls[0][0]).toStrictEqual({ + expect(res.jsonMock.mock.calls[0][0]).toStrictEqual({ searchMetadata: { searchStatus: 'errors', version: '2025-08-26', @@ -1641,7 +1766,7 @@ describe('Search controller WEB_SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(504); - expect(res.json.mock.calls[0][0]).toStrictEqual({ + expect(res.jsonMock.mock.calls[0][0]).toStrictEqual({ searchMetadata: { searchStatus: 'errors', version: '2025-08-26', diff --git a/package-lock.json b/package-lock.json index ca79573..9115778 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2392,9 +2392,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", - "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" diff --git a/package.json b/package.json index 406ea41..127a2db 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ }, "overrides": { "axios": "1.18.0", - "brace-expansion": "5.0.8", + "brace-expansion": "5.0.9", "js-yaml": "4.3.0", "lodash": "4.18.1", "lodash.unset": "4.18.0", diff --git a/routes/index.js b/routes/index.js index ce80d39..1314788 100644 --- a/routes/index.js +++ b/routes/index.js @@ -343,6 +343,24 @@ content: { } } } +#swagger.responses[422] = { +description: 'The serialized successful response exceeded the configured maximum size; reduce resultsPerPage and retry', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} #swagger.responses[429] = { description: 'Application rate limit exceeded, the cold CPE search queue is full or exceeded its maximum wait, or a shared semantic search operation reached its waiter limit', headers: { diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index 68ab08c..c5ca3cc 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -2,6 +2,7 @@ const { request } = require('../test/http'); const mockSearchOpenSearch = jest.fn(); const mockBasicSearch = jest.fn(); +const mockGetSearchMaxResponseBytes = jest.fn(() => 10 * 1024 * 1024); jest.mock('../utils/searchProvider', () => ({ getBasicSearchManager: jest.fn(() => ({ @@ -9,6 +10,7 @@ jest.mock('../utils/searchProvider', () => ({ })), getInFlightMaxWaitersPerOperation: jest.fn(() => 500), getPostFilterEvaluationChunkSize: jest.fn(() => 250), + getSearchMaxResponseBytes: mockGetSearchMaxResponseBytes, getSearchRequestDeadlineMs: jest.fn(() => 30000), isOpenSearchResponseError: jest.fn(() => false), isSearchProviderConfigurationError: jest.fn(() => false), @@ -53,6 +55,8 @@ describe('search route integration', () => { beforeEach(() => { mockSearchOpenSearch.mockReset(); mockBasicSearch.mockReset(); + mockGetSearchMaxResponseBytes.mockReset(); + mockGetSearchMaxResponseBytes.mockReturnValue(10 * 1024 * 1024); }); afterEach(() => { @@ -151,6 +155,46 @@ describe('search route integration', () => { expect(requestErrorLog.mock.calls[0][0]).not.toContain('sensitive'); }); + test('POST /search returns one complete 422 response when the successful payload exceeds its byte limit', async () => { + const performanceLog = jest.spyOn(console, 'log').mockImplementation(() => {}); + mockGetSearchMaxResponseBytes.mockReturnValue(256); + mockSearchOpenSearch.mockResolvedValue( + advancedResponse(1, [ + { + _id: 'CVE-2024-0001', + _source: { + description: 'x'.repeat(512) + } + } + ]).data + ); + + const response = await request(createApp(), { + method: 'POST', + path: '/search', + body: { + keywordSearch: 'remote code', + resultsPerPage: 1, + pageNumber: 1 + } + }); + + expect(response.status).toBe(422); + expect(response.body).toStrictEqual({ + error: 'SEARCH_RESPONSE_TOO_LARGE', + message: 'Search response exceeds the maximum response size. Reduce resultsPerPage and try again.' + }); + expect(response.text).toBe(JSON.stringify(response.body)); + expect(JSON.parse(performanceLog.mock.calls[0][0])).toMatchObject({ + event: 'search.performance', + outcome: 'response_too_large', + statusCode: 422, + maxResponseBytes: 256, + attemptedResponseBytes: expect.any(Number), + attemptedResponseSerializationElapsedMs: expect.any(Number) + }); + }); + test('POST /search rejects unsupported CPE parts before invoking OpenSearch', async () => { const response = await request(createApp(), { method: 'POST', @@ -169,6 +213,23 @@ describe('search route integration', () => { expect(mockSearchOpenSearch).not.toHaveBeenCalled(); }); + test('POST /search rejects a malformed CWE identifier before invoking OpenSearch', async () => { + const response = await request(createApp(), { + method: 'POST', + path: '/search', + body: { + cweId: 'CWE-079' + } + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'CWE_ID_FORMAT', + message: 'The cweId parameter must use the format CWE- without leading zeros.' + }); + expect(mockSearchOpenSearch).not.toHaveBeenCalled(); + }); + test('POST /webSearch normalizes body fields and formats BasicSearchManager responses', async () => { mockBasicSearch.mockResolvedValue(basicResponse({ total: 1, hits: [{ _id: 'CVE-2024-0002' }] })); diff --git a/routes/swagger.js b/routes/swagger.js index 0629922..697cbef 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -43,10 +43,12 @@ const searchRequestProperties = { default: false }), cweId: searchRequestProperty( - 'Returns CVE records whose CNA data includes the provided Common Weakness Enumeration ID. ADP-provided CWE data is not searched.', + 'Returns CVE records whose CNA data includes the provided Common Weakness Enumeration ID. The accepted format is the CWE- prefix followed by a positive integer without leading zeros. Prefix matching is case-insensitive, surrounding whitespace is trimmed, and valid input is normalized to uppercase before search. Invalid values return HTTP 400. ADP-provided CWE data is not searched.', { type: 'string', + minLength: 5, maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cweId, + pattern: constants.CWE_ID_PATTERN, example: 'CWE-79' } ), diff --git a/test/contract/opensearch.js b/test/contract/opensearch.js index 56e71a8..3492d08 100644 --- a/test/contract/opensearch.js +++ b/test/contract/opensearch.js @@ -2,6 +2,7 @@ const assert = require('assert/strict'); require('dotenv').config(); const REQUIRED_ENVIRONMENT_VARIABLES = ['OpenSearchDomainEndpoint', 'OpenSearchCveIndex']; +const CPE_CRITERIA_KEYWORD_FIELD = 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword'; const FIELD_CAPABILITY_REQUIREMENTS = Object.freeze([ { name: 'cve-id-keyword', @@ -26,7 +27,7 @@ const FIELD_CAPABILITY_REQUIREMENTS = Object.freeze([ name: 'cpe-criteria-keyword', anyOf: [ { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword', + field: CPE_CRITERIA_KEYWORD_FIELD, types: ['keyword'] } ] @@ -61,23 +62,26 @@ function buildStandardBody(queryBuilder, params, options = {}) { } function buildSourceProjectionCheck(queryBuilder = require('../../utils/searchQueryBuilder')) { + const request = queryBuilder.buildSearchRequest({ + resultsPerPage: 1, + pageNumber: 1 + }); + const body = queryBuilder.buildCpeCandidateSearchBody(request.searchOptions, request.queryStringObjects); + body.query.bool.filter.push({ + exists: { + field: CPE_CRITERIA_KEYWORD_FIELD + } + }); + return { name: 'source-projection', - body: buildStandardBody( - queryBuilder, - { - resultsPerPage: 1, - pageNumber: 1 - }, - { - sourceIncludes: ['cveMetadata.cveId'] - } - ) + body }; } -function buildContractChecks(contractCveId, queryBuilder = require('../../utils/searchQueryBuilder')) { - assert.equal(typeof contractCveId, 'string'); +function buildContractChecks(contractCandidate, queryBuilder = require('../../utils/searchQueryBuilder')) { + assert.equal(typeof contractCandidate?.cveId, 'string'); + assert.equal(typeof contractCandidate?.criteria, 'string'); const buildBody = (params, options = {}) => { return buildStandardBody(queryBuilder, params, options); }; @@ -91,7 +95,7 @@ function buildContractChecks(contractCveId, queryBuilder = require('../../utils/ name: 'cve-id-keyword-and-total', body: buildBody( { - cveIds: contractCveId, + cveIds: contractCandidate.cveId, resultsPerPage: 1, pageNumber: 1 }, @@ -103,7 +107,7 @@ function buildContractChecks(contractCveId, queryBuilder = require('../../utils/ validate(body) { assert.equal(Number.isInteger(body.hits.total.value), true); assert.equal(body.hits.total.value >= 1, true); - assert.equal(body.hits.hits[0]?._source?.cveMetadata?.cveId, contractCveId); + assert.equal(body.hits.hits[0]?._source?.cveMetadata?.cveId, contractCandidate.cveId); } }, { @@ -134,11 +138,14 @@ function buildContractChecks(contractCveId, queryBuilder = require('../../utils/ name: 'cpe-candidate', body: { ...buildCandidateBody({ - virtualMatchString: 'cpe:2.3:a:contract-probe:contract-probe', + virtualMatchString: contractCandidate.criteria, resultsPerPage: 1, pageNumber: 1 }), - size: 0 + size: 1 + }, + async validate(body) { + await validateCpeCandidateResponse(body, contractCandidate.criteria); } } ]; @@ -165,19 +172,49 @@ function validateFieldCapabilities(body, requirements = FIELD_CAPABILITY_REQUIRE }); } -function getContractCveId(body) { +function getCandidateCpeMatch(hit) { + const cpeApplicability = hit._source.containers.cna.cpeApplicability; + const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; + + return applicabilityItems[0].nodes[0].cpeMatch[0]; +} + +async function getContractCandidate(body, candidateProcessor = require('../../utils/searchCandidateProcessor')) { assert.equal(body.hits.hits.length, 1, 'source projection did not return a CVE record'); const [hit] = body.hits.hits; + const processed = await candidateProcessor.filterSearchHitsInChunks({ data: body }, {}); const sourceCveId = hit?._source?.cveMetadata?.cveId; + const criteria = getCandidateCpeMatch(hit).criteria; assert.equal(typeof hit?._id, 'string'); assert.equal(typeof sourceCveId, 'string'); - return sourceCveId; + assert.equal(processed.filteredHits.length, 1, 'source projection did not return a valid CPE candidate'); + assert.equal(typeof criteria, 'string'); + + return { + cveId: sourceCveId, + criteria + }; +} + +async function validateCpeCandidateResponse( + body, + virtualMatchString, + candidateProcessor = require('../../utils/searchCandidateProcessor') +) { + assert.equal(body.hits.hits.length >= 1, true, 'cpe-candidate response did not return a CVE record'); + const processed = await candidateProcessor.filterSearchHitsInChunks( + { data: body }, + { + virtualMatchString + } + ); + assert.equal(processed.filteredHits.length >= 1, true, 'cpe-candidate response did not contain a matching CPE'); } -function validateContractResponse(check, body) { +async function validateContractResponse(check, body) { assert.ok(Array.isArray(body.hits.hits), `${check.name} response did not include hits.hits`); - check.validate?.(body); + await check.validate?.(body); } async function run({ @@ -199,15 +236,15 @@ async function run({ const sourceProjectionBody = await searchProvider.searchOpenSearch(sourceProjectionCheck.body, { operation: `contract-${sourceProjectionCheck.name}` }); - validateContractResponse(sourceProjectionCheck, sourceProjectionBody); - const contractCveId = getContractCveId(sourceProjectionBody); + await validateContractResponse(sourceProjectionCheck, sourceProjectionBody); + const contractCandidate = await getContractCandidate(sourceProjectionBody); completedChecks.push(sourceProjectionCheck.name); - for (const check of buildContractChecks(contractCveId, queryBuilder)) { + for (const check of buildContractChecks(contractCandidate, queryBuilder)) { const body = await searchProvider.searchOpenSearch(check.body, { operation: check.operation ?? `contract-${check.name}` }); - validateContractResponse(check, body); + await validateContractResponse(check, body); completedChecks.push(check.name); } @@ -235,11 +272,12 @@ module.exports = { buildSourceProjectionCheck, buildStandardBody, FIELD_CAPABILITY_REQUIREMENTS, - getContractCveId, + getContractCandidate, getSanitizedFailure, getRequiredFieldNames, requireConfiguration, run, + validateCpeCandidateResponse, validateFieldCapabilities, validateContractResponse }; diff --git a/test/contract/opensearch.test.js b/test/contract/opensearch.test.js index e95bb39..785c72a 100644 --- a/test/contract/opensearch.test.js +++ b/test/contract/opensearch.test.js @@ -17,6 +17,53 @@ function validResponse(overrides = {}) { }; } +function validCandidateHit() { + return { + _id: 'CVE-2024-0001', + _source: { + cveMetadata: { + cveId: 'CVE-2024-0001' + }, + containers: { + cna: { + cpeApplicability: [ + { + nodes: [ + { + cpeMatch: [ + { + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true + } + ] + } + ] + } + ] + } + } + } + }; +} + +function validCandidateResponse(hit = validCandidateHit()) { + return validResponse({ + hits: { + total: { + value: 1, + relation: 'eq' + }, + hits: [hit] + } + }); +} + +function mutateCandidate(mutator) { + const hit = validCandidateHit(); + mutator(hit); + return hit; +} + function validFieldCapabilities() { return { fields: Object.fromEntries( @@ -55,7 +102,10 @@ describe('OpenSearch contract check', () => { test('builds representative read-only production query shapes', () => { const sourceProjection = contract.buildSourceProjectionCheck(); - const checks = contract.buildContractChecks('CVE-2024-0001'); + const checks = contract.buildContractChecks({ + cveId: 'CVE-2024-0001', + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*' + }); expect([sourceProjection, ...checks].map(({ name }) => name)).toStrictEqual([ 'source-projection', @@ -66,7 +116,30 @@ describe('OpenSearch contract check', () => { 'cpe-candidate' ]); expect([sourceProjection, ...checks].every(({ body }) => body.size <= 1)).toBe(true); - expect(sourceProjection.body._source).toStrictEqual(['cveMetadata.cveId']); + expect(sourceProjection.body).toMatchObject({ + size: 1, + track_total_hits: false, + query: { + bool: { + filter: [ + { + exists: { + field: 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword' + } + } + ] + } + } + }); + expect(sourceProjection.body._source).toStrictEqual([ + 'cveMetadata.cveId', + 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria', + 'containers.cna.cpeApplicability.nodes.cpeMatch.vulnerable', + 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartIncluding', + 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartExcluding', + 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndIncluding', + 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndExcluding' + ]); expect(checks.find(({ name }) => name === 'cve-id-keyword-and-total').body).toMatchObject({ track_total_hits: true, sort: [ @@ -85,7 +158,7 @@ describe('OpenSearch contract check', () => { ]); expect(checks.find(({ name }) => name === 'cvss').body.query.bool.filter[0]).toHaveProperty('bool.should'); expect(checks.find(({ name }) => name === 'cpe-candidate').body).toMatchObject({ - size: 0, + size: 1, track_total_hits: false }); }); @@ -106,28 +179,76 @@ describe('OpenSearch contract check', () => { ); }); - test('requires source projection to return a concrete CVE record', () => { - expect(() => contract.getContractCveId(validResponse())).toThrow( + test('requires source projection to return a concrete CPE candidate', async () => { + await expect(contract.getContractCandidate(validResponse())).rejects.toThrow( 'source projection did not return a CVE record' ); - expect( - contract.getContractCveId( - validResponse({ - hits: { - hits: [ - { - _id: 'record-1', - _source: { - cveMetadata: { - cveId: 'CVE-2024-0001' - } - } - } - ] - } - }) - ) - ).toBe('CVE-2024-0001'); + await expect(contract.getContractCandidate(validCandidateResponse())).resolves.toStrictEqual({ + cveId: 'CVE-2024-0001', + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*' + }); + }); + + test.each([ + [ + 'missing CVE ID', + mutateCandidate((hit) => { + delete hit._source.cveMetadata.cveId; + }) + ], + [ + 'missing CPE applicability', + mutateCandidate((hit) => { + delete hit._source.containers.cna.cpeApplicability; + }) + ], + [ + 'missing CPE nodes', + mutateCandidate((hit) => { + delete hit._source.containers.cna.cpeApplicability[0].nodes; + }) + ], + [ + 'missing CPE matches', + mutateCandidate((hit) => { + delete hit._source.containers.cna.cpeApplicability[0].nodes[0].cpeMatch; + }) + ], + [ + 'invalid CPE criteria', + mutateCandidate((hit) => { + hit._source.containers.cna.cpeApplicability[0].nodes[0].cpeMatch[0].criteria = 'not-a-cpe'; + }) + ], + [ + 'missing vulnerable value', + mutateCandidate((hit) => { + delete hit._source.containers.cna.cpeApplicability[0].nodes[0].cpeMatch[0].vulnerable; + }) + ], + [ + 'invalid version range', + mutateCandidate((hit) => { + hit._source.containers.cna.cpeApplicability[0].nodes[0].cpeMatch[0].versionEndExcluding = 2; + }) + ] + ])('rejects a source projection with %s', async (_caseName, hit) => { + await expect(contract.getContractCandidate(validCandidateResponse(hit))).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_candidate' + }); + }); + + test('requires the CPE candidate query to return a matching candidate', async () => { + await expect( + contract.validateCpeCandidateResponse(validCandidateResponse(), 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*') + ).resolves.toBeUndefined(); + await expect( + contract.validateCpeCandidateResponse(validResponse(), 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*') + ).rejects.toThrow('cpe-candidate response did not return a CVE record'); + await expect( + contract.validateCpeCandidateResponse(validCandidateResponse(), 'cpe:2.3:a:other:product:*:*:*:*:*:*:*:*') + ).rejects.toThrow('cpe-candidate response did not contain a matching CPE'); }); test('runs every contract check and closes the provider', async () => { @@ -138,6 +259,9 @@ describe('OpenSearch contract check', () => { checkOpenSearchReadiness: jest.fn().mockResolvedValue(true), getOpenSearchFieldCapabilities: jest.fn().mockResolvedValue(validFieldCapabilities()), searchOpenSearch: jest.fn().mockImplementation((_body, { operation }) => { + if (operation === 'contract-source-projection' || operation === 'contract-cpe-candidate') { + return Promise.resolve(validCandidateResponse()); + } if (operation === 'standard') { return Promise.resolve( validResponse({ @@ -160,24 +284,6 @@ describe('OpenSearch contract check', () => { }) ); } - if (operation === 'contract-source-projection') { - return Promise.resolve( - validResponse({ - hits: { - hits: [ - { - _id: 'CVE-2024-0001', - _source: { - cveMetadata: { - cveId: 'CVE-2024-0001' - } - } - } - ] - } - }) - ); - } return Promise.resolve(validResponse()); }), closeSearchProviderClients: jest.fn().mockResolvedValue(undefined) diff --git a/utils/constants.js b/utils/constants.js index 8a1b1c7..b98fb65 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -8,6 +8,7 @@ const MAX_CVE_IDS = 100; const MIN_CVE_ID_SEQUENCE_DIGITS = 4; const MAX_CVE_ID_SEQUENCE_DIGITS = 19; const MAX_KEYWORD_SEARCH_TOKENS = 64; +const CWE_ID_PATTERN = '^\\s*[Cc][Ww][Ee]-[1-9][0-9]*\\s*$'; const JSON_REQUEST_BODY_LIMIT_KB = 100; const JSON_REQUEST_BODY_LIMIT_BYTES = JSON_REQUEST_BODY_LIMIT_KB * 1024; const DATE_TIME_PATTERN = @@ -31,6 +32,7 @@ const SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE = 250; const SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS = 20 * 1000; const SEARCH_REQUEST_DEADLINE_MS = 30 * 1000; const SEARCH_READINESS_TIMEOUT_MS = 5 * 1000; +const SEARCH_MAX_RESPONSE_BYTES = 10 * 1024 * 1024; const RUNTIME_SHUTDOWN_TIMEOUT_MS = 35 * 1000; const HTTP_HEADERS_TIMEOUT_MS = 10 * 1000; const HTTP_REQUEST_TIMEOUT_MS = 15 * 1000; @@ -141,6 +143,7 @@ module.exports = { MIN_CVE_ID_SEQUENCE_DIGITS, MAX_CVE_ID_SEQUENCE_DIGITS, MAX_KEYWORD_SEARCH_TOKENS, + CWE_ID_PATTERN, JSON_REQUEST_BODY_LIMIT_KB, JSON_REQUEST_BODY_LIMIT_BYTES, DATE_TIME_PATTERN, @@ -155,6 +158,7 @@ module.exports = { SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS, SEARCH_REQUEST_DEADLINE_MS, SEARCH_READINESS_TIMEOUT_MS, + SEARCH_MAX_RESPONSE_BYTES, RUNTIME_SHUTDOWN_TIMEOUT_MS, HTTP_HEADERS_TIMEOUT_MS, HTTP_REQUEST_TIMEOUT_MS, diff --git a/utils/cpe.js b/utils/cpe.js index dd75191..6402b6f 100644 --- a/utils/cpe.js +++ b/utils/cpe.js @@ -34,6 +34,9 @@ function parseCpe23(input) { if (components.length < 4 || components.length > CPE_COMPONENT_COUNT) { return { valid: false, reason: 'component_count' }; } + if (components.some((component) => component.trim() === '')) { + return { valid: false, reason: 'empty_component' }; + } if (!isValidCpePart(components[0], { allowLogicalValues: true })) { return { valid: false, reason: 'part' }; } diff --git a/utils/cpe.test.js b/utils/cpe.test.js index b8317a2..3579723 100644 --- a/utils/cpe.test.js +++ b/utils/cpe.test.js @@ -31,6 +31,19 @@ describe('CPE helpers', () => { ]); }); + test.each([ + 'cpe:2.3::vendor:product:1.0', + 'cpe:2.3:a:vendor::1.0', + 'cpe:2.3:a:vendor:product:1.0:', + 'cpe:2.3:a:vendor:product:1.0::*', + 'cpe:2.3:a:vendor:product:1.0: :*' + ])('parseCpeName rejects explicit empty component in %s', (value) => { + expect(cpe.parseCpeName(value)).toStrictEqual({ + valid: false, + reason: 'empty_component' + }); + }); + test.each([ 'not-a-cpe', 'cpe:2.3:a:vendor:product', diff --git a/utils/cweId.js b/utils/cweId.js new file mode 100644 index 0000000..045965b --- /dev/null +++ b/utils/cweId.js @@ -0,0 +1,20 @@ +const constants = require('./constants'); + +const CWE_ID_PATTERN = new RegExp(constants.CWE_ID_PATTERN); + +function normalizeCweId(value) { + return typeof value === 'string' ? value.trim().toUpperCase() : ''; +} + +function isValidCweId(value) { + return ( + typeof value === 'string' && + value.length <= constants.SEARCH_STRING_LENGTH_LIMITS.cweId && + CWE_ID_PATTERN.test(value) + ); +} + +module.exports = { + isValidCweId, + normalizeCweId +}; diff --git a/utils/cweId.test.js b/utils/cweId.test.js new file mode 100644 index 0000000..0f7e2f1 --- /dev/null +++ b/utils/cweId.test.js @@ -0,0 +1,33 @@ +const constants = require('./constants'); +const cweId = require('./cweId'); + +describe('cweId', () => { + test.each([ + ['canonical identifier', 'CWE-79'], + ['case-insensitive prefix', 'cwe-79'], + ['surrounding whitespace', ' CwE-123 '], + ['one-digit identifier', 'CWE-1'], + ['maximum-length identifier', `CWE-${'1'.repeat(constants.SEARCH_STRING_LENGTH_LIMITS.cweId - 4)}`] + ])('accepts a %s', (_caseName, value) => { + expect(cweId.isValidCweId(value)).toBe(true); + }); + + test.each([ + ['empty value', ''], + ['missing prefix', '79'], + ['zero identifier', 'CWE-0'], + ['leading zero', 'CWE-079'], + ['missing numeric identifier', 'CWE-'], + ['non-numeric identifier', 'CWE-noinfo'], + ['embedded whitespace', 'CWE- 79'], + ['suffix', 'CWE-79-extra'], + ['overlength identifier', `CWE-${'1'.repeat(constants.SEARCH_STRING_LENGTH_LIMITS.cweId - 3)}`], + ['non-string value', ['CWE-79']] + ])('rejects an identifier with %s', (_caseName, value) => { + expect(cweId.isValidCweId(value)).toBe(false); + }); + + test('normalizes surrounding whitespace and prefix case', () => { + expect(cweId.normalizeCweId(' cWe-79 ')).toBe('CWE-79'); + }); +}); diff --git a/utils/errors.js b/utils/errors.js index 9b442d8..3de68cc 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -123,6 +123,13 @@ class errors { ); } + cweIdFormatError() { + return makeError( + 'CWE_ID_FORMAT', + 'The cweId parameter must use the format CWE- without leading zeros.' + ); + } + noRejectedValueError() { return makeError('NO_REJECTED_VALUE', 'The noRejected field must be a boolean.'); } @@ -149,6 +156,13 @@ class errors { ); } + searchResponseTooLargeError() { + return makeError( + 'SEARCH_RESPONSE_TOO_LARGE', + 'Search response exceeds the maximum response size. Reduce resultsPerPage and try again.' + ); + } + searchQueryUnsupportedError() { return makeError( 'SEARCH_QUERY_UNSUPPORTED', diff --git a/utils/errors.test.js b/utils/errors.test.js index 582098e..e3b20fe 100644 --- a/utils/errors.test.js +++ b/utils/errors.test.js @@ -167,6 +167,14 @@ describe('error response helpers', () => { message: `The cveIds parameter cannot contain more than ${constants.MAX_CVE_IDS} CVE IDs.` } ], + [ + 'cweIdFormatError', + [], + { + error: 'CWE_ID_FORMAT', + message: 'The cweId parameter must use the format CWE- without leading zeros.' + } + ], [ 'noRejectedValueError', [], @@ -271,6 +279,14 @@ describe('error response helpers', () => { message: `The JSON request body cannot exceed ${constants.JSON_REQUEST_BODY_LIMIT_KB} KB.` } ], + [ + 'searchResponseTooLargeError', + [], + { + error: 'SEARCH_RESPONSE_TOO_LARGE', + message: 'Search response exceeds the maximum response size. Reduce resultsPerPage and try again.' + } + ], [ 'searchQueryUnsupportedError', [], diff --git a/utils/runtimeConfig.js b/utils/runtimeConfig.js index 77745c4..3558c7f 100644 --- a/utils/runtimeConfig.js +++ b/utils/runtimeConfig.js @@ -21,6 +21,7 @@ const RUNTIME_CONFIG_LIMITS = Object.freeze({ openSearchRequestTimeoutMs: Object.freeze({ min: 1, max: 5 * 60 * 1000 }), requestDeadlineMs: Object.freeze({ min: 1, max: 10 * 60 * 1000 }), readinessTimeoutMs: Object.freeze({ min: 1, max: 60 * 1000 }), + maxResponseBytes: Object.freeze({ min: 64 * 1024, max: 100 * 1024 * 1024 }), shutdownTimeoutMs: Object.freeze({ min: 1, max: 15 * 60 * 1000 }), httpHeadersTimeoutMs: Object.freeze({ min: 1, max: 60 * 1000 }), httpRequestTimeoutMs: Object.freeze({ min: 1, max: 5 * 60 * 1000 }), @@ -289,6 +290,11 @@ function resolveRuntimeConfig({ 'search.readinessTimeoutMs', readConfigValue(appConfig, 'search.readinessTimeoutMs', constants.SEARCH_READINESS_TIMEOUT_MS), RUNTIME_CONFIG_LIMITS.readinessTimeoutMs + ), + maxResponseBytes: parseInteger( + 'search.maxResponseBytes', + readConfigValue(appConfig, 'search.maxResponseBytes', constants.SEARCH_MAX_RESPONSE_BYTES), + RUNTIME_CONFIG_LIMITS.maxResponseBytes ) }, requestLogging: { @@ -398,7 +404,8 @@ function sanitizeRuntimeConfig(config) { postFilterEvaluationChunkSize: config.search.postFilterEvaluationChunkSize, openSearchRequestTimeoutMs: config.search.openSearchRequestTimeoutMs, requestDeadlineMs: config.search.requestDeadlineMs, - readinessTimeoutMs: config.search.readinessTimeoutMs + readinessTimeoutMs: config.search.readinessTimeoutMs, + maxResponseBytes: config.search.maxResponseBytes }, requestLogging: config.requestLogging, runtime: config.runtime, diff --git a/utils/runtimeConfig.test.js b/utils/runtimeConfig.test.js index 90777c4..f13350b 100644 --- a/utils/runtimeConfig.test.js +++ b/utils/runtimeConfig.test.js @@ -56,7 +56,8 @@ describe('runtimeConfig', () => { postFilterEvaluationChunkSize: constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE, openSearchRequestTimeoutMs: constants.SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS, requestDeadlineMs: constants.SEARCH_REQUEST_DEADLINE_MS, - readinessTimeoutMs: constants.SEARCH_READINESS_TIMEOUT_MS + readinessTimeoutMs: constants.SEARCH_READINESS_TIMEOUT_MS, + maxResponseBytes: constants.SEARCH_MAX_RESPONSE_BYTES }, requestLogging: { successSampleRate: 0.1, @@ -101,6 +102,7 @@ describe('runtimeConfig', () => { 'search.openSearchRequestTimeoutMs': '25000', 'search.requestDeadlineMs': '45000', 'search.readinessTimeoutMs': '7000', + 'search.maxResponseBytes': '20971520', 'requestLogging.successSampleRate': '0.25', 'requestLogging.healthChecks': 'TRUE', 'runtime.shutdownTimeoutMs': '60000', @@ -138,7 +140,8 @@ describe('runtimeConfig', () => { postFilterEvaluationChunkSize: 100, openSearchRequestTimeoutMs: 25000, requestDeadlineMs: 45000, - readinessTimeoutMs: 7000 + readinessTimeoutMs: 7000, + maxResponseBytes: 20971520 }, requestLogging: { successSampleRate: 0.25, @@ -182,6 +185,7 @@ describe('runtimeConfig', () => { ['search.postFilterColdSearchConcurrency', {}, { 'search.postFilterColdSearchConcurrency': '101' }], ['search.postFilterColdSearchMaxQueueDepth', {}, { 'search.postFilterColdSearchMaxQueueDepth': '10001' }], ['search.postFilterEvaluationChunkSize', {}, { 'search.postFilterEvaluationChunkSize': '1.5' }], + ['search.maxResponseBytes', {}, { 'search.maxResponseBytes': '65535' }], ['runtime.http.headersTimeoutMs', {}, { 'runtime.http.headersTimeoutMs': '0' }], ['runtime.http.requestTimeoutMs', {}, { 'runtime.http.requestTimeoutMs': 'invalid' }], ['runtime.http.keepAliveTimeoutMs', {}, { 'runtime.http.keepAliveTimeoutMs': '60001' }], diff --git a/utils/searchCandidateProcessor.test.js b/utils/searchCandidateProcessor.test.js index d3bd3d3..b3ba995 100644 --- a/utils/searchCandidateProcessor.test.js +++ b/utils/searchCandidateProcessor.test.js @@ -155,6 +155,10 @@ describe('searchCandidateProcessor', () => { ['nested-only CPE applicability entry', nestedOnlyCpeCandidateHit()], ['missing CPE criteria', cpeCandidateHit({ criteria: undefined })], ['invalid CPE criteria', cpeCandidateHit({ criteria: 'not-a-cpe-name' })], + [ + 'CPE criteria with an explicit empty component', + cpeCandidateHit({ criteria: 'cpe:2.3:a:vendor:product:*:*::*:*:*:*:*' }) + ], ['missing CPE vulnerability flag', cpeCandidateHit({ vulnerable: undefined })], ['invalid CPE range boundary', cpeCandidateHit({ versionEndExcluding: 2 })], [ diff --git a/utils/searchPerformance.js b/utils/searchPerformance.js index 3e365ae..1b5b1b5 100644 --- a/utils/searchPerformance.js +++ b/utils/searchPerformance.js @@ -4,6 +4,15 @@ const searchProvider = require('./searchProvider'); const EVENT_LOOP_DELAY_RESOLUTION_MS = 20; const EVENT_LOOP_DELAY_WINDOW_MS = 10000; const RESPONSE_JSON_INSTRUMENTED = Symbol('responseJsonInstrumented'); +const SHARED_OPERATION_METRIC_FIELDS = [ + 'coldSearchQueueWaitElapsedMs', + 'coldSearchQueueState', + 'openSearchElapsedMs', + 'openSearchTookMs', + 'pageFetchElapsedMs', + 'pageFetchTookMs', + 'postFilterElapsedMs' +]; let eventLoopDelaySampler; function markTime() { @@ -18,6 +27,50 @@ function roundMs(value) { return value === undefined ? undefined : Math.round(value * 100) / 100; } +async function captureOperationOutcome(run) { + const operationMetrics = {}; + + try { + return { + status: 'fulfilled', + result: await run(operationMetrics), + operationMetrics + }; + } catch (error) { + return { + status: 'rejected', + error, + operationMetrics + }; + } +} + +function mergeOperationMetrics(metrics, operationMetrics) { + for (const field of SHARED_OPERATION_METRIC_FIELDS) { + if (operationMetrics?.[field] !== undefined) { + metrics[field] = operationMetrics[field]; + } + } + + if (operationMetrics?.openSearchOperations?.length) { + metrics.openSearchOperations ??= []; + metrics.openSearchOperations.push(...operationMetrics.openSearchOperations); + } + if (Number.isFinite(operationMetrics?.maxOpenSearchActiveRequests)) { + metrics.maxOpenSearchActiveRequests = Math.max( + metrics.maxOpenSearchActiveRequests ?? 0, + operationMetrics.maxOpenSearchActiveRequests + ); + } +} + +function consumeOperationOutcome(outcome, metrics) { + mergeOperationMetrics(metrics, outcome.operationMetrics); + if (outcome.status === 'rejected') throw outcome.error; + + return outcome.result; +} + function getCacheStatus(metrics, postFilters) { if (!Object.keys(postFilters ?? {}).length) return 'not_applicable'; if (metrics.cacheHit === true) return 'hit'; @@ -102,24 +155,33 @@ function instrumentJsonResponse(res, metrics) { res[RESPONSE_JSON_INSTRUMENTED] = true; res.json = function instrumentedJson(body) { const serializationStart = markTime(); - const result = originalJson.call(this, body); - metrics.responseSerializationElapsedMs = elapsedMs(serializationStart); + try { + return originalJson.call(this, body); + } catch (err) { + if (err?.code === 'SEARCH_RESPONSE_TOO_LARGE') { + metrics.attemptedResponseSerializationElapsedMs = elapsedMs(serializationStart); + } + throw err; + } finally { + metrics.responseSerializationElapsedMs = elapsedMs(serializationStart); - const contentLength = Number(this.getHeader?.('content-length')); - if (Number.isFinite(contentLength) && contentLength >= 0) { - metrics.responseBytes = contentLength; + const contentLength = Number(this.getHeader?.('content-length')); + if (Number.isFinite(contentLength) && contentLength >= 0) { + metrics.responseBytes = contentLength; + } } - - return result; }; } function beginSearchRequest(metrics, res) { - if (!searchProvider.shouldLogSearchPerformance()) return false; + const collectDetailedMetrics = searchProvider.shouldLogSearchPerformance(); + if (collectDetailedMetrics || typeof res?.send === 'function') { + instrumentJsonResponse(res, metrics); + } + if (!collectDetailedMetrics) return false; metrics.heapUsedStartBytes = process.memoryUsage().heapUsed; getEventLoopDelaySampler(); - instrumentJsonResponse(res, metrics); return true; } @@ -159,6 +221,10 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte inFlightState: metrics.inFlightState, coldSearchQueueWaitElapsedMs: roundMs(metrics.coldSearchQueueWaitElapsedMs), coldSearchQueueState: metrics.coldSearchQueueState, + attemptedResponseBytes: metrics.attemptedResponseBytes, + maxResponseBytes: metrics.maxResponseBytes, + attemptedResponseSerializationElapsedMs: roundMs(metrics.attemptedResponseSerializationElapsedMs), + responseSerializationElapsedMs: roundMs(metrics.responseSerializationElapsedMs), totalElapsedMs: roundMs(metrics.totalElapsedMs) }; @@ -179,7 +245,6 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte cachedResultCveIdCount: metrics.cachedResultCveIdCount, cacheState: metrics.cacheState, responseBytes: metrics.responseBytes, - responseSerializationElapsedMs: roundMs(metrics.responseSerializationElapsedMs), heapUsedBytes: metrics.heapUsedBytes, heapUsedDeltaBytes: metrics.heapUsedDeltaBytes, processEventLoopDelayMeanMs: roundMs(metrics.processEventLoopDelayMeanMs), @@ -209,12 +274,15 @@ function resetForTests() { module.exports = { beginSearchRequest, + captureOperationOutcome, completeSearchRequest, + consumeOperationOutcome, createEventLoopDelaySampler, elapsedMs, getCacheStatus, logSearchPerformance, markTime, + mergeOperationMetrics, resetForTests, roundMs }; diff --git a/utils/searchPerformance.test.js b/utils/searchPerformance.test.js index 8ea90c3..37f470d 100644 --- a/utils/searchPerformance.test.js +++ b/utils/searchPerformance.test.js @@ -133,6 +133,54 @@ describe('searchPerformance', () => { sampler.stop(); }); + test('captures and merges fulfilled shared-operation metrics without copying request state', async () => { + const outcome = await searchPerformance.captureOperationOutcome(async (operationMetrics) => { + operationMetrics.cacheHit = true; + operationMetrics.openSearchOperations = ['cpe-candidate']; + operationMetrics.maxOpenSearchActiveRequests = 3; + operationMetrics.openSearchElapsedMs = 12.5; + operationMetrics.postFilterElapsedMs = 2.5; + return 'candidate-result'; + }); + const metrics = { + cacheHit: false, + openSearchOperations: ['standard'], + maxOpenSearchActiveRequests: 2 + }; + + expect(searchPerformance.consumeOperationOutcome(outcome, metrics)).toBe('candidate-result'); + expect(metrics).toStrictEqual({ + cacheHit: false, + openSearchOperations: ['standard', 'cpe-candidate'], + maxOpenSearchActiveRequests: 3, + openSearchElapsedMs: 12.5, + postFilterElapsedMs: 2.5 + }); + }); + + test('preserves the original shared-operation error after merging available metrics', async () => { + const error = new Error('candidate failed'); + const outcome = await searchPerformance.captureOperationOutcome(async (operationMetrics) => { + operationMetrics.openSearchOperations = ['cpe-candidate']; + operationMetrics.openSearchElapsedMs = 4.5; + throw error; + }); + const metrics = {}; + let caught; + + try { + searchPerformance.consumeOperationOutcome(outcome, metrics); + } catch (err) { + caught = err; + } + + expect(caught).toBe(error); + expect(metrics).toStrictEqual({ + openSearchOperations: ['cpe-candidate'], + openSearchElapsedMs: 4.5 + }); + }); + test('logSearchPerformance emits redacted search timing metadata when enabled', () => { mockShouldLogSearchPerformance.mockReturnValue(true); @@ -263,17 +311,21 @@ describe('searchPerformance', () => { expect(console.log).not.toHaveBeenCalled(); }); - test('does not instrument response or process resources when detailed logging is disabled', () => { + test('captures serialization timing but not process resources when detailed logging is disabled', () => { mockShouldLogSearchPerformance.mockReturnValue(false); const metrics = {}; - const json = jest.fn(); - const res = { json }; + const res = { + getHeader: jest.fn(() => undefined), + json: jest.fn(() => res), + send: jest.fn(() => res) + }; expect(searchPerformance.beginSearchRequest(metrics, res)).toBe(false); + res.json({ status: 'ok' }); searchPerformance.completeSearchRequest(metrics); - expect(res.json).toBe(json); - expect(metrics).toStrictEqual({}); + expect(metrics.responseSerializationElapsedMs).toEqual(expect.any(Number)); + expect(metrics.heapUsedBytes).toBeUndefined(); }); test('captures response size and process resource metrics only when detailed logging is enabled', () => { diff --git a/utils/searchProvider.js b/utils/searchProvider.js index e7ffe6a..3009b28 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -327,6 +327,10 @@ function getSearchRequestDeadlineMs() { return runtimeConfig.getRuntimeConfig().search.requestDeadlineMs; } +function getSearchMaxResponseBytes() { + return runtimeConfig.getRuntimeConfig().search.maxResponseBytes; +} + function resetSearchProviderForTests() { basicSearchManager = undefined; searchReader = undefined; @@ -350,6 +354,7 @@ module.exports = { getPostFilterColdSearchMaxWaitMs, getPostFilterEvaluationChunkSize, getOpenSearchRequestTimeoutMs, + getSearchMaxResponseBytes, getSearchRequestDeadlineMs, getSearchProviderConfig, getSearchReader, diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index 78616c5..922cfac 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -736,6 +736,7 @@ describe('searchProvider', () => { 'search.openSearchRequestTimeoutMs', 'search.requestDeadlineMs', 'search.readinessTimeoutMs', + 'search.maxResponseBytes', 'runtime.shutdownTimeoutMs' ].includes(key) ); @@ -751,6 +752,7 @@ describe('searchProvider', () => { 'search.openSearchRequestTimeoutMs': '25000', 'search.requestDeadlineMs': '45000', 'search.readinessTimeoutMs': '7000', + 'search.maxResponseBytes': '20971520', 'runtime.shutdownTimeoutMs': '60000' }; return values[key]; @@ -766,6 +768,7 @@ describe('searchProvider', () => { expect(searchProvider.getOpenSearchRequestTimeoutMs()).toBe(25000); expect(searchProvider.getSearchRequestDeadlineMs()).toBe(45000); expect(searchProvider.getOpenSearchReadinessTimeoutMs()).toBe(7000); + expect(searchProvider.getSearchMaxResponseBytes()).toBe(20971520); }); test('rejects an invalid explicit CPE post-filter runtime value', () => { diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index f30edc2..9104654 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -1,6 +1,7 @@ const constants = require('./constants'); const cpe = require('./cpe'); const cveId = require('./cveId'); +const cweId = require('./cweId'); const cvss = require('./cvss'); const { MAX_RESULTS_LIMIT, RESULTS_PER_PAGE } = constants; @@ -362,6 +363,13 @@ function getFilterFields(params) { return; } + if (param === 'cweId') { + if (!cweId.isValidCweId(params[param])) throwInvalidQueryValue(param); + + fieldsToReturn.push(buildMatchPhraseAcrossFields(cweId.normalizeCweId(params[param]), fieldSpec)); + return; + } + if (param === 'cpeName') { const filter = buildCpeNameFilter(params[param]); if (filter) fieldsToReturn.push(filter); diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index 79f9c2c..bb04d01 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -238,6 +238,7 @@ describe('searchQueryBuilder buildSearchRequest()', () => { test.each([ ['cpeName', { cpeName: 'not-a-cpe-name' }, 'Invalid cpeName value reached query construction.'], + ['cweId', { cweId: 'CWE-079' }, 'Invalid cweId value reached query construction.'], [ 'virtualMatchString', { virtualMatchString: 'not-a-cpe-match-string' }, @@ -387,7 +388,7 @@ describe('searchQueryBuilder buildSearchRequest()', () => { test('buildOpenSearchBody builds the complete standard search request', () => { const output = queryBuilder.buildSearchRequest({ keywordSearch: 'remote code', - cweId: 'CWE-79', + cweId: ' cwe-79 ', resultsPerPage: 10, pageNumber: 3 }); diff --git a/utils/searchResponseLimiter.js b/utils/searchResponseLimiter.js new file mode 100644 index 0000000..a56da95 --- /dev/null +++ b/utils/searchResponseLimiter.js @@ -0,0 +1,61 @@ +const RESPONSE_SEND_INSTRUMENTED = Symbol('responseSendInstrumented'); + +class SearchResponseTooLargeError extends Error { + constructor(responseBytes, maxResponseBytes) { + super('Search response exceeds the configured maximum response size.'); + this.name = 'SearchResponseTooLargeError'; + this.code = 'SEARCH_RESPONSE_TOO_LARGE'; + this.responseBytes = responseBytes; + this.maxResponseBytes = maxResponseBytes; + } +} + +function getBodyByteLength(body) { + if (Buffer.isBuffer(body)) return body.length; + if (typeof body === 'string') return Buffer.byteLength(body); + + return undefined; +} + +function isSuccessfulResponse(res) { + return res.statusCode >= 200 && res.statusCode < 300; +} + +function instrumentSearchResponse(res, { maxResponseBytes, metrics, checkDeadline }) { + if (typeof res?.send !== 'function' || res[RESPONSE_SEND_INSTRUMENTED]) return; + + const originalSend = res.send; + res[RESPONSE_SEND_INSTRUMENTED] = true; + res.send = function limitedSearchResponse(body) { + if (!isSuccessfulResponse(this)) { + return originalSend.call(this, body); + } + + const responseBytes = getBodyByteLength(body); + if (responseBytes === undefined) { + return originalSend.call(this, body); + } + + checkDeadline(); + + if (responseBytes > maxResponseBytes) { + metrics.attemptedResponseBytes = responseBytes; + metrics.maxResponseBytes = maxResponseBytes; + throw new SearchResponseTooLargeError(responseBytes, maxResponseBytes); + } + + metrics.responseBytes = responseBytes; + return originalSend.call(this, body); + }; +} + +function isSearchResponseTooLargeError(err) { + return err instanceof SearchResponseTooLargeError || err?.code === 'SEARCH_RESPONSE_TOO_LARGE'; +} + +module.exports = { + getBodyByteLength, + instrumentSearchResponse, + isSearchResponseTooLargeError, + SearchResponseTooLargeError +}; diff --git a/utils/searchResponseLimiter.test.js b/utils/searchResponseLimiter.test.js new file mode 100644 index 0000000..b738639 --- /dev/null +++ b/utils/searchResponseLimiter.test.js @@ -0,0 +1,92 @@ +const searchResponseLimiter = require('./searchResponseLimiter'); + +function createResponse(statusCode = 200) { + const originalSend = jest.fn(() => 'sent'); + + return { + originalSend, + res: { + statusCode, + send: originalSend + } + }; +} + +describe('searchResponseLimiter', () => { + test('allows a successful response at the byte limit after checking the deadline', () => { + const { originalSend, res } = createResponse(); + const checkDeadline = jest.fn(); + const metrics = {}; + searchResponseLimiter.instrumentSearchResponse(res, { + maxResponseBytes: 4, + metrics, + checkDeadline + }); + + expect(res.send('test')).toBe('sent'); + expect(checkDeadline).toHaveBeenCalledTimes(1); + expect(originalSend).toHaveBeenCalledWith('test'); + expect(metrics.responseBytes).toBe(4); + }); + + test('counts UTF-8 bytes and rejects an oversized success before sending it', () => { + const { originalSend, res } = createResponse(); + const checkDeadline = jest.fn(); + const metrics = {}; + searchResponseLimiter.instrumentSearchResponse(res, { + maxResponseBytes: 3, + metrics, + checkDeadline + }); + + let thrownError; + try { + res.send('\u00e9\u00e9'); + } catch (err) { + thrownError = err; + } + + expect(thrownError).toMatchObject({ + code: 'SEARCH_RESPONSE_TOO_LARGE', + responseBytes: 4, + maxResponseBytes: 3 + }); + expect(checkDeadline).toHaveBeenCalledTimes(1); + expect(originalSend).not.toHaveBeenCalled(); + expect(metrics).toMatchObject({ + attemptedResponseBytes: 4, + maxResponseBytes: 3 + }); + }); + + test('does not limit or deadline-check an error response', () => { + const { originalSend, res } = createResponse(422); + const checkDeadline = jest.fn(); + const metrics = {}; + searchResponseLimiter.instrumentSearchResponse(res, { + maxResponseBytes: 1, + metrics, + checkDeadline + }); + + expect(res.send('complete error body')).toBe('sent'); + expect(checkDeadline).not.toHaveBeenCalled(); + expect(originalSend).toHaveBeenCalledWith('complete error body'); + expect(metrics).toStrictEqual({}); + }); + + test('propagates a post-serialization deadline failure before sending', () => { + const { originalSend, res } = createResponse(); + const deadlineError = Object.assign(new Error('deadline'), { code: 'SEARCH_TIMEOUT' }); + searchResponseLimiter.instrumentSearchResponse(res, { + maxResponseBytes: 100, + metrics: {}, + checkDeadline: jest.fn(() => { + throw deadlineError; + }) + }); + + expect(() => res.send('test')).toThrow(deadlineError); + expect(originalSend).not.toHaveBeenCalled(); + }); +});