From 01a0cbf389a583952e31211f20711ca351bb4cb0 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 12:19:31 -0400 Subject: [PATCH 01/10] refactor: remove unused internal exports --- controllers/searchController.js | 16 ++---- routes/search.integration.test.js | 3 + server.js | 11 ---- server.test.js | 48 +--------------- utils/searchProvider.js | 3 - utils/searchQueryBuilder.js | 5 +- utils/searchQueryBuilder.test.js | 92 ++++++++++++++++--------------- 7 files changed, 61 insertions(+), 117 deletions(-) diff --git a/controllers/searchController.js b/controllers/searchController.js index 8853867..16dc69d 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -82,16 +82,16 @@ function recordUpstreamFailure(metrics, err, failureType) { function isOpenSearchUnavailableError(err) { return ( - searchProvider.isSearchProviderContractError?.(err) === true || - searchProvider.isSearchProviderConfigurationError?.(err) === true || + searchProvider.isSearchProviderContractError(err) === true || + searchProvider.isSearchProviderConfigurationError(err) === true || openSearchUnavailableErrorNames.has(err?.name) || openSearchUnavailableStatusCodes.has(getOpenSearchErrorStatusCode(err)) ); } function getOpenSearchUnavailableFailureType(err) { - if (searchProvider.isSearchProviderContractError?.(err)) return 'provider_contract'; - if (searchProvider.isSearchProviderConfigurationError?.(err)) return 'provider_configuration'; + if (searchProvider.isSearchProviderContractError(err)) return 'provider_contract'; + if (searchProvider.isSearchProviderConfigurationError(err)) return 'provider_configuration'; return 'unavailable'; } @@ -128,12 +128,8 @@ function buildWebSearchSort(sort) { return undefined; } -function getHitCveId(hit) { - return hit?._source?.cveMetadata?.cveId ?? hit?._id; -} - function getHitCveIds(hits) { - return hits.map(getHitCveId).filter(Boolean); + return hits.map((hit) => hit._source.cveMetadata.cveId); } function validateAndOrderPageHits(hits, cveIds) { @@ -435,7 +431,7 @@ async function searchHandler(req, res, next) { return res.status(429).json(error.searchInFlightWaiterLimitError()); } - if (searchProvider.isOpenSearchResponseError?.(err)) { + if (searchProvider.isOpenSearchResponseError(err)) { const failureType = err?.failureType ?? 'malformed_response'; recordUpstreamFailure(metrics, err, failureType); diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index 9aab62a..bc79a6d 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -11,6 +11,9 @@ jest.mock('../utils/searchProvider', () => ({ getInFlightMaxWaitersPerOperation: jest.fn(() => 500), getPostFilterEvaluationChunkSize: jest.fn(() => 250), getSearchRequestDeadlineMs: jest.fn(() => 30000), + isOpenSearchResponseError: jest.fn(() => false), + isSearchProviderConfigurationError: jest.fn(() => false), + isSearchProviderContractError: jest.fn(() => false), searchOpenSearch: mockSearchOpenSearch, shouldLogSearchPerformance: jest.fn(() => false) })); diff --git a/server.js b/server.js index 6dd8bd4..9089b9b 100644 --- a/server.js +++ b/server.js @@ -127,15 +127,6 @@ function createApp() { return app; } -function getRuntimeConfig(port) { - const config = runtimeConfig.validateRuntimeConfig(); - const effectivePort = resolveListenPort(port, config.server.port); - - return { - baseURL: config.server.baseURL || `http://127.0.0.1:${effectivePort}` - }; -} - function resolveListenPort(port, defaultPort) { if (port === undefined) return defaultPort; @@ -354,8 +345,6 @@ module.exports = { createRequestLogger, handleUnexpectedError, installShutdownHandlers, - getRedactedRequestPath: requestLogging.getRedactedRequestPath, - getRuntimeConfig, handleStartupError, rejectRequestsDuringShutdown, shutdownServer, diff --git a/server.test.js b/server.test.js index debf205..a558198 100644 --- a/server.test.js +++ b/server.test.js @@ -40,8 +40,6 @@ const { configureHttpServer, createApp, createRequestLogger, - getRedactedRequestPath, - getRuntimeConfig, handleStartupError, handleUnexpectedError, installShutdownHandlers, @@ -49,6 +47,7 @@ const { startServer, swaggerSetupOptions } = require('./server'); +const requestLogging = require('./utils/requestLogging'); const runtimeConfig = require('./utils/runtimeConfig'); const runtimeLifecycle = require('./utils/runtimeLifecycle'); @@ -68,7 +67,6 @@ function createFakeServer() { } describe('server application bootstrap', () => { - const originalBaseUrl = process.env.BASE_URL; const originalNodeEnv = process.env.NODE_ENV; const originalPort = process.env.PORT; @@ -77,7 +75,6 @@ describe('server application bootstrap', () => { }); afterEach(() => { - restoreEnv('BASE_URL', originalBaseUrl); restoreEnv('NODE_ENV', originalNodeEnv); restoreEnv('PORT', originalPort); runtimeLifecycle.resetForTests(); @@ -284,47 +281,6 @@ describe('server application bootstrap', () => { expect(mockSearch).not.toHaveBeenCalled(); }); - test('getRuntimeConfig preserves baseURL behavior when PORT is set without BASE_URL', () => { - delete process.env.BASE_URL; - process.env.PORT = '4567'; - - expect(getRuntimeConfig('4567')).toStrictEqual({ - baseURL: 'http://127.0.0.1:4567' - }); - }); - - test('getRuntimeConfig uses the effective default port when PORT is not configured', () => { - delete process.env.BASE_URL; - delete process.env.PORT; - - expect(getRuntimeConfig(3000)).toStrictEqual({ - baseURL: 'http://127.0.0.1:3000' - }); - }); - - test('getRuntimeConfig uses BASE_URL when it is already configured', () => { - process.env.BASE_URL = 'https://search.example'; - process.env.PORT = '4567'; - - expect(getRuntimeConfig('4567')).toStrictEqual({ - baseURL: 'https://search.example' - }); - }); - - test('getRedactedRequestPath omits query strings from logged URLs', () => { - expect( - getRedactedRequestPath({ - path: '/search', - originalUrl: '/search?keywordSearch=apache' - }) - ).toBe('/search'); - expect( - getRedactedRequestPath({ - originalUrl: '/webSearch?query=apache' - }) - ).toBe('/webSearch'); - }); - test('createRequestLogger disables request logging in test environments', () => { process.env.NODE_ENV = 'test'; const next = jest.fn(); @@ -340,7 +296,7 @@ describe('server application bootstrap', () => { createRequestLogger(); - expect(mockMorgan.token).toHaveBeenCalledWith('redacted-url', getRedactedRequestPath); + expect(mockMorgan.token).toHaveBeenCalledWith('redacted-url', requestLogging.getRedactedRequestPath); expect(mockMorgan.token).toHaveBeenCalledWith('request-id', expect.any(Function)); expect(mockMorgan).toHaveBeenCalledWith( ':method :redacted-url :status :res[content-length] - :response-time ms request-id=:request-id', diff --git a/utils/searchProvider.js b/utils/searchProvider.js index 8a92326..e7ffe6a 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -359,10 +359,7 @@ module.exports = { resetSearchProviderForTests, searchOpenSearch, shouldLogSearchPerformance, - validateOpenSearchResponse: searchResponseValidator.validateOpenSearchResponse, validateSearchReader, - OPEN_SEARCH_RESPONSE_FAILURE_TYPES: searchResponseValidator.OPEN_SEARCH_RESPONSE_FAILURE_TYPES, - OpenSearchResponseError: searchResponseValidator.OpenSearchResponseError, SearchProviderConfigurationError, SearchProviderContractError }; diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index 2a1f62b..f30edc2 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -616,8 +616,5 @@ module.exports = { buildCpeCandidateSearchBody, buildCveIdsSearchBody, buildOpenSearchBody, - buildQueryStrings, - buildSearchRequest, - getFilterFields, - processParams + buildSearchRequest }; diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index 71e580b..79f9c2c 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -46,6 +46,14 @@ function anyOf(clauses) { }; } +function buildQueryStrings(params) { + return queryBuilder.buildSearchRequest(params).queryStringObjects; +} + +function getFilterFields(params) { + return queryBuilder.buildSearchRequest(params).searchOptions.filters; +} + describe('searchQueryBuilder buildSearchRequest()', () => { test('buildSearchRequest builds filters, query strings, date ranges, and pagination options', () => { const output = queryBuilder.buildSearchRequest({ @@ -493,8 +501,8 @@ describe('searchQueryBuilder buildSearchRequest()', () => { }); }); -describe('searchQueryBuilder buildQueryStrings() function', () => { - test('buildQueryStrings() function correctly builds array of one simple_query_string object when keywordSearch is passed.', () => { +describe('searchQueryBuilder buildSearchRequest() query strings', () => { + test('buildSearchRequest() builds one simple_query_string object when keywordSearch is passed.', () => { const testParams = { keywordSearch: 'test* params*' }; @@ -502,20 +510,20 @@ describe('searchQueryBuilder buildQueryStrings() function', () => { const testQueryString = [ { simple_query_string: { - query: 'test* params*', + query: 'test\\** params\\**', default_operator: 'AND', fields: ['containers.cna.descriptions.value'] } } ]; - const output = queryBuilder.buildQueryStrings(testParams); + const output = buildQueryStrings(testParams); expect(output).toHaveLength(1); expect(output).toStrictEqual(testQueryString); }); - test('buildQueryStrings() ignores non-keyword fields instead of generating query_string clauses.', () => { - const output = queryBuilder.buildQueryStrings({ + test('buildSearchRequest() ignores non-keyword fields instead of generating query string clauses.', () => { + const output = buildQueryStrings({ cweId: 'CWE-79', virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809' }); @@ -524,8 +532,8 @@ describe('searchQueryBuilder buildQueryStrings() function', () => { }); }); -describe('searchQueryBuilder getFilterFields() function', () => { - test('getFilterFields() function correctly builds array of one matchPhrase object when one parameter is passed.', () => { +describe('searchQueryBuilder buildSearchRequest() filters', () => { + test('buildSearchRequest() builds one match phrase when one filter parameter is passed.', () => { const testParams = { cweId: 'CWE-79' }; @@ -538,12 +546,12 @@ describe('searchQueryBuilder getFilterFields() function', () => { } ]; - const output = queryBuilder.getFilterFields(testParams); + const output = getFilterFields(testParams); expect(output).toHaveLength(1); expect(output).toStrictEqual(testMatchPhrase); }); - test('getFilterFields() function correctly builds default and severity filter objects when multiple parameters are passed.', () => { + test('buildSearchRequest() builds default and severity filters when multiple parameters are passed.', () => { const testParams = { cweId: 'CWE-79', cvssV4Severity: 'HIGH' @@ -562,13 +570,13 @@ describe('searchQueryBuilder getFilterFields() function', () => { } ]; - const output = queryBuilder.getFilterFields(testParams); + const output = getFilterFields(testParams); expect(output).toHaveLength(2); expect(output).toStrictEqual(testMatchPhrase); }); - test('getFilterFields() builds exact terms filter for comma-separated cveIds.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds an exact terms filter for comma-separated cveIds.', () => { + const output = getFilterFields({ cveIds: 'cve-2024-0002, CVE-2024-0001,CVE-2024-0002' }); @@ -581,8 +589,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds rejected-state exclusions for noRejected.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds rejected-state exclusions for noRejected.', () => { + const output = getFilterFields({ noRejected: true }); @@ -610,24 +618,24 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() ignores noRejected when it is false.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() ignores noRejected when it is false.', () => { + const output = getFilterFields({ noRejected: false }); expect(output).toStrictEqual([]); }); - test('getFilterFields() does not retain legacy empty-string noRejected behavior.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() does not retain legacy empty-string noRejected behavior.', () => { + const output = getFilterFields({ noRejected: '' }); expect(output).toStrictEqual([]); }); - test('getFilterFields() builds a cvssV2 severity range filter on baseScore.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds a cvssV2 severity range filter on baseScore.', () => { + const output = getFilterFields({ cvssV2Severity: 'MEDIUM' }); @@ -643,8 +651,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds a same-vector regexp filter for CNA cvssV2 metrics.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds a same-vector regexp filter for CNA cvssV2 metrics.', () => { + const output = getFilterFields({ cvssV2Metrics: 'AV:N/AC:L/Au:N' }); @@ -653,8 +661,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds exact-version and wildcard-version CPE Name candidate filters.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds exact-version and wildcard-version CPE Name candidate filters.', () => { + const output = getFilterFields({ cpeName: 'cpe:2.3:a:artifex:ghostscript:10.05.1:*:*:*:*:*:*:*' }); @@ -687,8 +695,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { expect(output[0].bool.should).toHaveLength(2); }); - test('getFilterFields() leaves CPE version range applicability to semantic post-filtering.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() leaves CPE version range applicability to semantic post-filtering.', () => { + const output = getFilterFields({ cpeName: 'cpe:2.3:a:artifex:ghostscript:10.05.1:*:*:*:*:*:*:*' }); @@ -703,14 +711,12 @@ describe('searchQueryBuilder getFilterFields() function', () => { test.each([ ['cpeName', { cpeName: 'not-a-cpe-name' }], ['virtualMatchString', { virtualMatchString: 'not-a-cpe-match-string' }] - ])('getFilterFields() fails closed when invalid %s reaches query construction', (param, params) => { - expect(() => queryBuilder.getFilterFields(params)).toThrow( - `Invalid ${param} value reached query construction.` - ); + ])('buildSearchRequest() fails closed when invalid %s reaches query construction', (param, params) => { + expect(() => getFilterFields(params)).toThrow(`Invalid ${param} value reached query construction.`); }); - test('getFilterFields() builds combined cvssV3 severity filters across v3.0 and v3.1.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() builds combined cvssV3 severity filters across v3.0 and v3.1.', () => { + const output = getFilterFields({ cvssV3Severity: 'critical' }); @@ -735,8 +741,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() constrains versioned cvssV3 metrics to the matching CNA field.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() constrains versioned cvssV3 metrics to the matching CNA field.', () => { + const output = getFilterFields({ cvssV3Metrics: 'CVSS:3.1/AC:L/AV:N' }); @@ -745,8 +751,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() searches both v3 versions when cvssV3Metrics omits a header.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() searches both v3 versions when cvssV3Metrics omits a header.', () => { + const output = getFilterFields({ cvssV3Metrics: 'AV:N' }); @@ -758,8 +764,8 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() escapes regexp metacharacters in CVSS metric filters.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() escapes regexp metacharacters in CVSS metric filters.', () => { + const output = getFilterFields({ cvssV4Metrics: 'CVSS#4.0/AV:N/AC:*?/UI:P' }); @@ -770,16 +776,16 @@ describe('searchQueryBuilder getFilterFields() function', () => { expect(JSON.stringify(output)).not.toContain('containers.adp'); }); - test('getFilterFields() fails closed when invalid CVSS metrics reach query construction.', () => { + test('buildSearchRequest() fails closed when invalid CVSS metrics reach query construction.', () => { expect(() => - queryBuilder.getFilterFields({ + getFilterFields({ cvssV3Metrics: 'CVSS:3.1' }) ).toThrow('Invalid cvssV3Metrics value reached query construction.'); }); - test('getFilterFields() does not build filter objects for incorrect parameters.', () => { - const output = queryBuilder.getFilterFields({ + test('buildSearchRequest() does not build filter objects for incorrect parameters.', () => { + const output = getFilterFields({ test: 'EXTERNAL' }); From ca97ea41538b526bae66a5e84d21a2728c702be0 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 12:23:27 -0400 Subject: [PATCH 02/10] refactor: remove obsolete base URL config --- .env.example | 1 - README.md | 3 +-- utils/runtimeConfig.js | 3 --- utils/runtimeConfig.test.js | 24 ++---------------------- 4 files changed, 3 insertions(+), 28 deletions(-) diff --git a/.env.example b/.env.example index ef0af0e..f2680a2 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,5 @@ NODE_CONFIG_ENV=devel PORT=3000 -BASE_URL= OpenSearchCveIndex=cve-index-local OpenSearchAllowUnknownSslCerts=false OpenSearchDomainEndpoint= diff --git a/README.md b/README.md index 1b3dac7..8a9f7e0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ Rename `.env.example` to `.env` and replace or check the following values. - `NODE_CONFIG_ENV` - node-config profile name; use `devel` locally and `prod` in production images - `PORT` - port used for the search API, defaulted to 3000 for development -- `BASE_URL` - optional absolute public URL used by local runtime helpers; credentials are not permitted - `OpenSearchCveIndex` - existing CVE index name in your OpenSearch deployment - `OpenSearchAllowUnknownSslCerts` - set to `true` only for local development against self-signed certificates - `OpenSearchDomainEndpoint` - absolute URL to an existing OpenSearch deployment. This API does not create or manage the OpenSearch index or mapping. @@ -72,7 +71,7 @@ SearchPostFilterColdSearchMaxWaitMs < SearchRequestDeadlineMs HttpHeadersTimeoutMs <= HttpRequestTimeoutMs ``` -`OpenSearchDomainEndpoint` must be an absolute HTTP or HTTPS URL, and `OpenSearchCveIndex` must be non-empty. `BASE_URL`, when supplied, must also be an absolute HTTP or HTTPS URL and cannot contain credentials. OpenSearch connectivity is not required at startup; `/api/ready-check` remains responsible for dependency readiness. +`OpenSearchDomainEndpoint` must be an absolute HTTP or HTTPS URL, and `OpenSearchCveIndex` must be non-empty. OpenSearch connectivity is not required at startup; `/api/ready-check` remains responsible for dependency readiness. Production images set both `NODE_ENV=production` and `NODE_CONFIG_ENV=prod`, matching `config/prod.jsonc`. Successful startup emits a sanitized `runtime.lifecycle` JSON event with effective safety settings. Startup failures identify only the invalid configuration path and never log configured values, URLs, index names, credentials, or exception messages. diff --git a/utils/runtimeConfig.js b/utils/runtimeConfig.js index 9fc709f..77745c4 100644 --- a/utils/runtimeConfig.js +++ b/utils/runtimeConfig.js @@ -189,12 +189,9 @@ function resolveRuntimeConfig({ readConfigValue(appConfig, 'search.index', defaultSearchProvider.index), { maxLength: 255 } ); - const baseURL = parseOptionalString('BASE_URL', env.BASE_URL, { maxLength: 4096 }); - const config = { server: { port: parseInteger('PORT', env.PORT ?? DEFAULT_PORT, RUNTIME_CONFIG_LIMITS.port), - baseURL: parseHttpUrl('BASE_URL', baseURL, { allowCredentials: false }), nodeEnvironment: String(env.NODE_ENV || 'development'), configEnvironment: String(env.NODE_CONFIG_ENV || env.NODE_ENV || 'default') }, diff --git a/utils/runtimeConfig.test.js b/utils/runtimeConfig.test.js index 850ead3..90777c4 100644 --- a/utils/runtimeConfig.test.js +++ b/utils/runtimeConfig.test.js @@ -37,7 +37,6 @@ describe('runtimeConfig', () => { expect(config).toMatchObject({ server: { port: 3000, - baseURL: '', nodeEnvironment: 'test', configEnvironment: 'test' }, @@ -115,7 +114,6 @@ describe('runtimeConfig', () => { }, { PORT: '4000', - BASE_URL: 'https://api.example', NODE_CONFIG_ENV: 'prod' } ); @@ -123,7 +121,6 @@ describe('runtimeConfig', () => { expect(config).toMatchObject({ server: { port: 4000, - baseURL: 'https://api.example', nodeEnvironment: 'test', configEnvironment: 'prod' }, @@ -244,29 +241,12 @@ describe('runtimeConfig', () => { expect(() => runtimeConfig.validateRuntimeConfig(config)).toThrow(path); }); - test('rejects credential-bearing BASE_URL values', () => { - expect(() => - resolve( - {}, - { - BASE_URL: 'https://admin:secret@api.example' - } - ) - ).toThrow('BASE_URL must not contain credentials.'); - }); - - test('sanitized startup metadata does not expose provider or base URLs', () => { - const config = resolve( - {}, - { - BASE_URL: 'https://api.example' - } - ); + test('sanitized startup metadata does not expose provider configuration', () => { + const config = resolve(); const serialized = JSON.stringify(runtimeConfig.sanitizeRuntimeConfig(config)); expect(serialized).not.toContain('search.example'); expect(serialized).not.toContain('test-index'); - expect(serialized).not.toContain('api.example'); }); }); From c8bbb5c6e20f8693cf814622df08a7441aa9e982 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 12:35:11 -0400 Subject: [PATCH 03/10] refactor: simplify search validation --- controllers/middleware.js | 66 ++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/controllers/middleware.js b/controllers/middleware.js index a822cc3..af2c0ce 100644 --- a/controllers/middleware.js +++ b/controllers/middleware.js @@ -44,9 +44,11 @@ const dateRangePairs = [ ['lastModStartDate', 'lastModEndDate'], ['pubStartDate', 'pubEndDate'] ]; -const versionRangeParams = ['versionStart', 'versionStartType', 'versionEnd', 'versionEndType']; -const versionRangeValueParams = ['versionStart', 'versionEnd']; -const versionRangeTypeParams = ['versionStartType', 'versionEndType']; +const versionRangeBounds = [ + ['versionStart', 'versionStartType', 'versionStartPairError'], + ['versionEnd', 'versionEndType', 'versionEndPairError'] +]; +const versionRangeParams = versionRangeBounds.flatMap(([valueParam, typeParam]) => [valueParam, typeParam]); const nonStringSearchParams = [ 'keywordExactMatch', 'cveIds', @@ -89,20 +91,13 @@ function parseCveIds(value) { return cveIds; } -function getRequestedResultsPerPage(params) { - return hasParam(params, 'resultsPerPage') ? Number(params.resultsPerPage) : constants.RESULTS_PER_PAGE; -} - -function getRequestedPageNumber(params) { - return hasParam(params, 'pageNumber') ? Number(params.pageNumber) : 1; -} - -function exceedsSearchWindow(params) { - const resultsPerPage = getRequestedResultsPerPage(params); - const pageNumber = getRequestedPageNumber(params); - const from = resultsPerPage * (pageNumber - 1); +function getPagination(params) { + const resultsPerPage = hasParam(params, 'resultsPerPage') + ? Number(params.resultsPerPage) + : constants.RESULTS_PER_PAGE; + const pageNumber = hasParam(params, 'pageNumber') ? Number(params.pageNumber) : 1; - return from + resultsPerPage > constants.MAX_SEARCH_WINDOW; + return { resultsPerPage, pageNumber, from: resultsPerPage * (pageNumber - 1) }; } function isValidDateTime(value) { @@ -161,26 +156,28 @@ function validateParams(req, res, next) { return res.status(400).json(error.noRejectedValueError()); } + const parsedVirtualMatchString = params.virtualMatchString + ? cpe.parseVirtualMatchString(params.virtualMatchString) + : null; + if (hasAnyParam(params, versionRangeParams)) { if (!params.virtualMatchString) { return res.status(400).json(error.versionRangeVirtualMatchStringRequiredError()); } - if (hasParam(params, 'versionStart') !== hasParam(params, 'versionStartType')) { - return res.status(400).json(error.versionStartPairError()); - } - - if (hasParam(params, 'versionEnd') !== hasParam(params, 'versionEndType')) { - return res.status(400).json(error.versionEndPairError()); + for (const [valueParam, typeParam, pairError] of versionRangeBounds) { + if (hasParam(params, valueParam) !== hasParam(params, typeParam)) { + return res.status(400).json(error[pairError]()); + } } - for (const param of versionRangeValueParams) { + for (const [param] of versionRangeBounds) { if (hasParam(params, param) && !isSingleNonEmptyString(params[param])) { return res.status(400).json(error.versionRangeValueError(param)); } } - for (const param of versionRangeTypeParams) { + for (const [, param] of versionRangeBounds) { if ( hasParam(params, param) && (!isSingleNonEmptyString(params[param]) || @@ -190,12 +187,11 @@ function validateParams(req, res, next) { } } - const virtualMatchString = cpe.parseVirtualMatchString(params.virtualMatchString); - if (!virtualMatchString.valid) { + if (!parsedVirtualMatchString.valid) { return res.status(400).json(error.virtualMatchStringVersionRangeFormatError()); } - if (virtualMatchString.hasVersionComponent) { + if (parsedVirtualMatchString.hasVersionComponent) { return res.status(400).json(error.virtualMatchStringVersionComponentError()); } @@ -230,8 +226,7 @@ function validateParams(req, res, next) { } if (params?.virtualMatchString) { - const virtualMatchString = cpe.parseVirtualMatchString(params.virtualMatchString); - if (!virtualMatchString.valid) { + if (!parsedVirtualMatchString.valid) { return res.status(400).json(error.virtualMatchStringFormatError()); } } @@ -246,29 +241,28 @@ function validateParams(req, res, next) { return res.status(404).json(error.invalidCpeNameError()); } + const pagination = getPagination(params); + //Ensures that resultsPerPage is a positive integer if ( hasParam(params, 'resultsPerPage') && - (!Number.isInteger(Number(params.resultsPerPage)) || Number(params.resultsPerPage) < 1) + (!Number.isInteger(pagination.resultsPerPage) || pagination.resultsPerPage < 1) ) { return res.status(400).json(error.resultsPerPageError()); } //Ensures that resultsPerPage is not above the OpenSearch page size cap - if (hasParam(params, 'resultsPerPage') && Number(params.resultsPerPage) > constants.MAX_RESULTS_LIMIT) { + if (hasParam(params, 'resultsPerPage') && pagination.resultsPerPage > constants.MAX_RESULTS_LIMIT) { return res.status(400).json(error.maxResultsError()); } //Ensures that pageNumber is a positive integer - if ( - hasParam(params, 'pageNumber') && - (!Number.isInteger(Number(params.pageNumber)) || Number(params.pageNumber) < 1) - ) { + if (hasParam(params, 'pageNumber') && (!Number.isInteger(pagination.pageNumber) || pagination.pageNumber < 1)) { return res.status(400).json(error.pageNumberError()); } //Ensures shallow pagination stays within the OpenSearch from/size result window - if (exceedsSearchWindow(params)) { + if (pagination.from + pagination.resultsPerPage > constants.MAX_SEARCH_WINDOW) { return res.status(400).json(error.searchWindowError()); } From fee1897e1034e85b687147c05da8894abcc84c56 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 12:43:46 -0400 Subject: [PATCH 04/10] test: consolidate search validation coverage --- controllers/middleware.test.js | 175 +++++------------------------- routes/cpeName.test.js | 62 ----------- routes/search.integration.test.js | 95 ---------------- 3 files changed, 26 insertions(+), 306 deletions(-) delete mode 100644 routes/cpeName.test.js diff --git a/controllers/middleware.test.js b/controllers/middleware.test.js index 6f82844..6bfe8d4 100644 --- a/controllers/middleware.test.js +++ b/controllers/middleware.test.js @@ -592,132 +592,42 @@ describe('middleware validation function', () => { expectValidationSuccess(); }); - test('ValidateParams returns an error when a non enumerator value is passed for cvssV2Severity.', () => { - validate({ - cvssV2Severity: 'critical' - }); - - expectValidationError( - 'INVALID_CVSSV2_SEVERITY', - 'critical is not an allowed input for the cvssV2Severity. Please use LOW, MEDIUM, or HIGH.' - ); - }); - - test('ValidateParams successfully accepts enumerator value medium for cvssV2Severity.', () => { - validate({ - cvssV2Severity: 'medium' - }); - - expectValidationSuccess(); - }); - - test('ValidateParams returns an error when a non enumerator value is passed for cvssV3Severity.', () => { - validate({ - cvssV3Severity: 'Test' - }); - - expectValidationError( - 'INVALID_CVSSV3_SEVERITY', - 'Test is not an allowed input for the cvssV3Severity. Please use LOW, MEDIUM, HIGH, or CRITICAL.' - ); - }); - - test('ValidateParams successfully accepts enumerator value critical for cvssV3Severity.', () => { - validate({ - cvssV3Severity: 'critical' - }); - - expectValidationSuccess(); - }); - - test('ValidateParams returns an error when a non enumerator value is passed for cvssV3_0Severity.', () => { - validate({ - cvssV3_0Severity: 'Test' - }); - - expectValidationError( - 'INVALID_CVSSV3_0_SEVERITY', - 'Test is not an allowed input for the cvssV3_0Severity. Please use LOW, MEDIUM, HIGH, or CRITICAL.' - ); - }); - - test('ValidateParams successfully accepts enumerator value high for cvssV3_0Severity.', () => { - validate({ - cvssV3_0Severity: 'high' - }); - - expectValidationSuccess(); - }); - - test('ValidateParams returns an error when a non enumerator value is passed for cvssV3_1Severity.', () => { - validate({ - cvssV3_1Severity: 'Test' - }); - - expectValidationError( - 'INVALID_CVSSV3_1_SEVERITY', - 'Test is not an allowed input for the cvssV3_1Severity. Please use LOW, MEDIUM, HIGH, or CRITICAL.' - ); - }); - - test('ValidateParams successfully accepts enumerator value critical for cvssV3_1Severity.', () => { - validate({ - cvssV3_1Severity: 'critical' - }); - - expectValidationSuccess(); - }); - - test('ValidateParams returns an error when a non enumerator value is passed for cvssV4Severity.', () => { - validate({ - cvssV4Severity: 'Test' - }); + test.each([ + ['cvssV2Severity', 'critical', 'INVALID_CVSSV2_SEVERITY', 'LOW, MEDIUM, or HIGH'], + ['cvssV3Severity', 'Test', 'INVALID_CVSSV3_SEVERITY', 'LOW, MEDIUM, HIGH, or CRITICAL'], + ['cvssV3_0Severity', 'Test', 'INVALID_CVSSV3_0_SEVERITY', 'LOW, MEDIUM, HIGH, or CRITICAL'], + ['cvssV3_1Severity', 'Test', 'INVALID_CVSSV3_1_SEVERITY', 'LOW, MEDIUM, HIGH, or CRITICAL'], + ['cvssV4Severity', 'Test', 'INVALID_CVSSV4_SEVERITY', 'LOW, MEDIUM, HIGH, or CRITICAL'] + ])('ValidateParams rejects an invalid %s value.', (param, value, errorCode, allowedValues) => { + validate({ [param]: value }); expectValidationError( - 'INVALID_CVSSV4_SEVERITY', - 'Test is not an allowed input for the cvssV4Severity. Please use LOW, MEDIUM, HIGH, or CRITICAL.' + errorCode, + `${value} is not an allowed input for the ${param}. Please use ${allowedValues}.` ); }); - test('ValidateParams successfully accepts enumerator value high for cvssV4Severity.', () => { - validate({ - cvssV4Severity: 'high' - }); + test.each([ + ['cvssV2Severity', 'medium'], + ['cvssV3Severity', 'critical'], + ['cvssV3_0Severity', 'high'], + ['cvssV3_1Severity', 'critical'], + ['cvssV4Severity', 'high'] + ])('ValidateParams accepts a valid %s value.', (param, value) => { + validate({ [param]: value }); expectValidationSuccess(); }); - test('ValidateParams returns an error when a non integer value is passed for resultsPerPage.', () => { - validate({ - resultsPerPage: 'Test' - }); - - expectValidationError( - 'RESULTS_PER_PAGE_INT', - 'The resultsPerPage parameter must be an integer with a value greater than 0.' - ); - }); - - test('ValidateParams returns an error when a non-positive integer value is passed for resultsPerPage.', () => { - validate({ - resultsPerPage: -1 - }); - - expectValidationError( - 'RESULTS_PER_PAGE_INT', - 'The resultsPerPage parameter must be an integer with a value greater than 0.' - ); - }); - - test('ValidateParams returns an error when resultsPerPage is zero.', () => { - validate({ - resultsPerPage: 0 - }); + test.each([ + ['resultsPerPage', 'non-integer', 'Test', 'RESULTS_PER_PAGE_INT'], + ['resultsPerPage', 'zero', 0, 'RESULTS_PER_PAGE_INT'], + ['pageNumber', 'non-integer', 'Test', 'PAGE_NUMBER_INT'], + ['pageNumber', 'zero', 0, 'PAGE_NUMBER_INT'] + ])('ValidateParams rejects %s with a %s value.', (param, _caseName, value, errorCode) => { + validate({ [param]: value }); - expectValidationError( - 'RESULTS_PER_PAGE_INT', - 'The resultsPerPage parameter must be an integer with a value greater than 0.' - ); + expectValidationError(errorCode, `The ${param} parameter must be an integer with a value greater than 0.`); }); test('ValidateParams returns an error when resultsPerPage exceeds the max result size.', () => { @@ -728,39 +638,6 @@ describe('middleware validation function', () => { expectValidationError('MAX_RESULTS_LIMIT', 'The resultsPerPage must be an integer less than or equal to 500.'); }); - test('ValidateParams returns an error when a non integer value is passed for pageNumber.', () => { - validate({ - pageNumber: 'Test' - }); - - expectValidationError( - 'PAGE_NUMBER_INT', - 'The pageNumber parameter must be an integer with a value greater than 0.' - ); - }); - - test('ValidateParams returns an error when a non-positive integer value is passed for pageNumber.', () => { - validate({ - pageNumber: -1 - }); - - expectValidationError( - 'PAGE_NUMBER_INT', - 'The pageNumber parameter must be an integer with a value greater than 0.' - ); - }); - - test('ValidateParams returns an error when pageNumber is zero.', () => { - validate({ - pageNumber: 0 - }); - - expectValidationError( - 'PAGE_NUMBER_INT', - 'The pageNumber parameter must be an integer with a value greater than 0.' - ); - }); - test('ValidateParams accepts pagination at the maximum search window boundary.', () => { validate({ resultsPerPage: 500, diff --git a/routes/cpeName.test.js b/routes/cpeName.test.js deleted file mode 100644 index 4e0d82e..0000000 --- a/routes/cpeName.test.js +++ /dev/null @@ -1,62 +0,0 @@ -const { createRouterApp, request } = require('../test/http'); - -jest.mock('../controllers/searchController', () => ({ - SEARCH: jest.fn((req, res) => - res.status(200).json({ - route: 'search', - params: req.searchParams - }) - ), - WEB_SEARCH: jest.fn() -})); - -const controller = require('../controllers/searchController'); -const router = require('./index'); - -describe('POST /search cpeName route behavior', () => { - let app; - - beforeEach(() => { - app = createRouterApp(router); - jest.clearAllMocks(); - }); - - test('rejects cpeName with wildcard version before the controller runs', async () => { - const cpeName = 'cpe:2.3:a:artifex:ghostscript:*:*:*:*:*:*:*:*'; - const response = await request(app, { - method: 'POST', - path: '/search', - body: { - cpeName - } - }); - - expect(response.status).toBe(404); - expect(response.body).toStrictEqual({ - error: 'INVALID_CPE_NAME', - message: - 'The cpeName parameter must be a CPE 2.3 name with part a, h, or o and non-wildcard vendor, product, and version components.' - }); - expect(controller.SEARCH).not.toHaveBeenCalled(); - }); - - test('allows valid cpeName through to the controller', async () => { - const cpeName = 'cpe:2.3:a:artifex:ghostscript:10.05.1:*:*:*:*:*:*:*'; - const response = await request(app, { - method: 'POST', - path: '/search', - body: { - cpeName - } - }); - - expect(response.status).toBe(200); - expect(controller.SEARCH).toHaveBeenCalledTimes(1); - expect(response.body).toStrictEqual({ - route: 'search', - params: { - cpeName - } - }); - }); -}); diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index bc79a6d..68ab08c 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -1,5 +1,4 @@ const { request } = require('../test/http'); -const constants = require('../utils/constants'); const mockSearchOpenSearch = jest.fn(); const mockBasicSearch = jest.fn(); @@ -152,59 +151,6 @@ describe('search route integration', () => { expect(requestErrorLog.mock.calls[0][0]).not.toContain('sensitive'); }); - test('POST /search rejects invalid dates before invoking OpenSearch', async () => { - const response = await request(createApp(), { - method: 'POST', - path: '/search', - body: { - pubStartDate: '2024-02-30T00:00:00.000Z' - } - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'DATE_TIME_FORMAT', - message: "The pubStartDate parameter must match the following format: yyyy-MM-ddTHH:mm:ss.SSS'Z'" - }); - expect(mockSearchOpenSearch).not.toHaveBeenCalled(); - }); - - test('POST /search rejects reversed date ranges before invoking OpenSearch', async () => { - const response = await request(createApp(), { - method: 'POST', - path: '/search', - body: { - pubStartDate: '2024-02-01T00:00:00.000Z', - pubEndDate: '2024-01-01T00:00:00.000Z' - } - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'DATE_RANGE_ORDER', - message: 'The pubStartDate parameter must be less than or equal to pubEndDate.' - }); - expect(mockSearchOpenSearch).not.toHaveBeenCalled(); - }); - - test('POST /search rejects CVSS headers without metric tokens before invoking OpenSearch', async () => { - const response = await request(createApp(), { - method: 'POST', - path: '/search', - body: { - cvssV3Metrics: 'CVSS:3.1' - } - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'CVSS_METRICS_FORMAT', - message: - 'The cvssV3Metrics parameter must contain at least one unique KEY:VALUE metric token and use a compatible CVSS version header when a header is provided.' - }); - expect(mockSearchOpenSearch).not.toHaveBeenCalled(); - }); - test('POST /search rejects unsupported CPE parts before invoking OpenSearch', async () => { const response = await request(createApp(), { method: 'POST', @@ -223,47 +169,6 @@ describe('search route integration', () => { expect(mockSearchOpenSearch).not.toHaveBeenCalled(); }); - test('POST /search rejects excessive keyword terms before invoking OpenSearch', async () => { - const response = await request(createApp(), { - method: 'POST', - path: '/search', - body: { - keywordSearch: Array(constants.MAX_KEYWORD_SEARCH_TOKENS + 1) - .fill('term') - .join(' ') - } - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'SEARCH_FIELD_TOKEN_LIMIT', - message: `The keywordSearch field must contain ${constants.MAX_KEYWORD_SEARCH_TOKENS} or fewer terms.` - }); - expect(mockSearchOpenSearch).not.toHaveBeenCalled(); - }); - - test('POST /search rejects an inverted virtualMatchString version range before invoking OpenSearch', async () => { - const response = await request(createApp(), { - method: 'POST', - path: '/search', - body: { - virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', - versionStart: '3.0', - versionStartType: 'including', - versionEnd: '2.0', - versionEndType: 'including' - } - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'VERSION_RANGE_ORDER', - message: - 'The versionStart parameter must be less than versionEnd. Equal values require both bounds to be including.' - }); - expect(mockSearchOpenSearch).not.toHaveBeenCalled(); - }); - test('POST /webSearch normalizes body fields and formats BasicSearchManager responses', async () => { mockBasicSearch.mockResolvedValue(basicResponse({ total: 1, hits: [{ _id: 'CVE-2024-0002' }] })); From 91d61f13f619739d90083d29f1fe0bf1ea101846 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 12:54:54 -0400 Subject: [PATCH 05/10] docs: clarify semantic CPE result limits --- README.md | 4 +++- api-docs/openapi.json | 6 +++--- api-docs/openapi.test.js | 17 +++++++++++++++++ routes/swagger.js | 6 +++--- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8a9f7e0..5946ff0 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ Publication and last-modified date filters support one-sided ranges. Values must `pageSize` reports the number of CVE records actually returned in the current response. `page` is the requested one-based page, except a search with no matches reports page `0`. An accepted page beyond the available matches returns an empty `hits` array and page size `0`. -Pagination is limited to complete pages inside the first 10,000 results: `from + resultsPerPage` must not exceed 10,000. `totalPages` reports the smaller of the pages implied by the match count and the complete pages accessible inside that window. For example, a page size of 30 allows at most 333 pages. The match total itself is not reduced by this pagination calculation. Point-in-time search and deep pagination remain deferred. +Pagination is limited to complete pages inside the first 10,000 results: `from + resultsPerPage` must not exceed 10,000. `totalPages` reports the smaller of the pages implied by the match count and the complete pages accessible inside that window. For example, a page size of 30 allows at most 333 pages. Standard-search totals are exact and are not reduced by this pagination calculation. Semantic CPE searches have a separate 10,000-candidate evaluation limit described below, so their totals can be incomplete. Point-in-time search and deep pagination remain deferred. Use `cveIds` to request up to 100 CVE records by comma-separated CVE ID. Each ID must contain the `CVE` prefix, a four-digit year, and a sequence number containing 4 to 19 digits. The prefix is accepted without regard to case and normalized to uppercase for the OpenSearch query. @@ -370,6 +370,8 @@ This is a process-wide safety limit, not a precise distributed quota. Each ECS t Searches that use `cpeName`, `virtualMatchString`, or `virtualMatchString` with version range parameters use a semantic CPE post-filter. The API first asks OpenSearch for up to 10,000 lightweight candidate records, filters those candidates in Node.js, then fetches the requested page of full CVE records by CVE ID. +The 10,000-candidate fetch is a hard evaluation boundary, not only a pagination limit. If a broad semantic CPE query produces more than 10,000 OpenSearch candidates, candidates beyond that boundary are not evaluated. The response can therefore omit later applicable CVE records, and its total reports only matches found within the evaluated candidate set. Add other filters or use a more specific CPE value where possible. Evaluating the complete candidate set requires a future point-in-time and `search_after` implementation. + The post-filter cache stores only the filtered CVE ID list and candidate count for that expensive candidate query. It does not cache full CVE records. On a cache hit, the API skips the 10,000-candidate OpenSearch query and semantic post-filtering, slices the cached CVE IDs for the requested page, and fetches that page of full CVE records by ID. Candidate retrieval asks OpenSearch for exact-version or wildcard-version criteria and leaves range applicability decisions to the semantic post-filter. The full-record page fetch uses an exact CVE ID set without total-hit counting or OpenSearch sorting; the API requires one unique full record for every requested CVE ID and restores the requested CVE ID order locally. Missing, duplicate, malformed, or unexpected page records fail closed with a sanitized HTTP 502 response and evict the associated semantic candidate cache entry so the next request must repeat candidate discovery. Standard searches retain exact total counting and deterministic sorting. diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 8a0e900..2bb9200 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -499,7 +499,7 @@ "type": "string", "maxLength": 2048, "example": "cpe:2.3:a:freetype:freetype:2.8.1:*:*:*:*:*:*:*", - "description": "Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches." + "description": "Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Semantic evaluation is limited to the first 10000 OpenSearch candidates, so broad searches may omit later matches and report incomplete totals. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches." }, "isVulnerable": { "type": "boolean", @@ -581,7 +581,7 @@ "type": "string", "maxLength": 2048, "example": "cpe:2.3:o:microsoft:windows_10_1809", - "description": "Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component." + "description": "Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component. Semantic evaluation, including version range searches, is limited to the first 10000 OpenSearch candidates, so broad searches may omit later matches and report incomplete totals." } } }, @@ -594,7 +594,7 @@ "value": { "type": "integer", "minimum": 0, - "description": "Number of records matched by the active search strategy." + "description": "Number of records matched by the active search strategy. Standard-search totals are exact. For semantic cpeName and virtualMatchString searches, this value counts matches found within the first 10000 OpenSearch candidates and may be incomplete." }, "relation": { "type": "string", diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index e51e77a..66115a7 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -184,6 +184,23 @@ describe('OpenAPI contract', () => { expect(openapi.paths['/search'].post.responses['429'].description).toContain('waiter limit'); }); + test('documents the bounded semantic CPE candidate window', () => { + const searchProperties = openapi.components.schemas.SearchRequest.properties; + const totalDescription = openapi.components.schemas.SearchHitTotal.properties.value.description; + + ['cpeName', 'virtualMatchString'].forEach((field) => { + expect(searchProperties[field].description).toContain( + `first ${constants.MAX_SEARCH_WINDOW} OpenSearch candidates` + ); + expect(searchProperties[field].description).toContain('may omit later matches'); + expect(searchProperties[field].description).toContain('incomplete totals'); + }); + expect(searchProperties.virtualMatchString.description).toContain('including version range searches'); + expect(totalDescription).toContain('Standard-search totals are exact'); + expect(totalDescription).toContain(`first ${constants.MAX_SEARCH_WINDOW} OpenSearch candidates`); + expect(totalDescription).toContain('may be incomplete'); + }); + test('describes POST /search as a paginated result operation', () => { const operation = openapi.paths['/search'].post; diff --git a/routes/swagger.js b/routes/swagger.js index aba5b68..2982644 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -126,7 +126,7 @@ const searchRequestProperties = { } ), cpeName: searchRequestProperty( - "Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches.", + `Returns CVE records whose individual CPE Match Criteria entries apply to the provided CPE 2.3 asset name. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. The part must be a, h, or o; vendor, product, and version must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Semantic evaluation is limited to the first ${constants.MAX_SEARCH_WINDOW} OpenSearch candidates, so broad searches may omit later matches and report incomplete totals. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches.`, { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cpeName, @@ -226,7 +226,7 @@ const searchRequestProperties = { } ), virtualMatchString: searchRequestProperty( - "Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component.", + `Returns CVE records whose individual CPE Match Criteria entries are compatible with the provided CPE 2.3 match string. The search evaluates direct cpeApplicability.nodes[].cpeMatch entries independently; it does not evaluate a node's operator or negate value as a complete system applicability expression. CPE string literal comparisons are insensitive to lexical case. When supplied, the part must be a, h, o, *, or -. Omitted trailing components and * components act as broad matches; explicit empty colon-delimited components are invalid. When used with versionStart or versionEnd, provide a partial CPE 2.3 match string without a version component. Semantic evaluation, including version range searches, is limited to the first ${constants.MAX_SEARCH_WINDOW} OpenSearch candidates, so broad searches may omit later matches and report incomplete totals.`, { type: 'string', maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.virtualMatchString, @@ -243,7 +243,7 @@ const searchResponseSchemas = { value: { type: 'integer', minimum: 0, - description: 'Number of records matched by the active search strategy.' + description: `Number of records matched by the active search strategy. Standard-search totals are exact. For semantic cpeName and virtualMatchString searches, this value counts matches found within the first ${constants.MAX_SEARCH_WINDOW} OpenSearch candidates and may be incomplete.` }, relation: { type: 'string', From 03cdddaba419a0ed1c16300631fa297bd8ce0a13 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 13:37:01 -0400 Subject: [PATCH 06/10] test: streamline search controller coverage --- controllers/searchController.test.js | 777 +------------------------ utils/searchCandidateProcessor.test.js | 9 + 2 files changed, 35 insertions(+), 751 deletions(-) diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index b5c4ec7..8b68cd0 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -57,7 +57,6 @@ jest.mock('../utils/searchProvider', () => ({ })); const controller = require('./searchController.js'); -const searchCandidateProcessor = require('../utils/searchCandidateProcessor'); const searchInFlightRegistry = require('../utils/searchInFlightRegistry'); const searchPerformance = require('../utils/searchPerformance'); const searchPostFilterCache = require('../utils/searchPostFilterCache'); @@ -88,23 +87,6 @@ function rawResponse(totalValue, hits = []) { return advancedResponse(totalValue, hits).data; } -function cvssRegexp(field, value) { - return { - regexp: { - [`${field}.keyword`]: { - value, - case_insensitive: true, - flags: 'INTERSECTION|ANYSTRING', - rewrite: 'constant_score' - } - } - }; -} - -function cvssValue(...tokens) { - return tokens.map((token) => `(@/)?${token}(/@)?`).join('&'); -} - function basicResponse({ ok = true, status = 'ok', total = 0, hits = [], notes = [], errors = [] } = {}) { return { status, @@ -231,19 +213,13 @@ describe('Search controller SEARCH handler', () => { jest.restoreAllMocks(); }); - test('SEARCH builds filters, query strings, date ranges, and pagination options', async () => { + test('SEARCH executes a standard search and formats pagination metadata', async () => { const resp = advancedResponse(25, [{ _id: 'CVE-2024-0001' }]); mockApiSearch.mockResolvedValue(resp); await controller.SEARCH( { searchParams: { - keywordSearch: 'remote code', - cveIds: 'CVE-2024-0001', - pubStartDate: '2024-01-01T00:00:00.000Z', - pubEndDate: '2024-01-31T00:00:00.000Z', - lastModStartDate: '2024-02-01T00:00:00.000Z', - lastModEndDate: '2024-02-29T00:00:00.000Z', resultsPerPage: 10, pageNumber: '3' } @@ -252,60 +228,11 @@ describe('Search controller SEARCH handler', () => { next ); - expect(mockApiSearch).toHaveBeenCalledWith( - { - query: { - bool: { - must: [ - { - simple_query_string: { - query: 'remote* code*', - default_operator: 'AND', - fields: ['containers.cna.descriptions.value'] - } - } - ], - filter: [ - { - terms: { - 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] - } - }, - { - range: { - 'cveMetadata.datePublished': { - gte: '2024-01-01T00:00:00.000Z', - lte: '2024-01-31T00:00:00.000Z' - } - } - }, - { - range: { - 'cveMetadata.dateUpdated': { - gte: '2024-02-01T00:00:00.000Z', - lte: '2024-02-29T00:00:00.000Z' - } - } - } - ] - } - }, - sort: [ - { - _score: { - order: 'desc' - } - }, - { - 'cveMetadata.cveId.keyword': { - order: 'asc' - } - } - ], - track_total_hits: true, + expect(mockSearchOpenSearch).toHaveBeenCalledWith( + expect.objectContaining({ size: 10, from: 20 - }, + }), expect.objectContaining({ operation: 'standard', signal: expect.any(AbortSignal), @@ -322,227 +249,6 @@ describe('Search controller SEARCH handler', () => { }); }); - test('SEARCH sends keywordSearch through match_phrase when keywordExactMatch is true', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(1)); - - await controller.SEARCH( - { - searchParams: { - keywordSearch: 'remote code', - keywordExactMatch: 'true' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool).toStrictEqual({ - must: [], - filter: [ - { - match_phrase: { - 'containers.cna.descriptions.value': 'remote code' - } - } - ] - }); - }); - - test('SEARCH builds an exact terms filter for cveIds', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(2)); - - await controller.SEARCH( - { - searchParams: { - cveIds: 'CVE-2024-0001,CVE-2024-0002' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - terms: { - 'cveMetadata.cveId.keyword': ['CVE-2024-0001', 'CVE-2024-0002'] - } - } - ]); - expect(mockApiSearch.mock.calls[0][0].query.bool.must).toStrictEqual([]); - }); - - test('SEARCH builds rejected-state exclusions for noRejected', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(2)); - - await controller.SEARCH( - { - searchParams: { - noRejected: true - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - bool: { - must_not: [ - { - terms: { - 'cveMetadata.state.keyword': ['REJECT', 'REJECTED', 'Rejected'] - } - }, - { - terms: { - 'containers.cna.x_legacyV4Record.CVE_data_meta.STATE.keyword': [ - 'REJECT', - 'REJECTED', - 'Rejected' - ] - } - } - ] - } - } - ]); - expect(mockApiSearch.mock.calls[0][0].query.bool.must).toStrictEqual([]); - }); - - test('SEARCH processes virtualMatchString into a structured CPE criteria filter', async () => { - mockSearchRaw.mockResolvedValue(rawResponse(0)); - - await controller.SEARCH( - { - searchParams: { - virtualMatchString: 'cpe:2.3:a:microsoft:windows:*:*:*:*:*:*:*:*' - } - }, - res, - next - ); - - expect(mockSearchRaw.mock.calls[0][0]._source).toStrictEqual([ - 'cveMetadata.cveId', - 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria', - 'containers.cna.cpeApplicability.nodes.cpeMatch.vulnerable', - 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartIncluding', - 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartExcluding', - 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndIncluding', - 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndExcluding' - ]); - expect(mockSearchRaw.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - bool: { - should: [ - 'cpe:2.3:a:microsoft:windows:', - 'cpe:2.3:a:microsoft:*:', - 'cpe:2.3:a:*:windows:', - 'cpe:2.3:a:*:*:', - 'cpe:2.3:*:microsoft:windows:', - 'cpe:2.3:*:microsoft:*:', - 'cpe:2.3:*:*:windows:', - 'cpe:2.3:*:*:*:' - ].map((value) => ({ - prefix: { - 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword': { - value, - case_insensitive: true - } - } - })), - minimum_should_match: 1 - } - } - ]); - expect(mockApiSearch).not.toHaveBeenCalled(); - }); - - test('SEARCH builds filters for CWE, CPE Name criteria, and vulnerable CPE parameters', async () => { - mockSearchRaw.mockResolvedValue(rawResponse(0)); - - await controller.SEARCH( - { - searchParams: { - cweId: 'CWE-79', - cpeName: 'cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*', - isVulnerable: 'true' - } - }, - res, - next - ); - - const body = mockSearchRaw.mock.calls[0][0]; - const filters = body.query.bool.filter; - - expect(filters).toHaveLength(3); - expect(filters[0]).toStrictEqual({ - match_phrase: { - 'containers.cna.problemTypes.descriptions.cweId': 'CWE-79' - } - }); - expect(filters[1].bool.minimum_should_match).toBe(1); - expect(filters[1].bool.should[0]).toStrictEqual({ - bool: { - should: [ - 'cpe:2.3:a:vendor:product:1.0:', - 'cpe:2.3:a:vendor:*:1.0:', - 'cpe:2.3:a:*:product:1.0:', - 'cpe:2.3:a:*:*:1.0:', - 'cpe:2.3:*:vendor:product:1.0:', - 'cpe:2.3:*:vendor:*:1.0:', - 'cpe:2.3:*:*:product:1.0:', - 'cpe:2.3:*:*:*:1.0:' - ].map((value) => ({ - prefix: { - 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword': { - value, - case_insensitive: true - } - } - })), - minimum_should_match: 1 - } - }); - expect(filters[2]).toStrictEqual({ - term: { - 'containers.cna.cpeApplicability.nodes.cpeMatch.vulnerable': true - } - }); - expect(body.size).toBe(10000); - expect(body.from).toBe(0); - expect(mockApiSearch).not.toHaveBeenCalled(); - }); - - test('SEARCH requires the matching CPE entry itself to be vulnerable', async () => { - const matchingNotVulnerable = cpeHit('CVE-2024-0001', { - criteria: 'cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*', - vulnerable: false - }); - matchingNotVulnerable._source.containers.cna.cpeApplicability.nodes[0].cpeMatch.push({ - criteria: 'cpe:2.3:a:other:product:1.0:*:*:*:*:*:*:*', - vulnerable: true - }); - mockSearchRaw.mockResolvedValue(rawResponse(1, [matchingNotVulnerable])); - - await controller.SEARCH( - { - searchParams: { - cpeName: 'cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*', - isVulnerable: true - } - }, - res, - next - ); - - expect(res.status).toHaveBeenCalledWith(200); - expect(res.json.mock.calls[0][0].data.hits.total.value).toBe(0); - expect(res.json.mock.calls[0][0].data.hits.hits).toStrictEqual([]); - expect(mockApiSearch).not.toHaveBeenCalled(); - }); - test('SEARCH applies semantic CPE version filtering before building response metadata', async () => { const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', @@ -582,29 +288,32 @@ describe('Search controller SEARCH handler', () => { next ); - expect(mockApiSearch).toHaveBeenCalledWith( - { - query: { - bool: { - must: [], - filter: [ - { - terms: { - 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] - } - } - ] - } - }, - track_total_hits: false, - size: 1, + expect(mockSearchOpenSearch).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + size: 10000, from: 0 - }, + }), + expect.objectContaining({ + operation: 'cpe-candidate', + signal: expect.any(AbortSignal) + }) + ); + expect(mockSearchOpenSearch).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + size: 1, + from: 0, + track_total_hits: false + }), expect.objectContaining({ operation: 'page-fetch', signal: expect.any(AbortSignal) }) ); + expect(mockApiSearch.mock.calls[0][0].query.bool.filter[0].terms).toStrictEqual({ + 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] + }); expect(res.status).toHaveBeenCalledWith(200); expect(res.json).toHaveBeenCalledWith({ page: 1, @@ -622,169 +331,6 @@ describe('Search controller SEARCH handler', () => { }); }); - test('chunked filtering rejects malformed candidate hit collections', async () => { - await expect( - searchCandidateProcessor.filterSearchHitsInChunks( - { - data: { - hits: {} - } - }, - {}, - undefined, - 100 - ) - ).rejects.toMatchObject({ - code: 'OPEN_SEARCH_RESPONSE_INVALID', - failureType: 'malformed_response' - }); - }); - - test('chunked filtering does not combine platform and version matches from different CPE entries', async () => { - const candidate = cpeHit('CVE-2026-0250', { - criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:macos:*:*:*:*:*', - vulnerable: true, - versionStartIncluding: '6.0', - versionEndExcluding: '6.1' - }); - candidate._source.containers.cna.cpeApplicability.nodes[0].cpeMatch.push( - { - criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:macos:*:*:*:*:*', - vulnerable: true, - versionStartIncluding: '6.2', - versionEndExcluding: '6.2.7' - }, - { - criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:android:*:*:*:*:*', - vulnerable: true, - versionStartIncluding: '6.1', - versionEndExcluding: '6.1.13' - }, - { - criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:chrome_os:*:*:*:*:*', - vulnerable: true, - versionStartIncluding: '6.1', - versionEndExcluding: '6.1.13' - } - ); - - const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(1, [candidate]), - { - cpeName: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:6.1.1:*:macos:*:*:*:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - - expect(filteredHits).toStrictEqual([]); - }); - - test('chunked filtering matches CPE string literals without lexical case', async () => { - const candidate = cpeHit('CVE-2026-0251', { - criteria: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:*:*:*:*:*:macOS:*:*', - vulnerable: true, - versionStartIncluding: '6.0', - versionEndExcluding: '6.1' - }); - - const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(1, [candidate]), - { - cpeName: 'cpe:2.3:a:palo_alto_networks:globalprotect_app:6.0.1:*:*:*:*:macos:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - - expect(filteredHits).toStrictEqual([candidate]); - }); - - test('chunked filtering preserves exact version padding while normalizing numeric range segments', async () => { - const exactCandidate = cpeHit('CVE-2025-9392', { - criteria: 'cpe:2.3:o:linksys:re6250_firmware:1.0.04.001:*:*:*:*:*:*:*', - vulnerable: true - }); - const rangeCandidate = cpeHit('CVE-2026-33280', { - criteria: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:*:*:*:*:*:*:*:*', - vulnerable: true, - versionEndExcluding: '6.02' - }); - - const exactResult = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(1, [exactCandidate]), - { - cpeName: 'cpe:2.3:o:linksys:re6250_firmware:1.0.4.1:*:*:*:*:*:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - const includedRangeResult = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(1, [rangeCandidate]), - { - cpeName: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:6.1:*:*:*:*:*:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - const excludedRangeResult = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(1, [rangeCandidate]), - { - cpeName: 'cpe:2.3:o:buffalo:wsr3600be4-kh_firmware:6.2:*:*:*:*:*:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - - expect(exactResult.filteredHits).toStrictEqual([]); - expect(includedRangeResult.filteredHits).toStrictEqual([rangeCandidate]); - expect(excludedRangeResult.filteredHits).toStrictEqual([]); - }); - - test('chunked filtering preserves CPE NA matches across numeric version ranges', async () => { - const criteria = 'cpe:2.3:a:microsoft:edge:*:*:*:*:*:android:*:*'; - const candidates = [ - cpeHit('CVE-2020-17153', { - criteria, - vulnerable: true, - versionStartIncluding: '-' - }), - cpeHit('CVE-2024-38208', { - criteria, - vulnerable: true, - versionStartIncluding: '1.0.0', - versionEndExcluding: '128.0.2739.42' - }), - cpeHit('CVE-2026-35429', { - criteria, - vulnerable: true, - versionStartIncluding: '1.0.0', - versionEndExcluding: '148.0.3967.55' - }) - ]; - - const { filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(candidates.length, candidates), - { - cpeName: 'cpe:2.3:a:microsoft:edge:-:*:*:*:*:android:*:*', - isVulnerable: true - }, - undefined, - 100 - ); - - expect(filteredHits.map((hit) => hit._id)).toStrictEqual([ - 'CVE-2020-17153', - 'CVE-2024-38208', - 'CVE-2026-35429' - ]); - }); - test('SEARCH applies semantic virtualMatchString version range filtering before building response metadata', async () => { const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' @@ -825,21 +371,9 @@ describe('Search controller SEARCH handler', () => { next ); - expect(mockSearchRaw.mock.calls[0][0].size).toBe(10000); - expect(mockSearchRaw.mock.calls[0][0].from).toBe(0); - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - terms: { - 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] - } - } - ]); - expect(mockApiSearch.mock.calls[0][0]).toMatchObject({ - track_total_hits: false, - size: 1, - from: 0 + expect(mockApiSearch.mock.calls[0][0].query.bool.filter[0].terms).toStrictEqual({ + 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] }); - expect(mockApiSearch.mock.calls[0][0].sort).toBeUndefined(); expect(mockApiSearch.mock.calls[0][1]).toEqual( expect.objectContaining({ operation: 'page-fetch', @@ -1144,85 +678,6 @@ describe('Search controller SEARCH handler', () => { expect(secondRes.status).toHaveBeenCalledWith(200); }); - test('SEARCH applies semantic virtualMatchString filtering before building response metadata', async () => { - const applicable = cpeHit('CVE-2024-0001', { - criteria: 'cpe:2.3:o:microsoft:windows_10_1809:-:*:*:*:*:*:x86:*' - }); - const notApplicable = cpeHit('CVE-2024-0002', { - criteria: 'cpe:2.3:o:microsoft:windows_11_23h2:-:*:*:*:*:*:x86:*' - }); - const fullApplicable = { - _id: 'CVE-2024-0001', - _source: { - cveMetadata: { - cveId: 'CVE-2024-0001' - }, - containers: { - cna: { - title: 'full record' - } - } - } - }; - - mockSearchRaw.mockResolvedValue(rawResponse(2, [applicable, notApplicable])); - mockApiSearch.mockResolvedValue(advancedResponse(1, [fullApplicable])); - - await controller.SEARCH( - { - searchParams: { - virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809', - resultsPerPage: '1', - pageNumber: '1' - } - }, - res, - next - ); - - expect(mockSearchRaw.mock.calls[0][0].size).toBe(10000); - expect(mockSearchRaw.mock.calls[0][0].from).toBe(0); - expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ - page: 1, - pageSize: 1, - totalPages: 1, - status: 'ok', - data: { - hits: { - total: { - value: 1 - }, - hits: [fullApplicable] - } - } - }); - }); - - test('chunked semantic filtering stops after cancellation between batches', async () => { - const hits = [ - cpeHit('CVE-2024-0001', { - criteria: 'cpe:2.3:o:microsoft:windows_10_1809:-:*:*:*:*:*:x86:*' - }), - cpeHit('CVE-2024-0002', { - criteria: 'cpe:2.3:o:microsoft:windows_10_1809:-:*:*:*:*:*:x86:*' - }) - ]; - const abortController = new AbortController(); - const filtering = searchCandidateProcessor.filterSearchHitsInChunks( - advancedResponse(2, hits), - { - virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809' - }, - abortController.signal, - 1 - ); - - abortController.abort(new Error('cancelled')); - - await expect(filtering).rejects.toThrow('cancelled'); - }); - test('SEARCH returns 429 when the cold semantic post-filter queue is full', async () => { const candidateSearch = deferred(); const firstRes = createResponse(); @@ -1525,186 +980,6 @@ describe('Search controller SEARCH handler', () => { expect(console.log.mock.calls[0][0]).not.toContain('remote code'); }); - test('SEARCH builds version-specific CVSS v3 severity filters', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(1)); - - await controller.SEARCH( - { - searchParams: { - cvssV3_0Severity: 'high', - cvssV3_1Severity: 'critical' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - term: { - 'containers.cna.metrics.cvssV3_0.baseSeverity.keyword': 'HIGH' - } - }, - { - term: { - 'containers.cna.metrics.cvssV3_1.baseSeverity.keyword': 'CRITICAL' - } - } - ]); - }); - - test('SEARCH builds cvssV2 metric filters', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(1)); - - await controller.SEARCH( - { - searchParams: { - cvssV2Metrics: 'AV:N/AC:L/Au:N' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - cvssRegexp('containers.cna.metrics.cvssV2_0.vectorString', cvssValue('AC:L', 'Au:N', 'AV:N')) - ]); - }); - - test('SEARCH builds version-specific CVSS v3 metric filters', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(1)); - - await controller.SEARCH( - { - searchParams: { - cvssV3_0Metrics: 'CVSS:3.0/AV:N/AC:L', - cvssV3_1Metrics: 'CVSS:3.1/PR:N/UI:R' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.filter).toStrictEqual([ - cvssRegexp('containers.cna.metrics.cvssV3_0.vectorString', cvssValue('AC:L', 'AV:N')), - cvssRegexp('containers.cna.metrics.cvssV3_1.vectorString', cvssValue('PR:N', 'UI:R')) - ]); - }); - - test('SEARCH escapes Lucene reserved characters in keywordSearch terms', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(1)); - - await controller.SEARCH( - { - searchParams: { - keywordSearch: 'c++ path:/tmp (test)' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.must).toStrictEqual([ - { - simple_query_string: { - query: 'c\\+\\+* path\\:\\/tmp* \\(test\\)*', - default_operator: 'AND', - fields: ['containers.cna.descriptions.value'] - } - } - ]); - }); - - test('SEARCH keeps whitespace-only keywordSearch as an empty simple query string', async () => { - mockApiSearch.mockResolvedValue(advancedResponse(0)); - - await controller.SEARCH( - { - searchParams: { - keywordSearch: ' ' - } - }, - res, - next - ); - - expect(mockApiSearch.mock.calls[0][0].query.bool.must).toStrictEqual([ - { - simple_query_string: { - query: '', - default_operator: 'AND', - fields: ['containers.cna.descriptions.value'] - } - } - ]); - }); - - test('SEARCH preserves escaped colons inside virtualMatchString segments', async () => { - mockSearchRaw.mockResolvedValue(rawResponse(0)); - - await controller.SEARCH( - { - searchParams: { - virtualMatchString: 'cpe:2.3:a:vendor:product\\:edition:*:*:*:*:*:*:*:*' - } - }, - res, - next - ); - - expect(mockSearchRaw.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - bool: { - should: [ - 'cpe:2.3:a:vendor:product\\:edition:', - 'cpe:2.3:a:vendor:*:', - 'cpe:2.3:a:*:product\\:edition:', - 'cpe:2.3:a:*:*:', - 'cpe:2.3:*:vendor:product\\:edition:', - 'cpe:2.3:*:vendor:*:', - 'cpe:2.3:*:*:product\\:edition:', - 'cpe:2.3:*:*:*:' - ].map((value) => ({ - prefix: { - 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword': { - value, - case_insensitive: true - } - } - })), - minimum_should_match: 1 - } - } - ]); - expect(mockApiSearch).not.toHaveBeenCalled(); - }); - - test('SEARCH builds a broad CPE filter for wildcard-only virtualMatchString segments', async () => { - mockSearchRaw.mockResolvedValue(rawResponse(0)); - - await controller.SEARCH( - { - searchParams: { - virtualMatchString: 'cpe:2.3:*:*:*:*:*:*:*:*:*:*:*' - } - }, - res, - next - ); - - expect(mockSearchRaw.mock.calls[0][0].query.bool.filter).toStrictEqual([ - { - regexp: { - 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword': { - value: 'cpe:2\\.3:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*', - case_insensitive: true - } - } - } - ]); - expect(mockApiSearch).not.toHaveBeenCalled(); - }); - test('SEARCH returns zeroed pagination metadata when OpenSearch returns zero hits', async () => { const resp = advancedResponse(0); mockApiSearch.mockResolvedValue(resp); diff --git a/utils/searchCandidateProcessor.test.js b/utils/searchCandidateProcessor.test.js index 9e277f5..d3bd3d3 100644 --- a/utils/searchCandidateProcessor.test.js +++ b/utils/searchCandidateProcessor.test.js @@ -69,6 +69,15 @@ describe('searchCandidateProcessor', () => { jest.restoreAllMocks(); }); + test('rejects a malformed OpenSearch hits response', async () => { + await expect( + searchCandidateProcessor.filterSearchHitsInChunks({ data: { hits: {} } }, {}) + ).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_response' + }); + }); + test('validates and filters candidates in the same processing pass', async () => { const matching = cpeCandidateHit({ versionStartIncluding: '2.9', From 87a893f59c8e20dba5f243b1562a5bb86680b446 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 13:56:45 -0400 Subject: [PATCH 07/10] test: consolidate delivery policy coverage --- config/runtime-config.test.js | 40 ----------------------------------- test/delivery-policy.test.js | 40 ++++++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index 7f24e84..77486cf 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -79,46 +79,6 @@ describe('runtime container and compose configuration', () => { expect(dockerignore).toContain('**/reports'); }); - test('CI builds, scans, and smoke tests the production runtime image', () => { - const workflow = readProjectFile('.github', 'workflows', 'quality.yml'); - - expect(workflow).toContain('pull_request:'); - expect(workflow).toContain('push:'); - expect(workflow).toContain('merge_group:'); - expect(workflow).toContain('name: Dependency Review'); - expect(workflow).toContain('actions/dependency-review-action@'); - expect(workflow).toContain('name: Runtime Container'); - expect(workflow).toContain('docker build --tag cve-search-api:pr .'); - expect(workflow).toContain('aquasecurity/trivy-action@'); - expect(workflow).toContain('trivyignores: .trivyignore'); - expect(workflow).toContain('version: v0.69.3'); - expect(workflow).toContain('http://127.0.0.1:4100/api/health-check'); - expect(workflow).toContain('http://127.0.0.1:4100/api-docs/openapi.json'); - expect(workflow).toContain(`--format '{{.Config.User}}'`); - expect(workflow).toContain(`--format '{{json .Config.Cmd}}'`); - expect(workflow).not.toContain('/api/ready-check'); - }); - - test('manual OpenSearch contract workflow uses a protected environment and secrets', () => { - const workflow = readProjectFile('.github', 'workflows', 'opensearch-contract.yml'); - - expect(workflow).toContain('workflow_dispatch:'); - expect(workflow).toContain('type: environment'); - expect(workflow).toContain('environment: ${{ inputs.environment }}'); - expect(workflow).toContain('${{ secrets.OPENSEARCH_DOMAIN_ENDPOINT }}'); - expect(workflow).toContain('${{ secrets.OPENSEARCH_CVE_INDEX }}'); - expect(workflow).toContain('npm run test:opensearch-contract'); - expect(workflow).not.toMatch(/https?:\/\//); - }); - - test('Dependabot tracks npm, Docker, and GitHub Actions dependencies', () => { - const dependabot = readProjectFile('.github', 'dependabot.yml'); - - expect(dependabot).toContain('package-ecosystem: npm'); - expect(dependabot).toContain('package-ecosystem: docker'); - expect(dependabot).toContain('package-ecosystem: github-actions'); - }); - test('server-only compose file joins the existing OpenSearch 3.5 network', () => { const compose = readProjectFile('docker-compose.yaml'); diff --git a/test/delivery-policy.test.js b/test/delivery-policy.test.js index 65af699..3986ed6 100644 --- a/test/delivery-policy.test.js +++ b/test/delivery-policy.test.js @@ -25,22 +25,54 @@ describe('delivery security policy', () => { }); test('exposes OpenSearch secrets only to the trusted contract execution step', () => { - const workflow = readWorkflow('.github/workflows/opensearch-contract.yml'); + const workflowPath = '.github/workflows/opensearch-contract.yml'; + const workflowSource = readRepositoryFile(workflowPath); + const workflow = readWorkflow(workflowPath); const job = workflow.jobs.contract; const installStep = job.steps.find(({ name }) => name === 'Install production dependencies'); const contractStep = job.steps.find(({ name }) => name === 'Run read-only contract checks'); + expect(workflow.on.workflow_dispatch.inputs.environment).toMatchObject({ + required: true, + type: 'environment' + }); expect(job.if).toBe("github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main'"); + expect(job.environment).toBe('${{ inputs.environment }}'); expect(job.env).toStrictEqual({ NODE_ENV: 'production', NODE_CONFIG_ENV: 'prod' }); expect(installStep.env).toBeUndefined(); + expect(contractStep.run).toBe('npm run test:opensearch-contract'); expect(contractStep.env).toStrictEqual({ OpenSearchDomainEndpoint: '${{ secrets.OPENSEARCH_DOMAIN_ENDPOINT }}', OpenSearchCveIndex: '${{ secrets.OPENSEARCH_CVE_INDEX }}', OpenSearchAllowUnknownSslCerts: "${{ vars.OPENSEARCH_ALLOW_UNKNOWN_SSL_CERTS || 'false' }}" }); + expect(workflowSource).not.toMatch(/https?:\/\//); + }); + + test('runs repository quality and production image validation for supported events', () => { + const workflow = readWorkflow('.github/workflows/quality.yml'); + const dependencyReviewStep = workflow.jobs['dependency-review'].steps.find( + ({ name }) => name === 'Review dependency changes' + ); + const qualityStep = workflow.jobs.quality.steps.find(({ name }) => name === 'Run quality checks'); + const containerJob = workflow.jobs.container; + const buildStep = containerJob.steps.find(({ name }) => name === 'Build production image'); + const validationStep = containerJob.steps.find(({ name }) => name === 'Validate runtime image'); + + expect(Object.keys(workflow.on)).toStrictEqual(['pull_request', 'push', 'merge_group']); + expect(workflow.on.push.branches).toStrictEqual(['dev', 'main']); + expect(dependencyReviewStep.uses).toMatch(/^actions\/dependency-review-action@[a-f0-9]{40}$/); + expect(qualityStep.run).toBe('npm run quality'); + expect(containerJob.name).toBe('Runtime Container'); + expect(buildStep.run).toBe('docker build --tag cve-search-api:pr .'); + expect(validationStep.run).toContain('http://127.0.0.1:4100/api/health-check'); + expect(validationStep.run).toContain('http://127.0.0.1:4100/api-docs/openapi.json'); + expect(validationStep.run).toContain(`--format '{{.Config.User}}'`); + expect(validationStep.run).toContain(`--format '{{json .Config.Cmd}}'`); + expect(validationStep.run).not.toContain('/api/ready-check'); }); test('reports all severe image findings before enforcing the fixable vulnerability gate', () => { @@ -61,13 +93,15 @@ describe('delivery security policy', () => { 'exit-code': 0, 'ignore-unfixed': false, severity: 'CRITICAL,HIGH', - trivyignores: '/dev/null' + trivyignores: '/dev/null', + version: 'v0.69.3' } }); expect(steps[gateIndex].with).toMatchObject({ 'exit-code': 1, 'ignore-unfixed': true, - trivyignores: '.trivyignore' + trivyignores: '.trivyignore', + version: 'v0.69.3' }); }); From 248fe68eeee71dc4d7296bbe1d832cde0be877e8 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 14:00:38 -0400 Subject: [PATCH 08/10] test: organize OpenAPI contract coverage --- api-docs/openapi.test.js | 68 +++++++++++++++------------------------- 1 file changed, 26 insertions(+), 42 deletions(-) diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 66115a7..8ed149b 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -6,9 +6,7 @@ const packageInfo = require('../package.json'); const constants = require('../utils/constants'); const { generateOpenApi } = require('../routes/swagger'); -function jsonRequestBodySchema(path, method) { - const schema = openapi.paths[path][method].requestBody.content['application/json'].schema; - +function resolveSchema(schema) { if (!schema.$ref) return schema; return schema.$ref @@ -17,13 +15,6 @@ function jsonRequestBodySchema(path, method) { .reduce((value, key) => value[key], openapi); } -function resolveSchemaRef(ref) { - return ref - .replace('#/', '') - .split('/') - .reduce((value, key) => value[key], openapi); -} - describe('OpenAPI contract', () => { test('documents the supported route surface', () => { expect(openapi.paths['/search'].post.operationId).toBe('search'); @@ -62,8 +53,8 @@ describe('OpenAPI contract', () => { test('POST /search documents all JSON body fields accepted by the router and controller', () => { const operation = openapi.paths['/search'].post; - const schema = jsonRequestBodySchema('/search', 'post'); const mediaType = operation.requestBody.content['application/json']; + const schema = resolveSchema(mediaType.schema); expect(operation.parameters || []).toStrictEqual([]); expect(operation.requestBody.description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`); @@ -89,18 +80,6 @@ describe('OpenAPI contract', () => { }); expect(schema.properties.noRejected.type).toBe('boolean'); expect(schema.properties.noRejected.description).toContain('When true'); - expect(schema.properties.isVulnerable.description).toContain('same CPE Match Criteria entry'); - expect(schema.properties.isVulnerable.description).toContain('False or omitted'); - expect(schema.properties.cpeName.description).toContain('Maven-style ordering'); - expect(schema.properties.cpeName.description).toContain('part must be a, h, or o'); - expect(schema.properties.cpeName.description).toContain( - 'Every concrete non-version component must be compatible with the same CPE Match Criteria entry' - ); - expect(schema.properties.cpeName.description).toContain('optional * components remain broad matches'); - expect(schema.properties.cpeName.description).toContain('insensitive to lexical case'); - expect(schema.properties.cpeName.description).toContain('exact CPE Match Criteria preserve leading zeros'); - expect(schema.properties.cpeName.description).toContain('entries independently'); - expect(schema.properties.cpeName.description).toContain("does not evaluate a node's operator or negate value"); expect(schema.properties.versionStartType.enum).toStrictEqual(constants.CPE_VERSION_RANGE_TYPES); expect(schema.properties.versionEndType.enum).toStrictEqual(constants.CPE_VERSION_RANGE_TYPES); expect(schema.properties.resultsPerPage.maximum).toBe(constants.MAX_RESULTS_LIMIT); @@ -115,19 +94,6 @@ describe('OpenAPI contract', () => { expect(schema.properties.versionEnd.description).toContain( 'prerelease qualifiers before an unqualified release' ); - expect(schema.properties.virtualMatchString.description).toContain('CPE Match Criteria'); - expect(schema.properties.virtualMatchString.description).toContain('compatible'); - expect(schema.properties.virtualMatchString.description).toContain('insensitive to lexical case'); - expect(schema.properties.virtualMatchString.description).toContain('without a version component'); - expect(schema.properties.virtualMatchString.description).toContain('part must be a, h, o, *, or -'); - expect(schema.properties.virtualMatchString.description).toContain( - 'explicit empty colon-delimited components are invalid' - ); - expect(schema.properties.virtualMatchString.description).toContain('entries independently'); - expect(schema.properties.virtualMatchString.description).toContain( - "does not evaluate a node's operator or negate value" - ); - expect(schema.properties.virtualMatchString.example).toBe('cpe:2.3:o:microsoft:windows_10_1809'); expect(schema.properties.cweId.description).toContain('CNA data'); expect(schema.properties.cweId.description).toContain('ADP-provided CWE data is not searched'); expect(schema.properties.cvssV3Metrics.description).toContain('same stored vector string'); @@ -166,22 +132,37 @@ describe('OpenAPI contract', () => { expect(operation.responses['410'].description).toContain('use POST /search'); }); - test('documents cpeName as a CPE Match Criteria filter with required components', () => { - const cpeName = openapi.components.schemas.SearchRequest.properties.cpeName; + test('documents semantic CPE filter behavior', () => { + const { cpeName, isVulnerable, virtualMatchString } = openapi.components.schemas.SearchRequest.properties; + expect(isVulnerable.description).toContain('same CPE Match Criteria entry'); + expect(isVulnerable.description).toContain('False or omitted'); expect(cpeName.description).toContain('CPE Match Criteria'); expect(cpeName.description).toContain('part must be a, h, or o'); + expect(cpeName.description).toContain( + 'Every concrete non-version component must be compatible with the same CPE Match Criteria entry' + ); + expect(cpeName.description).toContain('optional * components remain broad matches'); expect(cpeName.description).toContain('version range fields'); expect(cpeName.description).toContain('insensitive to lexical case'); expect(cpeName.description).toContain('exact CPE Match Criteria preserve leading zeros'); + expect(cpeName.description).toContain('Maven-style ordering'); expect(cpeName.description).toContain('logical value NA'); expect(cpeName.description).toContain('numeric range boundaries'); + expect(cpeName.description).toContain('entries independently'); + expect(cpeName.description).toContain("does not evaluate a node's operator or negate value"); expect(cpeName.description).toContain('HTTP 404'); expect(cpeName.description).toContain('virtualMatchString'); expect(cpeName.example).toBe('cpe:2.3:a:freetype:freetype:2.8.1:*:*:*:*:*:*:*'); - expect(openapi.paths['/search'].post.responses['404'].description).toBe('Invalid cpeName value'); - expect(openapi.paths['/search'].post.responses['429'].description).toContain('rate limit'); - expect(openapi.paths['/search'].post.responses['429'].description).toContain('waiter limit'); + expect(virtualMatchString.description).toContain('CPE Match Criteria'); + expect(virtualMatchString.description).toContain('compatible'); + expect(virtualMatchString.description).toContain('insensitive to lexical case'); + expect(virtualMatchString.description).toContain('without a version component'); + expect(virtualMatchString.description).toContain('part must be a, h, o, *, or -'); + expect(virtualMatchString.description).toContain('explicit empty colon-delimited components are invalid'); + expect(virtualMatchString.description).toContain('entries independently'); + expect(virtualMatchString.description).toContain("does not evaluate a node's operator or negate value"); + expect(virtualMatchString.example).toBe('cpe:2.3:o:microsoft:windows_10_1809'); }); test('documents the bounded semantic CPE candidate window', () => { @@ -216,7 +197,10 @@ describe('OpenAPI contract', () => { expect(responses['400'].description).toContain('Malformed JSON'); expect(responses['400'].description).toContain('CVSS vector'); expect(responses['400'].description).toContain('date range'); + expect(responses['404'].description).toBe('Invalid cpeName value'); expect(responses['415'].description).toContain('application/json'); + expect(responses['429'].description).toContain('rate limit'); + expect(responses['429'].description).toContain('waiter limit'); expect(responses['502'].description).toContain('invalid response'); expect(responses['503'].description).toContain('OpenSearch'); expect(responses['503'].headers['Retry-After']).toStrictEqual({ @@ -230,7 +214,7 @@ describe('OpenAPI contract', () => { test('documents the complete POST /search success and error response shapes', () => { const responses = openapi.paths['/search'].post.responses; const successRef = responses['200'].content['application/json'].schema.$ref; - const successSchema = resolveSchemaRef(successRef); + const successSchema = resolveSchema(responses['200'].content['application/json'].schema); expect(successRef).toBe('#/components/schemas/SearchResponse'); expect(successSchema.required).toStrictEqual(['page', 'pageSize', 'totalPages', 'status', 'data']); From cee5d8866648d7e5a335226c5da272b2a3cc506b Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 14:44:11 -0400 Subject: [PATCH 09/10] docs: align documentation with current API behavior --- README.md | 20 +-- api-docs/openapi.json | 307 +++++++++++++++++++++++++++++++++++++-- api-docs/openapi.test.js | 67 +++++++-- config/default.jsonc | 2 +- routes/index.js | 242 ++++++++++++++++++++++++++++-- routes/swagger.js | 53 +++++++ utils/constants.js | 14 +- 7 files changed, 659 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 5946ff0..4514cda 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Running the API locally -Rename `.env.example` to `.env` and replace or check the following values. +Copy `.env.example` to `.env` and replace or check the following values. - `NODE_CONFIG_ENV` - node-config profile name; use `devel` locally and `prod` in production images - `PORT` - port used for the search API, defaulted to 3000 for development @@ -33,7 +33,7 @@ Rename `.env.example` to `.env` and replace or check the following values. - `NPM_CA_CERT` - optional absolute host path to a corporate or self-signed CA certificate bundle used only during local Docker Compose builds ```bash -mv .env.example .env +cp .env.example .env ``` ### Runtime Configuration Validation @@ -101,9 +101,9 @@ docker compose -f docker-compose.yaml up --build The compose file runs only the Search API with `NODE_ENV=production` and `NODE_CONFIG_ENV=prod`. -## Deploying with npm +## Running with npm -**NOTE: The current locked `cve-core` dependency requires Node.js `>= 16.20.2 < 22`.** +Use Node.js 24.14.1 to match the repository's CI jobs and Docker build/runtime images. The current locked `cve-core` package metadata still declares Node.js `>= 16.20.2 < 22`, so npm can emit an engine warning during installation. Updating that upstream engine declaration belongs in the separately owned `cve-core` repository; this repository validates its current application and container behavior on Node.js 24.14.1. 1. Set `OpenSearchDomainEndpoint` in `.env` to an existing OpenSearch deployment: @@ -114,7 +114,7 @@ OpenSearchDomainEndpoint=https://search.example.com 2. Install dependencies: ```bash -npm install +npm ci ``` 3. Run the CVE Search API in development mode: @@ -129,7 +129,7 @@ npm run dev npm start ``` -The Search API will be served at `localhost:3000`. +The Search API will be served at `http://localhost:3000` by default, or at the port configured with `PORT`. Successful port binding emits a sanitized `runtime.lifecycle` event with the `started` outcome. Synchronous configuration failures and asynchronous listener failures such as an unavailable or unauthorized port emit `startup_failed`, set a nonzero process exit code, and do not emit `started`. Startup failure logs contain only a stable error code and, for configuration errors, the relevant configuration path; exception messages, stacks, addresses, and port details are omitted. @@ -292,11 +292,11 @@ The runtime container and Docker Compose serve the committed `api-docs/openapi.j npm run swagger-autogen ``` -The Swagger API docs will be served at `localhost:3000/api-docs` while the CVE Search API is running via Docker or npm. +The Swagger API docs will be served at `http://localhost:3000/api-docs` while the CVE Search API is running via Docker or npm. The Swagger UI is intentionally read-only: interactive request submission and the "Try it out" controls are disabled. Use an HTTP client or the documented curl examples to call the API. ## Testing -Unit, route, OpenAPI, runtime config, and error helper tests can be run with npm: +Unit, integration-style route, OpenAPI contract, runtime configuration, delivery policy, and utility tests can be run with npm: ```bash npm test @@ -313,7 +313,7 @@ npm run quality Pull requests, merge-queue validation, and pushes to `dev` or `main` run the quality gate and independently build the production `Dockerfile`. The container job reports all high and critical image vulnerabilities without blocking, then rejects fixed high and critical vulnerabilities that are not covered by an approved exception. It starts the image, checks process liveness and the packaged OpenAPI document, and verifies the runtime user and command. It intentionally does not call `/api/ready-check` or `/search`, because the standard CI workflow does not own or provision an OpenSearch index. -Pull requests also run GitHub dependency review and reject newly introduced high or critical vulnerabilities in runtime or development dependencies. The blocking production image scan excludes only `CVE-2026-39244`, an `adm-zip` advisory inherited from the separately owned `cve-core` dependency. The exception records its package owner, fixed version, and review deadline in `.trivyignore`; repository tests fail after that deadline or if the dependency provenance changes. The non-blocking scan still reports the exception and other unfixed severe findings. New advisories are not covered by the exception. +Pull requests also run GitHub dependency review and reject newly introduced high or critical vulnerabilities in runtime or development dependencies. Because this repository is private, its dependency graph and GitHub Code Security must remain enabled for that job to run. The blocking production image scan excludes only `CVE-2026-39244`, an `adm-zip` advisory inherited from the separately owned `cve-core` dependency. The exception records its package owner, fixed version, and review deadline in `.trivyignore`; repository tests fail after that deadline or if the dependency provenance changes. The non-blocking scan still reports the exception and other unfixed severe findings. New advisories are not covered by the exception. An opt-in, read-only OpenSearch contract check is available for an approved environment that can reach an existing index: @@ -390,7 +390,7 @@ The cache is intentionally small and short-lived: - Post-filter evaluation chunk: 250 CPE Match Criteria entries between cancellation checks and event-loop yields - OpenSearch request timeout: 20 seconds for each operation - Overall `/search` deadline: 30 seconds -- Eviction: oldest entries are removed when the cache exceeds its max size +- Eviction: least-recently-used entries are removed when the cache exceeds its max size The admission queue applies only to cold CPE candidate searches. Standard `/search` requests and page fetches are not placed in this queue. When the cold-search queue is full, new unique CPE post-filter searches fail closed with HTTP 429 and a `SEARCH_QUEUE_FULL` response. Identical requests for a search that is already in flight still wait on the shared in-flight result and do not consume additional queue slots, up to the per-operation waiter limit. Additional callers fail closed with HTTP 429 and `SEARCH_IN_FLIGHT_WAITER_LIMIT`. diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 2bb9200..c2ec8e4 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -44,6 +44,14 @@ }, "400": { "description": "Malformed JSON or invalid request body, parameter, CVSS vector, date range, or version range", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -61,6 +69,14 @@ }, "404": { "description": "Invalid cpeName value", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -71,6 +87,14 @@ }, "413": { "description": "The JSON request body exceeded the 100 KB limit", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -81,6 +105,14 @@ }, "415": { "description": "A non-empty request body did not use the application/json media type with UTF-8 encoding", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -109,6 +141,12 @@ "schema": { "type": "string" } + }, + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } } }, "content": { @@ -121,6 +159,14 @@ }, "500": { "description": "An unexpected internal server error occurred", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -131,6 +177,14 @@ }, "502": { "description": "OpenSearch returned an invalid response or an unexpected upstream error occurred", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -147,6 +201,12 @@ "schema": { "type": "integer" } + }, + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } } }, "content": { @@ -159,6 +219,14 @@ }, "504": { "description": "Search exceeded its OpenSearch or overall processing deadline", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -293,6 +361,13 @@ "type": "string" } } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } } } } @@ -304,11 +379,11 @@ "CVE Record" ], "summary": "Deprecated raw query search", - "description": "

Expected Behavior

Retrieves all matching CVE records after applying the JSON request body. This endpoint is deprecated and can be disabled by the service operator. Use POST /search instead.

", + "description": "

Expected Behavior

Retrieves one bounded window of matching CVE records after applying the JSON request body. The resultsTotal field reports the total match count independently from the returned data window. This endpoint is deprecated and can be disabled by the service operator. Use POST /search instead.

", "operationId": "webSearch", "responses": { "200": { - "description": "A list of matching CVE records", + "description": "A bounded window of matching CVE records", "headers": { "X-Request-ID": { "description": "Request correlation ID", @@ -316,20 +391,158 @@ "type": "string" } } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebSearchResponse" + } + } } }, "400": { - "description": "Bad Request" + "description": "Malformed JSON or invalid raw-query request fields", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ErrorResponse" + }, + { + "$ref": "#/components/schemas/ValidationErrorList" + } + ] + } + } + } }, "410": { - "description": "The deprecated endpoint is disabled; use POST /search" + "description": "The deprecated endpoint is disabled; use POST /search", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "413": { + "description": "The JSON request body exceeded the 100 KB limit", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "500": { + "description": "An unexpected internal server error occurred", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "502": { + "description": "OpenSearch returned an invalid response or an unexpected upstream error occurred", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebSearchResponse" + } + } + } + }, + "503": { + "description": "The service is shutting down", + "headers": { + "Retry-After": { + "description": "Seconds before retrying the request", + "schema": { + "type": "integer" + } + }, + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } }, "504": { - "description": "Search exceeded its OpenSearch request deadline" + "description": "Search exceeded its OpenSearch request deadline", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebSearchResponse" + } + } + } } }, "requestBody": { "required": true, + "description": "JSON raw-query search and optional bounded window controls. The JSON request body cannot exceed 100 KB.", "content": { "application/json": { "schema": { @@ -341,36 +554,46 @@ "properties": { "query": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "Non-empty raw search query interpreted by the installed cve-core BasicSearchManager." }, "sort": { "type": "object", "additionalProperties": false, + "required": [ + "property" + ], + "description": "Optional sort selection. sort.property is required when sort is provided; sort.order defaults to desc when omitted.", "properties": { "property": { "type": "string", "enum": [ "cveId", "relevancy" - ] + ], + "description": "Sort by CVE ID or OpenSearch relevance score." }, "order": { "type": "string", "enum": [ "asc", "desc" - ] + ], + "default": "desc", + "description": "Sort direction." } } }, "from": { "type": "integer", - "minimum": 0 + "minimum": 0, + "description": "Zero-based offset into matching records. from plus size cannot exceed 10000." }, "size": { "type": "integer", "minimum": 1, - "maximum": 500 + "maximum": 500, + "description": "Maximum number of records returned in this response. from plus size cannot exceed 10000." } } } @@ -715,6 +938,70 @@ } } }, + "WebSearchMetadata": { + "type": "object", + "required": [ + "searchStatus", + "version", + "cveCore", + "cveMitreCore", + "notes", + "errors" + ], + "additionalProperties": false, + "properties": { + "searchStatus": { + "type": "string" + }, + "version": { + "type": "string" + }, + "cveCore": { + "type": "string" + }, + "cveMitreCore": { + "type": "string" + }, + "queryTime_ms": { + "type": "number", + "minimum": 0 + }, + "notes": { + "type": "array", + "items": {} + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "WebSearchResponse": { + "type": "object", + "required": [ + "searchMetadata", + "resultsTotal", + "data" + ], + "additionalProperties": false, + "properties": { + "searchMetadata": { + "$ref": "#/components/schemas/WebSearchMetadata" + }, + "resultsTotal": { + "type": "integer", + "minimum": 0 + }, + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SearchHit" + } + } + } + }, "ErrorResponse": { "type": "object", "required": [ diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 8ed149b..166024b 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -121,15 +121,55 @@ describe('OpenAPI contract', () => { expect(operation.deprecated).toBe(true); expect(operation.description).toContain('Use POST /search instead'); + expect(operation.description).toContain('one bounded window'); + expect(operation.description).toContain('resultsTotal'); expect(operation.parameters || []).toStrictEqual([]); expect(operation.requestBody.required).toBe(true); + expect(operation.requestBody.description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`); expect(schema.required).toStrictEqual(['query']); expect(schema.additionalProperties).toBe(false); expect(schema.properties.query.type).toBe('string'); expect(schema.properties.sort.additionalProperties).toBe(false); + expect(schema.properties.sort.required).toStrictEqual(['property']); expect(schema.properties.sort.properties.property.enum).toStrictEqual(['cveId', 'relevancy']); + expect(schema.properties.sort.properties.order.default).toBe('desc'); expect(schema.properties.size.maximum).toBe(constants.MAX_RESULTS_LIMIT); + expect(operation.responses['200'].content['application/json'].schema).toStrictEqual({ + $ref: '#/components/schemas/WebSearchResponse' + }); + expect(operation.responses['400'].content['application/json'].schema.oneOf).toStrictEqual([ + { + $ref: '#/components/schemas/ErrorResponse' + }, + { + $ref: '#/components/schemas/ValidationErrorList' + } + ]); expect(operation.responses['410'].description).toContain('use POST /search'); + ['410', '413', '500', '503'].forEach((status) => { + expect(operation.responses[status].content['application/json'].schema).toStrictEqual({ + $ref: '#/components/schemas/ErrorResponse' + }); + }); + ['502', '504'].forEach((status) => { + expect(operation.responses[status].content['application/json'].schema).toStrictEqual({ + $ref: '#/components/schemas/WebSearchResponse' + }); + }); + + expect(openapi.components.schemas.WebSearchResponse.required).toStrictEqual([ + 'searchMetadata', + 'resultsTotal', + 'data' + ]); + expect(openapi.components.schemas.WebSearchMetadata.required).toStrictEqual([ + 'searchStatus', + 'version', + 'cveCore', + 'cveMitreCore', + 'notes', + 'errors' + ]); }); test('documents semantic CPE filter behavior', () => { @@ -232,7 +272,7 @@ describe('OpenAPI contract', () => { } ]); - ['404', '415', '429', '500', '502', '503', '504'].forEach((status) => { + ['404', '413', '415', '429', '500', '502', '503', '504'].forEach((status) => { expect(responses[status].content['application/json'].schema).toStrictEqual({ $ref: '#/components/schemas/ErrorResponse' }); @@ -242,17 +282,26 @@ describe('OpenAPI contract', () => { }); test('documents request correlation headers', () => { - expect(openapi.paths['/search'].post.responses['200'].headers['X-Request-ID']).toStrictEqual({ - description: 'Request correlation ID', - schema: { - type: 'string' - } + Object.values(openapi.paths).forEach((pathItem) => { + Object.values(pathItem).forEach((operation) => { + Object.values(operation.responses).forEach((response) => { + expect(response.headers['X-Request-ID']).toStrictEqual({ + description: 'Request correlation ID', + schema: { + type: 'string' + } + }); + }); + }); }); - expect(openapi.paths['/api/health-check'].get.responses['200'].headers['X-Request-ID']).toBeDefined(); - expect(openapi.paths['/api/ready-check'].get.responses['200'].headers['X-Request-ID']).toBeDefined(); + expect(openapi.paths['/api/ready-check'].get.responses['503'].headers['Retry-After']).toBeDefined(); expect(openapi.paths['/api/ready-check'].get.responses['503'].description).toContain('shutting down'); - expect(openapi.paths['/webSearch'].post.responses['200'].headers['X-Request-ID']).toBeDefined(); + expect(openapi.paths['/api/ready-check'].get.responses['503'].content['application/json'].schema).toStrictEqual( + { + $ref: '#/components/schemas/ErrorResponse' + } + ); }); test('committed OpenAPI artifact matches generated route documentation', async () => { diff --git a/config/default.jsonc b/config/default.jsonc index b04bc01..7f05ddc 100644 --- a/config/default.jsonc +++ b/config/default.jsonc @@ -2,7 +2,7 @@ // Default configuration // These values are overridable using other *.jsonc (e.g., prod.jsonc) // as well as using environment variables (e.g., in `.env`) - // - Each configuration is mapped using custom-environment-variables.jsonc to enable environment varialbe overrides. + // - Each configuration is mapped using custom-environment-variables.jsonc to enable environment variable overrides. // - For more information, see cve-core/src/adapters/config/AppConfig.ts // NOTE for consistency, all values need to be strings for proper type when using .env overrides // Explicit invalid values fail application startup. See README.md for accepted ranges and diff --git a/routes/index.js b/routes/index.js index e76a818..ce80d39 100644 --- a/routes/index.js +++ b/routes/index.js @@ -266,6 +266,14 @@ content: { } #swagger.responses[400] = { description: 'Malformed JSON or invalid request body, parameter, CVSS vector, date range, or version range', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -283,6 +291,14 @@ content: { } #swagger.responses[404] = { description: 'Invalid cpeName value', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -293,6 +309,14 @@ content: { } #swagger.responses[415] = { description: 'A non-empty request body did not use the application/json media type with UTF-8 encoding', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -303,6 +327,14 @@ content: { } #swagger.responses[413] = { description: 'The JSON request body exceeded the 100 KB limit', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -331,6 +363,12 @@ headers: { schema: { type: "string" } + }, + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } } }, content: { @@ -343,6 +381,14 @@ content: { } #swagger.responses[502] = { description: 'OpenSearch returned an invalid response or an unexpected upstream error occurred', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -359,6 +405,12 @@ headers: { schema: { type: "integer" } + }, + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } } }, content: { @@ -371,6 +423,14 @@ content: { } #swagger.responses[504] = { description: 'Search exceeded its OpenSearch or overall processing deadline', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -381,6 +441,14 @@ content: { } #swagger.responses[500] = { description: 'An unexpected internal server error occurred', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, content: { "application/json": { schema: { @@ -477,6 +545,13 @@ headers: { type: "string" } } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } } } */ @@ -492,9 +567,10 @@ router.post( #swagger.summary = "Deprecated raw query search" #swagger.description = "

Expected Behavior

-

Retrieves all matching CVE records after applying the JSON request body. This endpoint is deprecated and can be disabled by the service operator. Use POST /search instead.

+

Retrieves one bounded window of matching CVE records after applying the JSON request body. The resultsTotal field reports the total match count independently from the returned data window. This endpoint is deprecated and can be disabled by the service operator. Use POST /search instead.

#swagger.requestBody = { required: true, + description: "JSON raw-query search and optional bounded window controls. The JSON request body cannot exceed 100 KB.", content: { "application/json": { schema: { @@ -504,30 +580,38 @@ router.post( properties: { query: { type: "string", - minLength: 1 + minLength: 1, + description: "Non-empty raw search query interpreted by the installed cve-core BasicSearchManager." }, sort: { type: "object", additionalProperties: false, + required: ["property"], + description: "Optional sort selection. sort.property is required when sort is provided; sort.order defaults to desc when omitted.", properties: { property: { type: "string", - enum: ["cveId", "relevancy"] + enum: ["cveId", "relevancy"], + description: "Sort by CVE ID or OpenSearch relevance score." }, order: { type: "string", - enum: ["asc", "desc"] + enum: ["asc", "desc"], + default: "desc", + description: "Sort direction." } } }, from: { type: "integer", - minimum: 0 + minimum: 0, + description: "Zero-based offset into matching records. from plus size cannot exceed 10000." }, size: { type: "integer", minimum: 1, - maximum: 500 + maximum: 500, + description: "Maximum number of records returned in this response. from plus size cannot exceed 10000." } } } @@ -535,7 +619,7 @@ router.post( } } #swagger.responses[200] = { -description: 'A list of matching CVE records', +description: 'A bounded window of matching CVE records', headers: { "X-Request-ID": { description: "Request correlation ID", @@ -543,13 +627,153 @@ headers: { type: "string" } } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/WebSearchResponse" + } + } } } -#swagger.responses[504] = { -description: 'Search exceeded its OpenSearch request deadline', +#swagger.responses[400] = { +description: 'Malformed JSON or invalid raw-query request fields', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + oneOf: [ + { + $ref: "#/components/schemas/ErrorResponse" + }, + { + $ref: "#/components/schemas/ValidationErrorList" + } + ] + } + } +} } #swagger.responses[410] = { description: 'The deprecated endpoint is disabled; use POST /search', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} +#swagger.responses[413] = { +description: 'The JSON request body exceeded the 100 KB limit', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} +#swagger.responses[500] = { +description: 'An unexpected internal server error occurred', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} +#swagger.responses[502] = { +description: 'OpenSearch returned an invalid response or an unexpected upstream error occurred', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/WebSearchResponse" + } + } +} +} +#swagger.responses[503] = { +description: 'The service is shutting down', +headers: { + "Retry-After": { + description: "Seconds before retrying the request", + schema: { + type: "integer" + } + }, + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} +#swagger.responses[504] = { +description: 'Search exceeded its OpenSearch request deadline', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/WebSearchResponse" + } + } +} } */ requireWebSearchEnabled, diff --git a/routes/swagger.js b/routes/swagger.js index 2982644..0629922 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -348,6 +348,59 @@ const searchResponseSchemas = { } } }, + WebSearchMetadata: { + type: 'object', + required: ['searchStatus', 'version', 'cveCore', 'cveMitreCore', 'notes', 'errors'], + additionalProperties: false, + properties: { + searchStatus: { + type: 'string' + }, + version: { + type: 'string' + }, + cveCore: { + type: 'string' + }, + cveMitreCore: { + type: 'string' + }, + queryTime_ms: { + type: 'number', + minimum: 0 + }, + notes: { + type: 'array', + items: {} + }, + errors: { + type: 'array', + items: { + $ref: '#/components/schemas/ErrorResponse' + } + } + } + }, + WebSearchResponse: { + type: 'object', + required: ['searchMetadata', 'resultsTotal', 'data'], + additionalProperties: false, + properties: { + searchMetadata: { + $ref: '#/components/schemas/WebSearchMetadata' + }, + resultsTotal: { + type: 'integer', + minimum: 0 + }, + data: { + type: 'array', + items: { + $ref: '#/components/schemas/SearchHit' + } + } + } + }, ErrorResponse: { type: 'object', required: ['error', 'message'], diff --git a/utils/constants.js b/utils/constants.js index 483a1b2..8a1b1c7 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -59,15 +59,15 @@ const cpeMatchFields = { // Dictionary of fields to paths /** * keywordSearch: list of terms - * keywordExactMatch: list of terms, must be an exact match to terms in a description + * keywordExactMatch: boolean that requires keywordSearch terms to appear as an exact phrase * cveIds: comma-separated list of CVE-year-sequence identifiers - * noRejected: no-value flag that excludes rejected CVE records + * noRejected: boolean that excludes rejected CVE records when true * cweId: CWE-XXX - * cvssV2Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL - * cvssV3Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL - * cvssV3_0Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL - * cvssV3_1Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL - * cvssV4Severity: NONE, LOW, MEDIUM, HIGH, CRITICAL + * cvssV2Severity: LOW, MEDIUM, HIGH + * cvssV3Severity: LOW, MEDIUM, HIGH, CRITICAL + * cvssV3_0Severity: LOW, MEDIUM, HIGH, CRITICAL + * cvssV3_1Severity: LOW, MEDIUM, HIGH, CRITICAL + * cvssV4Severity: LOW, MEDIUM, HIGH, CRITICAL * cvssV2Metrics: CVSSv2 vector string * cvssV3Metrics: CVSSv3_0 or CVSSv3_1 vector string * cvssV3_0Metrics: CVSSv3_0 vector string From 443ae5846fb32a7a98af50917a144a183bb1afc8 Mon Sep 17 00:00:00 2001 From: Andrew Foote Date: Mon, 3 Aug 2026 16:33:19 -0400 Subject: [PATCH 10/10] Harden search validation and runtime safeguards --- .env.example | 1 + README.md | 20 +- api-docs/openapi.json | 24 ++- api-docs/openapi.test.js | 13 +- config/custom-environment-variables.jsonc | 3 +- config/default.jsonc | 4 +- config/runtime-config.test.js | 1 + controllers/middleware.js | 5 + controllers/middleware.test.js | 30 ++- controllers/searchController.js | 110 +++++++---- controllers/searchController.test.js | 219 +++++++++++++++++----- routes/index.js | 18 ++ routes/search.integration.test.js | 61 ++++++ routes/swagger.js | 4 +- test/contract/opensearch.js | 90 ++++++--- test/contract/opensearch.test.js | 188 +++++++++++++++---- utils/constants.js | 4 + utils/cpe.js | 3 + utils/cpe.test.js | 13 ++ utils/cweId.js | 20 ++ utils/cweId.test.js | 33 ++++ utils/errors.js | 14 ++ utils/errors.test.js | 16 ++ utils/runtimeConfig.js | 9 +- utils/runtimeConfig.test.js | 8 +- utils/searchCandidateProcessor.test.js | 4 + utils/searchPerformance.js | 88 ++++++++- utils/searchPerformance.test.js | 62 +++++- utils/searchProvider.js | 5 + utils/searchProvider.test.js | 3 + utils/searchQueryBuilder.js | 8 + utils/searchQueryBuilder.test.js | 3 +- utils/searchResponseLimiter.js | 61 ++++++ utils/searchResponseLimiter.test.js | 92 +++++++++ 34 files changed, 1054 insertions(+), 183 deletions(-) create mode 100644 utils/cweId.js create mode 100644 utils/cweId.test.js create mode 100644 utils/searchResponseLimiter.js create mode 100644 utils/searchResponseLimiter.test.js diff --git a/.env.example b/.env.example index f2680a2..19fb5bd 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,7 @@ SearchPostFilterEvaluationChunkSize=250 SearchOpenSearchRequestTimeoutMs=20000 SearchRequestDeadlineMs=30000 SearchReadinessTimeoutMs=5000 +SearchMaxResponseBytes=10485760 RequestLogSuccessSampleRate=0.1 RequestLogHealthChecks=false ShutdownTimeoutMs=35000 diff --git a/README.md b/README.md index 4514cda..1735ec9 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Copy `.env.example` to `.env` and replace or check the following values. - `SearchOpenSearchRequestTimeoutMs` - optional timeout for each OpenSearch request, default `20000` - `SearchRequestDeadlineMs` - optional overall `/search` processing deadline, default `30000` - `SearchReadinessTimeoutMs` - optional timeout for the OpenSearch index readiness check, default `5000` +- `SearchMaxResponseBytes` - optional maximum serialized successful `/search` response size in bytes, default `10485760` (10 MiB) - `RequestLogSuccessSampleRate` - optional fraction of successful non-health-check access logs to retain, default `0.1` - `RequestLogHealthChecks` - optional flag to retain successful health-check access logs, default `false` - `ShutdownTimeoutMs` - optional graceful HTTP shutdown deadline, default `35000` @@ -55,6 +56,7 @@ The following numeric bounds are enforced: | `SearchOpenSearchRequestTimeoutMs` | 1 through 300,000 | | `SearchRequestDeadlineMs` | 1 through 600,000 | | `SearchReadinessTimeoutMs` | 1 through 60,000 | +| `SearchMaxResponseBytes` | 65,536 through 104,857,600 | | `RequestLogSuccessSampleRate` | 0 through 1 | | `ShutdownTimeoutMs` | 1 through 900,000 | | `HttpHeadersTimeoutMs` | 1 through 60,000 | @@ -159,7 +161,7 @@ String inputs are bounded before query construction: | ---------------------------- | ---------------------------------------------------------------------: | | `keywordSearch` | 512 characters and 64 whitespace-delimited terms | | `cveIds` | 4,096 characters overall, 100 IDs, and 4-19 sequence digits per CVE ID | -| `cweId` | 32 characters | +| `cweId` | 32 characters and `CWE-` format | | `cpeName` | 2,048 characters | | `virtualMatchString` | 2,048 characters | | `versionStart`, `versionEnd` | 256 characters each | @@ -169,6 +171,8 @@ Requests that exceed a limit return HTTP 400 before contacting OpenSearch. These CVSS vector filters accept full or partial vectors containing at least one unique `KEY:VALUE` metric token. An optional vector header must identify a version supported by the selected parameter. Requested metric tokens can appear in any order, but all tokens must occur in the same stored vector string. CVSS and CWE filters search CNA-provided data only; ADP-provided enrichment is not currently included. +The `cweId` filter accepts a positive decimal CWE identifier such as `CWE-79`. The prefix is case-insensitive, surrounding whitespace is trimmed, and valid input is normalized to uppercase before query construction. Zero, leading-zero identifiers such as `CWE-079`, missing or non-numeric identifiers, and suffixes return HTTP 400 without contacting OpenSearch. + Publication and last-modified date filters support one-sided ranges. Values must use the exact UTC format `yyyy-MM-ddTHH:mm:ss.SSSZ`, including three fractional-second digits and the trailing `Z`. When both values are supplied, `pubStartDate` must be less than or equal to `pubEndDate`, and `lastModStartDate` must be less than or equal to `lastModEndDate`. ### Search Ordering and Pagination @@ -179,6 +183,8 @@ Publication and last-modified date filters support one-sided ranges. Values must Pagination is limited to complete pages inside the first 10,000 results: `from + resultsPerPage` must not exceed 10,000. `totalPages` reports the smaller of the pages implied by the match count and the complete pages accessible inside that window. For example, a page size of 30 allows at most 333 pages. Standard-search totals are exact and are not reduced by this pagination calculation. Semantic CPE searches have a separate 10,000-candidate evaluation limit described below, so their totals can be incomplete. Point-in-time search and deep pagination remain deferred. +Successful `/search` responses are also limited by serialized UTF-8 size. The default limit is 10 MiB and can be changed with `SearchMaxResponseBytes`. The API serializes the complete response and rechecks the overall search deadline before sending any bytes. If the successful payload exceeds the configured limit, the API returns HTTP 422 with `SEARCH_RESPONSE_TOO_LARGE`; reduce `resultsPerPage` and retry. Error responses bypass this success-payload limit so clients receive a complete error body. Match totals and the 500-record page-size limit are unchanged. + Use `cveIds` to request up to 100 CVE records by comma-separated CVE ID. Each ID must contain the `CVE` prefix, a four-digit year, and a sequence number containing 4 to 19 digits. The prefix is accepted without regard to case and normalized to uppercase for the OpenSearch query. ```bash @@ -321,7 +327,7 @@ An opt-in, read-only OpenSearch contract check is available for an approved envi npm run test:opensearch-contract ``` -Set `OpenSearchDomainEndpoint` and `OpenSearchCveIndex` in the environment or local `.env` before running it. `OpenSearchAllowUnknownSslCerts` remains optional and should be enabled only for local development against a self-signed certificate. The contract check sends a minimal readiness query, validates representative field capabilities, requires a concrete CVE record, and uses that record for a positive CVE-ID search before running the remaining bounded query shapes. It closes its OpenSearch client and emits sanitized JSON output. It never creates, updates, maps, or deletes indexes. +Set `OpenSearchDomainEndpoint` and `OpenSearchCveIndex` in the environment or local `.env` before running it. `OpenSearchAllowUnknownSslCerts` remains optional and should be enabled only for local development against a self-signed certificate. The contract check sends a minimal readiness query, validates representative field capabilities, and requires a concrete CVE record with a structurally valid direct CNA CPE applicability entry. It uses that record for a positive CVE-ID search and one of its validated CPE criteria values for a positive candidate search before running the remaining bounded query shapes. It closes its OpenSearch client and emits sanitized JSON output. It never creates, updates, maps, or deletes indexes. The contract check is not part of public pull-request CI. The `OpenSearch Contract` GitHub Actions workflow runs it only through a manual dispatch against a selected protected GitHub environment. Configure that environment with: @@ -331,13 +337,13 @@ The contract check is not part of public pull-request CI. The `OpenSearch Contra The workflow can run only from the trusted `dev` or `main` branch. The selected GitHub environment should additionally require reviewers and restrict deployment branches. OpenSearch secrets are exposed only to the final contract execution step, after checkout, Node setup, and dependency installation have completed. -The selected runner must have network access to the endpoint. Change the workflow to an approved self-hosted runner before use when the index is not reachable from GitHub-hosted runners. The workflow performs readiness, field-capability, positive source-projection, exact-total, CVE ID keyword-subfield, keyword, publication-date, CVSS, and CPE candidate checks. Every search is read-only and returns at most one record. Connection values and credentials remain in environment secrets rather than source files or workflow text. +The selected runner must have network access to the endpoint. Change the workflow to an approved self-hosted runner before use when the index is not reachable from GitHub-hosted runners. The workflow performs readiness, field-capability, positive CPE source-projection, exact-total, CVE ID keyword-subfield, keyword, publication-date, CVSS, and positive CPE candidate checks. Candidate responses are validated with the same CVE ID, applicability, node, CPE match, vulnerability, criteria, and version-range requirements used by application searches. Every search is read-only and returns at most one record. Connection values and credentials remain in environment secrets rather than source files or workflow text. The validation tiers have separate purposes: - Jest and `npm run quality` verify application behavior, API contracts, and regression coverage with controlled test doubles. - Pull-request container validation verifies production-image packaging, non-root execution, startup, liveness, and packaged API documentation. -- The optional OpenSearch contract check verifies that the installed client can communicate with a non-empty externally managed index whose representative searchable fields satisfy the application contract. +- The optional OpenSearch contract check verifies that the installed client can communicate with a non-empty externally managed index whose representative searchable fields and direct CNA CPE applicability records satisfy the application contract. - Load and performance testing remain in the separate automation performance-testing repository. Dependabot checks npm packages, Docker base-image digests, and GitHub Actions weekly. Version-update pull requests target the `dev` integration branch; GitHub security updates continue to follow the repository default-branch behavior. Both Dockerfiles pin the Node build image and Distroless runtime image to immutable multi-architecture digests; Dependabot updates those references through reviewable pull requests. @@ -350,7 +356,9 @@ Access logs omit query strings and request bodies. By default, the API retains 1 The API always emits a sanitized JSON `search.performance` event for `/search` queue saturation, shared-operation waiter saturation, queue wait timeout, search timeout, client disconnect, OpenSearch connection errors, and upstream search errors. These events include the request ID, outcome, HTTP status, strategy, cache state, total elapsed time, cancellation reason, and bounded queue or waiter state when applicable. Search terms, CPE values, request bodies, credentials, and upstream error details are never included. -Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, shared waiter counts, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, response bytes, JSON serialization time, process heap usage and per-request heap delta, cache entry and cached CVE ID counts, recent process-wide event-loop delay, and total handler elapsed time. Leave it disabled during normal operation unless the deployment logging volume is acceptable. +Set `SearchLogPerformanceTimings=true` on the API deployment when diagnosing slow `/search` requests. Successful requests then emit the same structured event with additional redacted timing details, including candidate counts, filtered counts, returned count, OpenSearch operation types, maximum observed active OpenSearch requests in the process, shared waiter counts, OpenSearch elapsed time, OpenSearch `took`, post-filter elapsed time, cold-search queue wait time, page fetch elapsed or wait time, response bytes, JSON serialization time, process heap usage and per-request heap delta, cache entry and cached CVE ID counts, recent process-wide event-loop delay, and total handler elapsed time. Oversized response failures are logged even when detailed timing is disabled and include the attempted byte count, configured maximum, and attempted serialization time. Leave detailed logging disabled during normal operation unless the deployment logging volume is acceptable. + +Candidate-discovery and page-fetch timings describe the underlying operation, which can be shared by multiple active requests. Every request that successfully waits for that operation reports the same operation timing while retaining its own cache status, in-flight status, waiter count, wait duration, deadline, response, and outcome. If the initiating client disconnects but another waiter remains, the surviving request still receives the completed operation metrics. Cache entries contain only candidate results, so a later cache hit does not replay historical candidate, post-filter, or queue timings; it reports only work performed for the current request, such as its page fetch. Event-loop delay uses one low-overhead process monitor sampled in rolling 10-second windows. Request events report the most recently completed window through `processEventLoopDelayMeanMs`, `processEventLoopDelayP99Ms`, `processEventLoopDelayMaxMs`, `processEventLoopDelayWindowMs`, and `processEventLoopDelaySampleAgeMs`. These values describe process-wide event-loop saturation near the request, not delay caused by that individual request. They are omitted until the first complete window is available, and concurrent requests read the same completed snapshot without resetting one another's measurements. @@ -415,7 +423,7 @@ This helps most when multiple users submit the same expensive CPE search within The cache and cold-search queue are a temporary per-process proof of concept. They do not share state across ECS tasks, so each task warms and admits work independently depending on load balancer routing. Track cache-hit and in-flight-share rates, cold-search queue waits and rejections, cold-versus-warm latency, and OpenSearch time to decide their long-term value. If the benefit is low, remove this layer; if it is material, replace it with an AWS-managed shared cache and admission-control design. -The request lifecycle and in-flight waiter registry are separate from the result cache, so a later shared-cache replacement does not need to change caller cancellation behavior. The per-process TTL, entry limit, cold-search concurrency, queue depth, queue wait, shared waiter limit, evaluation chunk size, OpenSearch timeout, and overall request deadline can be tuned with `SearchPostFilterCacheTtlMs`, `SearchPostFilterCacheMaxEntries`, `SearchPostFilterColdSearchConcurrency`, `SearchPostFilterColdSearchMaxQueueDepth`, `SearchPostFilterColdSearchMaxWaitMs`, `SearchInFlightMaxWaitersPerOperation`, `SearchPostFilterEvaluationChunkSize`, `SearchOpenSearchRequestTimeoutMs`, and `SearchRequestDeadlineMs`. +The request lifecycle and in-flight waiter registry are separate from the result cache, so a later shared-cache replacement does not need to change caller cancellation behavior. The per-process TTL, entry limit, cold-search concurrency, queue depth, queue wait, shared waiter limit, evaluation chunk size, OpenSearch timeout, overall request deadline, and successful response size can be tuned with `SearchPostFilterCacheTtlMs`, `SearchPostFilterCacheMaxEntries`, `SearchPostFilterColdSearchConcurrency`, `SearchPostFilterColdSearchMaxQueueDepth`, `SearchPostFilterColdSearchMaxWaitMs`, `SearchInFlightMaxWaitersPerOperation`, `SearchPostFilterEvaluationChunkSize`, `SearchOpenSearchRequestTimeoutMs`, `SearchRequestDeadlineMs`, and `SearchMaxResponseBytes`. ### HTTP Transport Timeouts diff --git a/api-docs/openapi.json b/api-docs/openapi.json index c2ec8e4..78cf60f 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -121,6 +121,24 @@ } } }, + "422": { + "description": "The serialized successful response exceeded the configured maximum size; reduce resultsPerPage and retry", + "headers": { + "X-Request-ID": { + "description": "Request correlation ID", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, "429": { "description": "Application rate limit exceeded, the cold CPE search queue is full or exceeded its maximum wait, or a shared semantic search operation reached its waiter limit", "headers": { @@ -635,9 +653,11 @@ }, "cweId": { "type": "string", + "minLength": 5, "maxLength": 32, + "pattern": "^\\s*[Cc][Ww][Ee]-[1-9][0-9]*\\s*$", "example": "CWE-79", - "description": "Returns CVE records whose CNA data includes the provided Common Weakness Enumeration ID. ADP-provided CWE data is not searched." + "description": "Returns CVE records whose CNA data includes the provided Common Weakness Enumeration ID. The accepted format is the CWE- prefix followed by a positive integer without leading zeros. Prefix matching is case-insensitive, surrounding whitespace is trimmed, and valid input is normalized to uppercase before search. Invalid values return HTTP 400. ADP-provided CWE data is not searched." }, "cvssV2Severity": { "type": "string", @@ -1029,4 +1049,4 @@ } } } -} \ No newline at end of file +} diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 166024b..192a6da 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -60,6 +60,10 @@ describe('OpenAPI contract', () => { expect(operation.requestBody.description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`); expect(mediaType.schema).toStrictEqual({ $ref: '#/components/schemas/SearchRequest' }); expect(operation.responses['413'].description).toContain(`${constants.JSON_REQUEST_BODY_LIMIT_KB} KB`); + expect(operation.responses['422'].description).toContain('reduce resultsPerPage'); + expect(operation.responses['422'].content['application/json'].schema).toStrictEqual({ + $ref: '#/components/schemas/ErrorResponse' + }); expect(schema.additionalProperties).toBe(false); expect(Object.keys(schema.properties)).toStrictEqual(constants.SEARCH_QUERY_PARAMS); Object.values(schema.properties).forEach((property) => { @@ -95,7 +99,13 @@ describe('OpenAPI contract', () => { 'prerelease qualifiers before an unqualified release' ); expect(schema.properties.cweId.description).toContain('CNA data'); + expect(schema.properties.cweId.description).toContain('CWE- prefix followed by a positive integer'); + expect(schema.properties.cweId.description).toContain('without leading zeros'); + expect(schema.properties.cweId.description).toContain('normalized to uppercase'); + expect(schema.properties.cweId.description).toContain('HTTP 400'); expect(schema.properties.cweId.description).toContain('ADP-provided CWE data is not searched'); + expect(schema.properties.cweId.minLength).toBe(5); + expect(schema.properties.cweId.pattern).toBe(constants.CWE_ID_PATTERN); expect(schema.properties.cvssV3Metrics.description).toContain('same stored vector string'); expect(schema.properties.cvssV3Metrics.description).toContain('CNA data'); expect(schema.properties.cvssV3Metrics.description).toContain('ADP-provided CVSS data is not searched'); @@ -239,6 +249,7 @@ describe('OpenAPI contract', () => { expect(responses['400'].description).toContain('date range'); expect(responses['404'].description).toBe('Invalid cpeName value'); expect(responses['415'].description).toContain('application/json'); + expect(responses['422'].description).toContain('reduce resultsPerPage'); expect(responses['429'].description).toContain('rate limit'); expect(responses['429'].description).toContain('waiter limit'); expect(responses['502'].description).toContain('invalid response'); @@ -272,7 +283,7 @@ describe('OpenAPI contract', () => { } ]); - ['404', '413', '415', '429', '500', '502', '503', '504'].forEach((status) => { + ['404', '413', '415', '422', '429', '500', '502', '503', '504'].forEach((status) => { expect(responses[status].content['application/json'].schema).toStrictEqual({ $ref: '#/components/schemas/ErrorResponse' }); diff --git a/config/custom-environment-variables.jsonc b/config/custom-environment-variables.jsonc index e9a2744..a72b60e 100644 --- a/config/custom-environment-variables.jsonc +++ b/config/custom-environment-variables.jsonc @@ -21,7 +21,8 @@ "postFilterEvaluationChunkSize": "SearchPostFilterEvaluationChunkSize", "openSearchRequestTimeoutMs": "SearchOpenSearchRequestTimeoutMs", "requestDeadlineMs": "SearchRequestDeadlineMs", - "readinessTimeoutMs": "SearchReadinessTimeoutMs" + "readinessTimeoutMs": "SearchReadinessTimeoutMs", + "maxResponseBytes": "SearchMaxResponseBytes" }, "requestLogging": { "successSampleRate": "RequestLogSuccessSampleRate", diff --git a/config/default.jsonc b/config/default.jsonc index 7f05ddc..86df8a5 100644 --- a/config/default.jsonc +++ b/config/default.jsonc @@ -36,7 +36,9 @@ // Limits total processing time for a /search request. "requestDeadlineMs": "30000", // Limits the minimal OpenSearch request used by /api/ready-check. - "readinessTimeoutMs": "5000" + "readinessTimeoutMs": "5000", + // Limits a successfully serialized /search response before any bytes are sent. + "maxResponseBytes": "10485760" }, // Access logging settings. Error responses are always logged. "requestLogging": { diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index 77486cf..6e91493 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -119,6 +119,7 @@ describe('runtime container and compose configuration', () => { ); expect(customEnvironmentVariables).toContain('"requestDeadlineMs": "SearchRequestDeadlineMs"'); expect(customEnvironmentVariables).toContain('"readinessTimeoutMs": "SearchReadinessTimeoutMs"'); + expect(customEnvironmentVariables).toContain('"maxResponseBytes": "SearchMaxResponseBytes"'); expect(customEnvironmentVariables).toContain('"successSampleRate": "RequestLogSuccessSampleRate"'); expect(customEnvironmentVariables).toContain('"healthChecks": "RequestLogHealthChecks"'); expect(customEnvironmentVariables).toContain('"shutdownTimeoutMs": "ShutdownTimeoutMs"'); diff --git a/controllers/middleware.js b/controllers/middleware.js index af2c0ce..f9831f0 100644 --- a/controllers/middleware.js +++ b/controllers/middleware.js @@ -2,6 +2,7 @@ const errors = require('../utils/errors'); const constants = require('../utils/constants'); const cpe = require('../utils/cpe'); const cveId = require('../utils/cveId'); +const cweId = require('../utils/cweId'); const cvss = require('../utils/cvss'); const error = new errors.errors(); @@ -152,6 +153,10 @@ function validateParams(req, res, next) { } } + if (hasParam(params, 'cweId') && !cweId.isValidCweId(params.cweId)) { + return res.status(400).json(error.cweIdFormatError()); + } + if (hasParam(params, 'noRejected') && ![true, false].includes(params.noRejected)) { return res.status(400).json(error.noRejectedValueError()); } diff --git a/controllers/middleware.test.js b/controllers/middleware.test.js index 6bfe8d4..0024e2e 100644 --- a/controllers/middleware.test.js +++ b/controllers/middleware.test.js @@ -97,7 +97,9 @@ describe('middleware validation function', () => { ['wildcard part', 'cpe:2.3:*:vendor:product:1.0:*:*:*:*:*:*:*'], ['wildcard vendor', 'cpe:2.3:a:*:product:1.0:*:*:*:*:*:*:*'], ['wildcard product', 'cpe:2.3:a:vendor:*:1.0:*:*:*:*:*:*:*'], - ['wildcard version', 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*'] + ['wildcard version', 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*'], + ['empty optional component', 'cpe:2.3:a:vendor:product:1.0::*'], + ['empty trailing component', 'cpe:2.3:a:vendor:product:1.0:'] ])('ValidateParams returns an error when cpeName has %s.', (_caseName, cpeName) => { validate({ cpeName @@ -401,6 +403,7 @@ describe('middleware validation function', () => { test.each([ ['keywordSearch', ''], + ['cweId', ''], ['cpeName', ' '], ['virtualMatchString', ''], ['pubStartDate', ''], @@ -485,6 +488,29 @@ describe('middleware validation function', () => { expectValidationError('CVE_IDS_LIMIT', 'The cveIds parameter cannot contain more than 100 CVE IDs.'); }); + test.each([ + ['a missing prefix', '79'], + ['a zero identifier', 'CWE-0'], + ['a leading zero', 'CWE-079'], + ['a missing numeric identifier', 'CWE-'], + ['a non-numeric identifier', 'CWE-noinfo'], + ['embedded whitespace', 'CWE- 79'], + ['a suffix', 'CWE-79-extra'] + ])('ValidateParams returns an error when cweId has %s.', (_caseName, cweId) => { + validate({ cweId }); + + expectValidationError( + 'CWE_ID_FORMAT', + 'The cweId parameter must use the format CWE- without leading zeros.' + ); + }); + + test.each(['CWE-79', 'cwe-79', ' CwE-79 '])('ValidateParams accepts valid cweId value %p.', (cweId) => { + validate({ cweId }); + + expectValidationSuccess(); + }); + test.each(Object.entries(constants.SEARCH_STRING_LENGTH_LIMITS))( 'ValidateParams returns an error when %s exceeds its maximum length.', (param, maxLength) => { @@ -501,7 +527,7 @@ describe('middleware validation function', () => { test.each([ ['keywordSearch', 'a'.repeat(constants.SEARCH_STRING_LENGTH_LIMITS.keywordSearch)], - ['cweId', 'a'.repeat(constants.SEARCH_STRING_LENGTH_LIMITS.cweId)] + ['cweId', `CWE-${'1'.repeat(constants.SEARCH_STRING_LENGTH_LIMITS.cweId - 4)}`] ])('ValidateParams accepts %s at its maximum length.', (param, value) => { validate({ [param]: value diff --git a/controllers/searchController.js b/controllers/searchController.js index 16dc69d..5f45f49 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -8,6 +8,7 @@ const searchInFlightRegistry = require('../utils/searchInFlightRegistry'); const searchPostFilterWorkQueue = require('../utils/searchPostFilterWorkQueue'); const searchProvider = require('../utils/searchProvider'); const searchQueryBuilder = require('../utils/searchQueryBuilder'); +const searchResponseLimiter = require('../utils/searchResponseLimiter'); const { OPEN_SEARCH_RESPONSE_FAILURE_TYPES, OpenSearchResponseError } = require('../utils/searchResponseValidator'); const constants = require('../utils/constants'); const error = new errors.errors(); @@ -197,15 +198,21 @@ async function fetchFullHitsByCveIds(cveIds, metrics, signal) { const waitStart = searchPerformance.markTime(); const inFlight = searchInFlightRegistry.acquire( inFlightKey, - async (operationSignal) => { - const body = searchQueryBuilder.buildCveIdsSearchBody(cveIds); - const pageFetchStart = searchPerformance.markTime(); - const pageData = await executeOpenSearch(body, 'page-fetch', operationSignal, metrics); - metrics.pageFetchElapsedMs = searchPerformance.elapsedMs(pageFetchStart); - metrics.pageFetchTookMs = pageData?.took; - - return validateAndOrderPageHits(pageData.hits.hits, cveIds); - }, + (operationSignal) => + searchPerformance.captureOperationOutcome(async (operationMetrics) => { + const body = searchQueryBuilder.buildCveIdsSearchBody(cveIds); + const pageFetchStart = searchPerformance.markTime(); + let pageData; + + try { + pageData = await executeOpenSearch(body, 'page-fetch', operationSignal, operationMetrics); + } finally { + operationMetrics.pageFetchElapsedMs = searchPerformance.elapsedMs(pageFetchStart); + } + operationMetrics.pageFetchTookMs = pageData?.took; + + return validateAndOrderPageHits(pageData.hits.hits, cveIds); + }), { signal, maxWaiters: searchProvider.getInFlightMaxWaitersPerOperation() @@ -214,31 +221,42 @@ async function fetchFullHitsByCveIds(cveIds, metrics, signal) { metrics.pageFetchInFlightHit = inFlight.shared; metrics.pageFetchInFlightWaiterCount = inFlight.waiterCount; - const result = await inFlight.promise; + const outcome = await inFlight.promise; if (inFlight.shared) { metrics.pageFetchWaitElapsedMs = searchPerformance.elapsedMs(waitStart); } - return result; + return searchPerformance.consumeOperationOutcome(outcome, metrics); } -async function buildPostFilterCandidateResult(candidateBody, postFilters, metrics, signal) { +async function buildPostFilterCandidateResult(candidateBody, postFilters, operationMetrics, signal) { const candidateStart = searchPerformance.markTime(); - const candidateData = await executeOpenSearch(candidateBody, 'cpe-candidate', signal, metrics); - metrics.openSearchElapsedMs = searchPerformance.elapsedMs(candidateStart); - metrics.openSearchTookMs = candidateData?.took; + let candidateData; + + try { + candidateData = await executeOpenSearch(candidateBody, 'cpe-candidate', signal, operationMetrics); + } finally { + operationMetrics.openSearchElapsedMs = searchPerformance.elapsedMs(candidateStart); + } + operationMetrics.openSearchTookMs = candidateData?.took; const candidateResp = { data: candidateData }; const postFilterStart = searchPerformance.markTime(); - const { candidateCount, filteredHits } = await searchCandidateProcessor.filterSearchHitsInChunks( - candidateResp, - postFilters, - signal, - searchProvider.getPostFilterEvaluationChunkSize() - ); - metrics.postFilterElapsedMs = searchPerformance.elapsedMs(postFilterStart); + let candidateResult; + + try { + candidateResult = await searchCandidateProcessor.filterSearchHitsInChunks( + candidateResp, + postFilters, + signal, + searchProvider.getPostFilterEvaluationChunkSize() + ); + } finally { + operationMetrics.postFilterElapsedMs = searchPerformance.elapsedMs(postFilterStart); + } + const { candidateCount, filteredHits } = candidateResult; return { candidateCount, @@ -257,24 +275,30 @@ async function getPostFilterCandidateResult(cacheKey, candidateBody, postFilters metrics.cacheHit = false; const inFlightKey = searchInFlightRegistry.buildKey('candidate', cacheKey); - const queueStart = searchPerformance.markTime(); const waitStart = searchPerformance.markTime(); const inFlight = searchInFlightRegistry.acquire( inFlightKey, - (operationSignal) => - searchPostFilterWorkQueue - .run( + (operationSignal) => { + const queueStart = searchPerformance.markTime(); + + return searchPerformance.captureOperationOutcome(async (operationMetrics) => { + const result = await searchPostFilterWorkQueue.run( () => { - metrics.coldSearchQueueWaitElapsedMs = searchPerformance.elapsedMs(queueStart); - metrics.coldSearchQueueState = searchPostFilterWorkQueue.getState(); - return buildPostFilterCandidateResult(candidateBody, postFilters, metrics, operationSignal); + operationMetrics.coldSearchQueueWaitElapsedMs = searchPerformance.elapsedMs(queueStart); + operationMetrics.coldSearchQueueState = searchPostFilterWorkQueue.getState(); + return buildPostFilterCandidateResult( + candidateBody, + postFilters, + operationMetrics, + operationSignal + ); }, { signal: operationSignal } - ) - .then((result) => { - searchPostFilterCache.set(cacheKey, result); - return result; - }), + ); + searchPostFilterCache.set(cacheKey, result); + return result; + }); + }, { signal, maxWaiters: searchProvider.getInFlightMaxWaitersPerOperation() @@ -283,12 +307,12 @@ async function getPostFilterCandidateResult(cacheKey, candidateBody, postFilters metrics.inFlightHit = inFlight.shared; metrics.inFlightWaiterCount = inFlight.waiterCount; - const result = await inFlight.promise; + const outcome = await inFlight.promise; if (inFlight.shared) { metrics.inFlightWaitElapsedMs = searchPerformance.elapsedMs(waitStart); } - return result; + return searchPerformance.consumeOperationOutcome(outcome, metrics); } async function executeOptimizedPostFilterSearch( @@ -366,6 +390,11 @@ async function searchHandler(req, res, next) { res, searchProvider.getSearchRequestDeadlineMs() ); + searchResponseLimiter.instrumentSearchResponse(res, { + maxResponseBytes: searchProvider.getSearchMaxResponseBytes(), + metrics, + checkDeadline: lifecycle.getRemainingMs + }); let searchOptions; let postFilters; let responseOptions; @@ -396,6 +425,15 @@ async function searchHandler(req, res, next) { ...resp }); } catch (err) { + if (searchResponseLimiter.isSearchResponseTooLargeError(err)) { + metrics.outcome = 'response_too_large'; + metrics.statusCode = 422; + metrics.attemptedResponseBytes = err.responseBytes; + metrics.maxResponseBytes = err.maxResponseBytes; + + return res.status(422).json(error.searchResponseTooLargeError()); + } + if (searchDeadline.isSearchClientDisconnectedError(err)) { metrics.outcome = 'client_disconnected'; metrics.statusCode = 499; diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 8b68cd0..0db51b4 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -24,6 +24,7 @@ const mockGetPostFilterColdSearchMaxWaitMs = jest.fn(() => 10000); const mockGetInFlightMaxWaitersPerOperation = jest.fn(() => 500); const mockGetPostFilterEvaluationChunkSize = jest.fn(() => 250); const mockGetSearchRequestDeadlineMs = jest.fn(() => 30000); +const mockGetSearchMaxResponseBytes = jest.fn(() => 10 * 1024 * 1024); const mockIsOpenSearchResponseError = jest.fn((err) => err?.code === 'OPEN_SEARCH_RESPONSE_INVALID'); const mockIsSearchProviderConfigurationError = jest.fn((err) => err?.code === 'SEARCH_PROVIDER_CONFIGURATION'); const mockIsSearchProviderContractError = jest.fn((err) => err?.code === 'SEARCH_PROVIDER_CONTRACT'); @@ -48,6 +49,7 @@ jest.mock('../utils/searchProvider', () => ({ getPostFilterColdSearchMaxWaitMs: mockGetPostFilterColdSearchMaxWaitMs, getInFlightMaxWaitersPerOperation: mockGetInFlightMaxWaitersPerOperation, getPostFilterEvaluationChunkSize: mockGetPostFilterEvaluationChunkSize, + getSearchMaxResponseBytes: mockGetSearchMaxResponseBytes, getSearchRequestDeadlineMs: mockGetSearchRequestDeadlineMs, isOpenSearchResponseError: mockIsOpenSearchResponseError, isSearchProviderConfigurationError: mockIsSearchProviderConfigurationError, @@ -64,11 +66,30 @@ const searchPostFilterCache = require('../utils/searchPostFilterCache'); function createResponse() { const res = {}; res.status = jest.fn(() => res); - res.json = jest.fn(() => res); + res.jsonMock = jest.fn(() => res); + res.json = res.jsonMock; res.set = jest.fn(() => res); return res; } +function createSerializableResponse() { + const res = { + statusCode: 200, + originalSend: jest.fn() + }; + res.status = jest.fn((statusCode) => { + res.statusCode = statusCode; + return res; + }); + res.originalSend.mockImplementation(() => res); + res.send = res.originalSend; + res.jsonMock = jest.fn((body) => res.send(JSON.stringify(body))); + res.json = res.jsonMock; + res.set = jest.fn(() => res); + res.getHeader = jest.fn(() => undefined); + return res; +} + function advancedResponse(totalValue, hits = []) { return { status: 'ok', @@ -187,6 +208,7 @@ describe('Search controller SEARCH handler', () => { mockGetPostFilterColdSearchMaxWaitMs.mockClear(); mockGetInFlightMaxWaitersPerOperation.mockClear(); mockGetPostFilterEvaluationChunkSize.mockClear(); + mockGetSearchMaxResponseBytes.mockClear(); mockGetSearchRequestDeadlineMs.mockClear(); mockIsOpenSearchResponseError.mockClear(); mockIsSearchProviderConfigurationError.mockClear(); @@ -198,6 +220,7 @@ describe('Search controller SEARCH handler', () => { mockGetPostFilterColdSearchMaxWaitMs.mockReturnValue(10000); mockGetInFlightMaxWaitersPerOperation.mockReturnValue(500); mockGetPostFilterEvaluationChunkSize.mockReturnValue(250); + mockGetSearchMaxResponseBytes.mockReturnValue(10 * 1024 * 1024); mockGetSearchRequestDeadlineMs.mockReturnValue(30000); searchPostFilterCache.clear(); searchInFlightRegistry.clear(); @@ -241,7 +264,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 3, pageSize: 1, totalPages: 3, @@ -249,6 +272,41 @@ describe('Search controller SEARCH handler', () => { }); }); + test('SEARCH converts an oversized successful payload to a sanitized 422 before sending it', async () => { + res = createSerializableResponse(); + mockGetSearchMaxResponseBytes.mockReturnValue(256); + mockApiSearch.mockResolvedValue( + advancedResponse(1, [ + { + _id: 'CVE-2024-0001', + _source: { + description: 'x'.repeat(512) + } + } + ]) + ); + + await controller.SEARCH( + { + searchParams: { + keywordSearch: 'remote code', + resultsPerPage: 1, + pageNumber: 1 + } + }, + res, + next + ); + + expect(res.status.mock.calls).toStrictEqual([[200], [422]]); + expect(res.originalSend).toHaveBeenCalledTimes(1); + expect(JSON.parse(res.originalSend.mock.calls[0][0])).toStrictEqual({ + error: 'SEARCH_RESPONSE_TOO_LARGE', + message: 'Search response exceeds the maximum response size. Reduce resultsPerPage and try again.' + }); + expect(next).not.toHaveBeenCalled(); + }); + test('SEARCH applies semantic CPE version filtering before building response metadata', async () => { const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', @@ -315,7 +373,7 @@ describe('Search controller SEARCH handler', () => { 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] }); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 1, pageSize: 1, totalPages: 1, @@ -381,7 +439,7 @@ describe('Search controller SEARCH handler', () => { }) ); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 1, pageSize: 1, totalPages: 1, @@ -398,6 +456,7 @@ describe('Search controller SEARCH handler', () => { }); test('SEARCH reuses cached semantic post-filter IDs for repeated virtualMatchString range searches', async () => { + mockShouldLogSearchPerformance.mockReturnValue(true); const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' }); @@ -427,10 +486,10 @@ describe('Search controller SEARCH handler', () => { mockSearchRaw.mockResolvedValue(rawResponse(2, [applicable, notApplicable])); mockApiSearch.mockResolvedValue(advancedResponse(1, [fullApplicable])); - await controller.SEARCH(req, res, next); + await controller.SEARCH({ requestId: 'cache-miss-request', ...req }, res, next); const secondRes = createResponse(); - await controller.SEARCH(req, secondRes, next); + await controller.SEARCH({ requestId: 'cache-hit-request', ...req }, secondRes, next); expect(mockSearchRaw).toHaveBeenCalledTimes(1); expect(mockApiSearch).toHaveBeenCalledTimes(2); @@ -448,7 +507,7 @@ describe('Search controller SEARCH handler', () => { }); expect(mockApiSearch.mock.calls[1][0].sort).toBeUndefined(); expect(secondRes.status).toHaveBeenCalledWith(200); - expect(secondRes.json).toHaveBeenCalledWith({ + expect(secondRes.jsonMock).toHaveBeenCalledWith({ page: 1, pageSize: 1, totalPages: 1, @@ -463,6 +522,24 @@ describe('Search controller SEARCH handler', () => { } } }); + + const performanceLogs = console.log.mock.calls.map(([entry]) => JSON.parse(entry)); + expect(performanceLogs.find(({ requestId }) => requestId === 'cache-miss-request')).toMatchObject({ + cacheStatus: 'miss', + openSearchOperations: ['cpe-candidate', 'page-fetch'], + openSearchElapsedMs: expect.any(Number), + postFilterElapsedMs: expect.any(Number) + }); + expect(performanceLogs.find(({ requestId }) => requestId === 'cache-hit-request')).toMatchObject({ + cacheStatus: 'hit', + openSearchOperations: ['page-fetch'] + }); + expect(performanceLogs.find(({ requestId }) => requestId === 'cache-hit-request')).not.toHaveProperty( + 'openSearchElapsedMs' + ); + expect(performanceLogs.find(({ requestId }) => requestId === 'cache-hit-request')).not.toHaveProperty( + 'postFilterElapsedMs' + ); }); test.each([ @@ -471,6 +548,7 @@ describe('Search controller SEARCH handler', () => { ['malformed', ['CVE-2024-0001'], [{ _id: 'CVE-2024-0001', _source: { cveMetadata: {} } }]], ['unexpected', ['CVE-2024-0001'], [fullCveHit('CVE-2024-9999')]] ])('SEARCH fails closed for a %s full-record page response', async (_caseName, candidateIds, pageHits) => { + mockShouldLogSearchPerformance.mockReturnValue(true); const candidates = candidateIds.map((id) => cpeHit(id, { criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' @@ -493,7 +571,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); @@ -506,7 +584,9 @@ describe('Search controller SEARCH handler', () => { requestId: `${_caseName}-page-fetch-request`, outcome: 'upstream_invalid_response', statusCode: 502, - upstreamFailureType: 'incomplete_page_fetch' + upstreamFailureType: 'incomplete_page_fetch', + openSearchOperations: ['cpe-candidate', 'page-fetch'], + pageFetchElapsedMs: expect.any(Number) }); expect(next).not.toHaveBeenCalled(); }); @@ -528,7 +608,7 @@ describe('Search controller SEARCH handler', () => { expect(mockSearchOpenSearch).not.toHaveBeenCalled(); expect(res.status).not.toHaveBeenCalled(); - expect(res.json).not.toHaveBeenCalled(); + expect(res.jsonMock).not.toHaveBeenCalled(); expect(next).toHaveBeenCalledWith(expect.any(TypeError)); } ); @@ -558,7 +638,7 @@ describe('Search controller SEARCH handler', () => { expect(mockApiSearch).toHaveBeenCalledTimes(2); expect(res.status).toHaveBeenCalledWith(502); expect(secondRes.status).toHaveBeenCalledWith(200); - expect(secondRes.json).toHaveBeenCalledWith( + expect(secondRes.jsonMock).toHaveBeenCalledWith( expect.objectContaining({ page: 1, pageSize: 1, @@ -568,6 +648,7 @@ describe('Search controller SEARCH handler', () => { }); test('SEARCH coalesces concurrent identical cold semantic post-filter searches', async () => { + mockShouldLogSearchPerformance.mockReturnValue(true); const candidateSearch = deferred(); const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' @@ -583,26 +664,26 @@ describe('Search controller SEARCH handler', () => { } } }; - const req = { - searchParams: { - virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', - versionStart: '2.2', - versionStartType: 'including', - versionEnd: '2.6', - versionEndType: 'excluding', - resultsPerPage: '1', - pageNumber: '1' - } + const searchParams = { + virtualMatchString: 'cpe:2.3:o:linux:linux_kernel', + versionStart: '2.2', + versionStartType: 'including', + versionEnd: '2.6', + versionEndType: 'excluding', + resultsPerPage: '1', + pageNumber: '1' }; + const firstReq = { requestId: 'first-shared-request', searchParams }; + const secondReq = { requestId: 'second-shared-request', searchParams }; const firstRes = createResponse(); const secondRes = createResponse(); mockSearchRaw.mockReturnValue(candidateSearch.promise); mockApiSearch.mockResolvedValue(advancedResponse(1, [fullApplicable])); - const firstSearch = controller.SEARCH(req, firstRes, next); + const firstSearch = controller.SEARCH(firstReq, firstRes, next); await Promise.resolve(); - const secondSearch = controller.SEARCH(req, secondRes, next); + const secondSearch = controller.SEARCH(secondReq, secondRes, next); candidateSearch.resolve(rawResponse(2, [applicable, notApplicable])); await Promise.all([firstSearch, secondSearch]); @@ -611,7 +692,7 @@ describe('Search controller SEARCH handler', () => { expect(mockApiSearch).toHaveBeenCalledTimes(1); expect(firstRes.status).toHaveBeenCalledWith(200); expect(secondRes.status).toHaveBeenCalledWith(200); - expect(secondRes.json).toHaveBeenCalledWith({ + expect(secondRes.jsonMock).toHaveBeenCalledWith({ page: 1, pageSize: 1, totalPages: 1, @@ -626,9 +707,31 @@ describe('Search controller SEARCH handler', () => { } } }); + + const performanceLogs = console.log.mock.calls.map(([entry]) => JSON.parse(entry)); + const firstLog = performanceLogs.find(({ requestId }) => requestId === 'first-shared-request'); + const secondLog = performanceLogs.find(({ requestId }) => requestId === 'second-shared-request'); + expect(firstLog).toMatchObject({ + cacheStatus: 'miss', + inFlightHit: false, + openSearchOperations: ['cpe-candidate', 'page-fetch'], + openSearchElapsedMs: expect.any(Number), + postFilterElapsedMs: expect.any(Number), + pageFetchElapsedMs: expect.any(Number) + }); + expect(secondLog).toMatchObject({ + cacheStatus: 'in_flight', + inFlightHit: true, + openSearchOperations: ['cpe-candidate', 'page-fetch'], + openSearchElapsedMs: expect.any(Number), + postFilterElapsedMs: expect.any(Number), + pageFetchElapsedMs: expect.any(Number), + inFlightWaitElapsedMs: expect.any(Number) + }); }); test('SEARCH keeps shared CPE work active when one client disconnects', async () => { + mockShouldLogSearchPerformance.mockReturnValue(true); const candidateSearch = deferred(); const applicable = cpeHit('CVE-2024-0001', { criteria: 'cpe:2.3:o:linux:linux_kernel:2.5:*:*:*:*:*:*:*' @@ -650,6 +753,8 @@ describe('Search controller SEARCH handler', () => { }; firstReq.searchParams = searchParams; secondReq.searchParams = searchParams; + firstReq.requestId = 'disconnected-shared-request'; + secondReq.requestId = 'surviving-shared-request'; const firstRes = createResponse(); const secondRes = createResponse(); let operationSignal; @@ -676,6 +781,18 @@ describe('Search controller SEARCH handler', () => { expect(mockApiSearch).toHaveBeenCalledTimes(1); expect(firstRes.status).not.toHaveBeenCalled(); expect(secondRes.status).toHaveBeenCalledWith(200); + + const performanceLogs = console.log.mock.calls.map(([entry]) => JSON.parse(entry)); + expect(performanceLogs.find(({ requestId }) => requestId === 'disconnected-shared-request')).toMatchObject({ + outcome: 'client_disconnected', + openSearchOperations: [] + }); + expect(performanceLogs.find(({ requestId }) => requestId === 'surviving-shared-request')).toMatchObject({ + outcome: 'success', + openSearchOperations: ['cpe-candidate', 'page-fetch'], + openSearchElapsedMs: expect.any(Number), + postFilterElapsedMs: expect.any(Number) + }); }); test('SEARCH returns 429 when the cold semantic post-filter queue is full', async () => { @@ -715,7 +832,7 @@ describe('Search controller SEARCH handler', () => { expect(secondRes.set).toHaveBeenCalledWith('Retry-After', '5'); expect(secondRes.status).toHaveBeenCalledWith(429); - expect(secondRes.json).toHaveBeenCalledWith({ + expect(secondRes.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_QUEUE_FULL', message: 'Search service is busy. Please retry later.' }); @@ -782,7 +899,7 @@ describe('Search controller SEARCH handler', () => { expect(secondRes.set).toHaveBeenCalledWith('Retry-After', '5'); expect(secondRes.status).toHaveBeenCalledWith(429); - expect(secondRes.json).toHaveBeenCalledWith({ + expect(secondRes.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_QUEUE_TIMEOUT', message: 'Search service is busy. Please retry later.' }); @@ -828,7 +945,7 @@ describe('Search controller SEARCH handler', () => { expect(secondRes.set).toHaveBeenCalledWith('Retry-After', '5'); expect(secondRes.status).toHaveBeenCalledWith(429); - expect(secondRes.json).toHaveBeenCalledWith({ + expect(secondRes.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_IN_FLIGHT_WAITER_LIMIT', message: 'Search service is busy. Please retry later.' }); @@ -848,6 +965,7 @@ describe('Search controller SEARCH handler', () => { }); test('SEARCH returns sanitized upstream errors when optimized post-filter search fails', async () => { + mockShouldLogSearchPerformance.mockReturnValue(true); mockSearchRaw.mockRejectedValue({ name: 'ResponseError', message: 'candidate search failed', @@ -858,6 +976,7 @@ describe('Search controller SEARCH handler', () => { await controller.SEARCH( { + requestId: 'candidate-failure-request', searchParams: { virtualMatchString: 'cpe:2.3:o:microsoft:windows_10_1809', resultsPerPage: '1', @@ -870,10 +989,16 @@ describe('Search controller SEARCH handler', () => { expect(mockApiSearch).not.toHaveBeenCalled(); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); + expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ + requestId: 'candidate-failure-request', + outcome: 'upstream_error', + openSearchOperations: ['cpe-candidate'], + openSearchElapsedMs: expect.any(Number) + }); expect(next).not.toHaveBeenCalled(); }); @@ -900,7 +1025,7 @@ describe('Search controller SEARCH handler', () => { expect(mockSearchRaw).toHaveBeenCalledTimes(2); for (const response of [res, secondRes]) { expect(response.status).toHaveBeenCalledWith(502); - expect(response.json).toHaveBeenCalledWith({ + expect(response.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); @@ -932,7 +1057,7 @@ describe('Search controller SEARCH handler', () => { expect(mockSearchRaw).toHaveBeenCalledTimes(2); for (const response of [res, secondRes]) { expect(response.status).toHaveBeenCalledWith(502); - expect(response.json).toHaveBeenCalledWith({ + expect(response.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); @@ -993,7 +1118,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 0, pageSize: 0, totalPages: 0, @@ -1017,7 +1142,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 2, pageSize: 0, totalPages: 1, @@ -1041,7 +1166,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ page: 333, pageSize: 1, totalPages: 333, @@ -1069,7 +1194,7 @@ describe('Search controller SEARCH handler', () => { expect(res.status).toHaveBeenCalledWith(503); expect(res.set).toHaveBeenCalledWith('Retry-After', '5'); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'OPEN_SEARCH_ERROR', message: 'Search backend is temporarily unavailable. Please try again later.' }); @@ -1104,7 +1229,7 @@ describe('Search controller SEARCH handler', () => { expect(res.status).toHaveBeenCalledWith(503); expect(res.set).toHaveBeenCalledWith('Retry-After', '5'); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'OPEN_SEARCH_ERROR', message: 'Search backend is temporarily unavailable. Please try again later.' }); @@ -1137,7 +1262,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(504); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_TIMEOUT', message: 'Search request timed out. Please try again later.' }); @@ -1169,7 +1294,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); @@ -1199,7 +1324,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(504); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_TIMEOUT', message: 'Search request timed out. Please try again later.' }); @@ -1236,7 +1361,7 @@ describe('Search controller SEARCH handler', () => { expect(mockGetSearchRequestDeadlineMs).toHaveBeenCalledTimes(1); expect(res.status).toHaveBeenCalledWith(504); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'SEARCH_TIMEOUT', message: 'Search request timed out. Please try again later.' }); @@ -1267,7 +1392,7 @@ describe('Search controller SEARCH handler', () => { await search; expect(res.status).not.toHaveBeenCalled(); - expect(res.json).not.toHaveBeenCalled(); + expect(res.jsonMock).not.toHaveBeenCalled(); expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ event: 'search.performance', requestId: 'client-disconnect-request', @@ -1299,7 +1424,7 @@ describe('Search controller SEARCH handler', () => { expect(mockApiSearch).not.toHaveBeenCalled(); expect(res.status).not.toHaveBeenCalled(); - expect(res.json).not.toHaveBeenCalled(); + expect(res.jsonMock).not.toHaveBeenCalled(); }); test('SEARCH returns sanitized errors from cve-core', async () => { @@ -1323,7 +1448,7 @@ describe('Search controller SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ error: 'UPSTREAM_SEARCH_ERROR', message: 'Search backend returned an error. Please try again later.' }); @@ -1351,7 +1476,7 @@ describe('Search controller SEARCH handler', () => { expect(next).toHaveBeenCalledWith(err); expect(res.status).not.toHaveBeenCalled(); - expect(res.json).not.toHaveBeenCalled(); + expect(res.jsonMock).not.toHaveBeenCalled(); expect(JSON.parse(console.log.mock.calls[0][0])).toMatchObject({ event: 'search.performance', requestId: 'internal-error-request', @@ -1425,7 +1550,7 @@ describe('Search controller WEB_SEARCH handler', () => { size: 10 }); expect(res.status).toHaveBeenCalledWith(200); - expect(res.json).toHaveBeenCalledWith({ + expect(res.jsonMock).toHaveBeenCalledWith({ searchMetadata: { searchStatus: 'ok', version: '2025-08-26', @@ -1570,7 +1695,7 @@ describe('Search controller WEB_SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json.mock.calls[0][0]).toStrictEqual({ + expect(res.jsonMock.mock.calls[0][0]).toStrictEqual({ searchMetadata: { searchStatus: 'errors', version: '2025-08-26', @@ -1604,7 +1729,7 @@ describe('Search controller WEB_SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(502); - expect(res.json.mock.calls[0][0]).toStrictEqual({ + expect(res.jsonMock.mock.calls[0][0]).toStrictEqual({ searchMetadata: { searchStatus: 'errors', version: '2025-08-26', @@ -1641,7 +1766,7 @@ describe('Search controller WEB_SEARCH handler', () => { ); expect(res.status).toHaveBeenCalledWith(504); - expect(res.json.mock.calls[0][0]).toStrictEqual({ + expect(res.jsonMock.mock.calls[0][0]).toStrictEqual({ searchMetadata: { searchStatus: 'errors', version: '2025-08-26', diff --git a/routes/index.js b/routes/index.js index ce80d39..1314788 100644 --- a/routes/index.js +++ b/routes/index.js @@ -343,6 +343,24 @@ content: { } } } +#swagger.responses[422] = { +description: 'The serialized successful response exceeded the configured maximum size; reduce resultsPerPage and retry', +headers: { + "X-Request-ID": { + description: "Request correlation ID", + schema: { + type: "string" + } + } +}, +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ErrorResponse" + } + } +} +} #swagger.responses[429] = { description: 'Application rate limit exceeded, the cold CPE search queue is full or exceeded its maximum wait, or a shared semantic search operation reached its waiter limit', headers: { diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index 68ab08c..c5ca3cc 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -2,6 +2,7 @@ const { request } = require('../test/http'); const mockSearchOpenSearch = jest.fn(); const mockBasicSearch = jest.fn(); +const mockGetSearchMaxResponseBytes = jest.fn(() => 10 * 1024 * 1024); jest.mock('../utils/searchProvider', () => ({ getBasicSearchManager: jest.fn(() => ({ @@ -9,6 +10,7 @@ jest.mock('../utils/searchProvider', () => ({ })), getInFlightMaxWaitersPerOperation: jest.fn(() => 500), getPostFilterEvaluationChunkSize: jest.fn(() => 250), + getSearchMaxResponseBytes: mockGetSearchMaxResponseBytes, getSearchRequestDeadlineMs: jest.fn(() => 30000), isOpenSearchResponseError: jest.fn(() => false), isSearchProviderConfigurationError: jest.fn(() => false), @@ -53,6 +55,8 @@ describe('search route integration', () => { beforeEach(() => { mockSearchOpenSearch.mockReset(); mockBasicSearch.mockReset(); + mockGetSearchMaxResponseBytes.mockReset(); + mockGetSearchMaxResponseBytes.mockReturnValue(10 * 1024 * 1024); }); afterEach(() => { @@ -151,6 +155,46 @@ describe('search route integration', () => { expect(requestErrorLog.mock.calls[0][0]).not.toContain('sensitive'); }); + test('POST /search returns one complete 422 response when the successful payload exceeds its byte limit', async () => { + const performanceLog = jest.spyOn(console, 'log').mockImplementation(() => {}); + mockGetSearchMaxResponseBytes.mockReturnValue(256); + mockSearchOpenSearch.mockResolvedValue( + advancedResponse(1, [ + { + _id: 'CVE-2024-0001', + _source: { + description: 'x'.repeat(512) + } + } + ]).data + ); + + const response = await request(createApp(), { + method: 'POST', + path: '/search', + body: { + keywordSearch: 'remote code', + resultsPerPage: 1, + pageNumber: 1 + } + }); + + expect(response.status).toBe(422); + expect(response.body).toStrictEqual({ + error: 'SEARCH_RESPONSE_TOO_LARGE', + message: 'Search response exceeds the maximum response size. Reduce resultsPerPage and try again.' + }); + expect(response.text).toBe(JSON.stringify(response.body)); + expect(JSON.parse(performanceLog.mock.calls[0][0])).toMatchObject({ + event: 'search.performance', + outcome: 'response_too_large', + statusCode: 422, + maxResponseBytes: 256, + attemptedResponseBytes: expect.any(Number), + attemptedResponseSerializationElapsedMs: expect.any(Number) + }); + }); + test('POST /search rejects unsupported CPE parts before invoking OpenSearch', async () => { const response = await request(createApp(), { method: 'POST', @@ -169,6 +213,23 @@ describe('search route integration', () => { expect(mockSearchOpenSearch).not.toHaveBeenCalled(); }); + test('POST /search rejects a malformed CWE identifier before invoking OpenSearch', async () => { + const response = await request(createApp(), { + method: 'POST', + path: '/search', + body: { + cweId: 'CWE-079' + } + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'CWE_ID_FORMAT', + message: 'The cweId parameter must use the format CWE- without leading zeros.' + }); + expect(mockSearchOpenSearch).not.toHaveBeenCalled(); + }); + test('POST /webSearch normalizes body fields and formats BasicSearchManager responses', async () => { mockBasicSearch.mockResolvedValue(basicResponse({ total: 1, hits: [{ _id: 'CVE-2024-0002' }] })); diff --git a/routes/swagger.js b/routes/swagger.js index 0629922..697cbef 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -43,10 +43,12 @@ const searchRequestProperties = { default: false }), cweId: searchRequestProperty( - 'Returns CVE records whose CNA data includes the provided Common Weakness Enumeration ID. ADP-provided CWE data is not searched.', + 'Returns CVE records whose CNA data includes the provided Common Weakness Enumeration ID. The accepted format is the CWE- prefix followed by a positive integer without leading zeros. Prefix matching is case-insensitive, surrounding whitespace is trimmed, and valid input is normalized to uppercase before search. Invalid values return HTTP 400. ADP-provided CWE data is not searched.', { type: 'string', + minLength: 5, maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cweId, + pattern: constants.CWE_ID_PATTERN, example: 'CWE-79' } ), diff --git a/test/contract/opensearch.js b/test/contract/opensearch.js index 56e71a8..3492d08 100644 --- a/test/contract/opensearch.js +++ b/test/contract/opensearch.js @@ -2,6 +2,7 @@ const assert = require('assert/strict'); require('dotenv').config(); const REQUIRED_ENVIRONMENT_VARIABLES = ['OpenSearchDomainEndpoint', 'OpenSearchCveIndex']; +const CPE_CRITERIA_KEYWORD_FIELD = 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword'; const FIELD_CAPABILITY_REQUIREMENTS = Object.freeze([ { name: 'cve-id-keyword', @@ -26,7 +27,7 @@ const FIELD_CAPABILITY_REQUIREMENTS = Object.freeze([ name: 'cpe-criteria-keyword', anyOf: [ { - field: 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword', + field: CPE_CRITERIA_KEYWORD_FIELD, types: ['keyword'] } ] @@ -61,23 +62,26 @@ function buildStandardBody(queryBuilder, params, options = {}) { } function buildSourceProjectionCheck(queryBuilder = require('../../utils/searchQueryBuilder')) { + const request = queryBuilder.buildSearchRequest({ + resultsPerPage: 1, + pageNumber: 1 + }); + const body = queryBuilder.buildCpeCandidateSearchBody(request.searchOptions, request.queryStringObjects); + body.query.bool.filter.push({ + exists: { + field: CPE_CRITERIA_KEYWORD_FIELD + } + }); + return { name: 'source-projection', - body: buildStandardBody( - queryBuilder, - { - resultsPerPage: 1, - pageNumber: 1 - }, - { - sourceIncludes: ['cveMetadata.cveId'] - } - ) + body }; } -function buildContractChecks(contractCveId, queryBuilder = require('../../utils/searchQueryBuilder')) { - assert.equal(typeof contractCveId, 'string'); +function buildContractChecks(contractCandidate, queryBuilder = require('../../utils/searchQueryBuilder')) { + assert.equal(typeof contractCandidate?.cveId, 'string'); + assert.equal(typeof contractCandidate?.criteria, 'string'); const buildBody = (params, options = {}) => { return buildStandardBody(queryBuilder, params, options); }; @@ -91,7 +95,7 @@ function buildContractChecks(contractCveId, queryBuilder = require('../../utils/ name: 'cve-id-keyword-and-total', body: buildBody( { - cveIds: contractCveId, + cveIds: contractCandidate.cveId, resultsPerPage: 1, pageNumber: 1 }, @@ -103,7 +107,7 @@ function buildContractChecks(contractCveId, queryBuilder = require('../../utils/ validate(body) { assert.equal(Number.isInteger(body.hits.total.value), true); assert.equal(body.hits.total.value >= 1, true); - assert.equal(body.hits.hits[0]?._source?.cveMetadata?.cveId, contractCveId); + assert.equal(body.hits.hits[0]?._source?.cveMetadata?.cveId, contractCandidate.cveId); } }, { @@ -134,11 +138,14 @@ function buildContractChecks(contractCveId, queryBuilder = require('../../utils/ name: 'cpe-candidate', body: { ...buildCandidateBody({ - virtualMatchString: 'cpe:2.3:a:contract-probe:contract-probe', + virtualMatchString: contractCandidate.criteria, resultsPerPage: 1, pageNumber: 1 }), - size: 0 + size: 1 + }, + async validate(body) { + await validateCpeCandidateResponse(body, contractCandidate.criteria); } } ]; @@ -165,19 +172,49 @@ function validateFieldCapabilities(body, requirements = FIELD_CAPABILITY_REQUIRE }); } -function getContractCveId(body) { +function getCandidateCpeMatch(hit) { + const cpeApplicability = hit._source.containers.cna.cpeApplicability; + const applicabilityItems = Array.isArray(cpeApplicability) ? cpeApplicability : [cpeApplicability]; + + return applicabilityItems[0].nodes[0].cpeMatch[0]; +} + +async function getContractCandidate(body, candidateProcessor = require('../../utils/searchCandidateProcessor')) { assert.equal(body.hits.hits.length, 1, 'source projection did not return a CVE record'); const [hit] = body.hits.hits; + const processed = await candidateProcessor.filterSearchHitsInChunks({ data: body }, {}); const sourceCveId = hit?._source?.cveMetadata?.cveId; + const criteria = getCandidateCpeMatch(hit).criteria; assert.equal(typeof hit?._id, 'string'); assert.equal(typeof sourceCveId, 'string'); - return sourceCveId; + assert.equal(processed.filteredHits.length, 1, 'source projection did not return a valid CPE candidate'); + assert.equal(typeof criteria, 'string'); + + return { + cveId: sourceCveId, + criteria + }; +} + +async function validateCpeCandidateResponse( + body, + virtualMatchString, + candidateProcessor = require('../../utils/searchCandidateProcessor') +) { + assert.equal(body.hits.hits.length >= 1, true, 'cpe-candidate response did not return a CVE record'); + const processed = await candidateProcessor.filterSearchHitsInChunks( + { data: body }, + { + virtualMatchString + } + ); + assert.equal(processed.filteredHits.length >= 1, true, 'cpe-candidate response did not contain a matching CPE'); } -function validateContractResponse(check, body) { +async function validateContractResponse(check, body) { assert.ok(Array.isArray(body.hits.hits), `${check.name} response did not include hits.hits`); - check.validate?.(body); + await check.validate?.(body); } async function run({ @@ -199,15 +236,15 @@ async function run({ const sourceProjectionBody = await searchProvider.searchOpenSearch(sourceProjectionCheck.body, { operation: `contract-${sourceProjectionCheck.name}` }); - validateContractResponse(sourceProjectionCheck, sourceProjectionBody); - const contractCveId = getContractCveId(sourceProjectionBody); + await validateContractResponse(sourceProjectionCheck, sourceProjectionBody); + const contractCandidate = await getContractCandidate(sourceProjectionBody); completedChecks.push(sourceProjectionCheck.name); - for (const check of buildContractChecks(contractCveId, queryBuilder)) { + for (const check of buildContractChecks(contractCandidate, queryBuilder)) { const body = await searchProvider.searchOpenSearch(check.body, { operation: check.operation ?? `contract-${check.name}` }); - validateContractResponse(check, body); + await validateContractResponse(check, body); completedChecks.push(check.name); } @@ -235,11 +272,12 @@ module.exports = { buildSourceProjectionCheck, buildStandardBody, FIELD_CAPABILITY_REQUIREMENTS, - getContractCveId, + getContractCandidate, getSanitizedFailure, getRequiredFieldNames, requireConfiguration, run, + validateCpeCandidateResponse, validateFieldCapabilities, validateContractResponse }; diff --git a/test/contract/opensearch.test.js b/test/contract/opensearch.test.js index e95bb39..785c72a 100644 --- a/test/contract/opensearch.test.js +++ b/test/contract/opensearch.test.js @@ -17,6 +17,53 @@ function validResponse(overrides = {}) { }; } +function validCandidateHit() { + return { + _id: 'CVE-2024-0001', + _source: { + cveMetadata: { + cveId: 'CVE-2024-0001' + }, + containers: { + cna: { + cpeApplicability: [ + { + nodes: [ + { + cpeMatch: [ + { + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*', + vulnerable: true + } + ] + } + ] + } + ] + } + } + } + }; +} + +function validCandidateResponse(hit = validCandidateHit()) { + return validResponse({ + hits: { + total: { + value: 1, + relation: 'eq' + }, + hits: [hit] + } + }); +} + +function mutateCandidate(mutator) { + const hit = validCandidateHit(); + mutator(hit); + return hit; +} + function validFieldCapabilities() { return { fields: Object.fromEntries( @@ -55,7 +102,10 @@ describe('OpenSearch contract check', () => { test('builds representative read-only production query shapes', () => { const sourceProjection = contract.buildSourceProjectionCheck(); - const checks = contract.buildContractChecks('CVE-2024-0001'); + const checks = contract.buildContractChecks({ + cveId: 'CVE-2024-0001', + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*' + }); expect([sourceProjection, ...checks].map(({ name }) => name)).toStrictEqual([ 'source-projection', @@ -66,7 +116,30 @@ describe('OpenSearch contract check', () => { 'cpe-candidate' ]); expect([sourceProjection, ...checks].every(({ body }) => body.size <= 1)).toBe(true); - expect(sourceProjection.body._source).toStrictEqual(['cveMetadata.cveId']); + expect(sourceProjection.body).toMatchObject({ + size: 1, + track_total_hits: false, + query: { + bool: { + filter: [ + { + exists: { + field: 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria.keyword' + } + } + ] + } + } + }); + expect(sourceProjection.body._source).toStrictEqual([ + 'cveMetadata.cveId', + 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria', + 'containers.cna.cpeApplicability.nodes.cpeMatch.vulnerable', + 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartIncluding', + 'containers.cna.cpeApplicability.nodes.cpeMatch.versionStartExcluding', + 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndIncluding', + 'containers.cna.cpeApplicability.nodes.cpeMatch.versionEndExcluding' + ]); expect(checks.find(({ name }) => name === 'cve-id-keyword-and-total').body).toMatchObject({ track_total_hits: true, sort: [ @@ -85,7 +158,7 @@ describe('OpenSearch contract check', () => { ]); expect(checks.find(({ name }) => name === 'cvss').body.query.bool.filter[0]).toHaveProperty('bool.should'); expect(checks.find(({ name }) => name === 'cpe-candidate').body).toMatchObject({ - size: 0, + size: 1, track_total_hits: false }); }); @@ -106,28 +179,76 @@ describe('OpenSearch contract check', () => { ); }); - test('requires source projection to return a concrete CVE record', () => { - expect(() => contract.getContractCveId(validResponse())).toThrow( + test('requires source projection to return a concrete CPE candidate', async () => { + await expect(contract.getContractCandidate(validResponse())).rejects.toThrow( 'source projection did not return a CVE record' ); - expect( - contract.getContractCveId( - validResponse({ - hits: { - hits: [ - { - _id: 'record-1', - _source: { - cveMetadata: { - cveId: 'CVE-2024-0001' - } - } - } - ] - } - }) - ) - ).toBe('CVE-2024-0001'); + await expect(contract.getContractCandidate(validCandidateResponse())).resolves.toStrictEqual({ + cveId: 'CVE-2024-0001', + criteria: 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*' + }); + }); + + test.each([ + [ + 'missing CVE ID', + mutateCandidate((hit) => { + delete hit._source.cveMetadata.cveId; + }) + ], + [ + 'missing CPE applicability', + mutateCandidate((hit) => { + delete hit._source.containers.cna.cpeApplicability; + }) + ], + [ + 'missing CPE nodes', + mutateCandidate((hit) => { + delete hit._source.containers.cna.cpeApplicability[0].nodes; + }) + ], + [ + 'missing CPE matches', + mutateCandidate((hit) => { + delete hit._source.containers.cna.cpeApplicability[0].nodes[0].cpeMatch; + }) + ], + [ + 'invalid CPE criteria', + mutateCandidate((hit) => { + hit._source.containers.cna.cpeApplicability[0].nodes[0].cpeMatch[0].criteria = 'not-a-cpe'; + }) + ], + [ + 'missing vulnerable value', + mutateCandidate((hit) => { + delete hit._source.containers.cna.cpeApplicability[0].nodes[0].cpeMatch[0].vulnerable; + }) + ], + [ + 'invalid version range', + mutateCandidate((hit) => { + hit._source.containers.cna.cpeApplicability[0].nodes[0].cpeMatch[0].versionEndExcluding = 2; + }) + ] + ])('rejects a source projection with %s', async (_caseName, hit) => { + await expect(contract.getContractCandidate(validCandidateResponse(hit))).rejects.toMatchObject({ + code: 'OPEN_SEARCH_RESPONSE_INVALID', + failureType: 'malformed_candidate' + }); + }); + + test('requires the CPE candidate query to return a matching candidate', async () => { + await expect( + contract.validateCpeCandidateResponse(validCandidateResponse(), 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*') + ).resolves.toBeUndefined(); + await expect( + contract.validateCpeCandidateResponse(validResponse(), 'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*') + ).rejects.toThrow('cpe-candidate response did not return a CVE record'); + await expect( + contract.validateCpeCandidateResponse(validCandidateResponse(), 'cpe:2.3:a:other:product:*:*:*:*:*:*:*:*') + ).rejects.toThrow('cpe-candidate response did not contain a matching CPE'); }); test('runs every contract check and closes the provider', async () => { @@ -138,6 +259,9 @@ describe('OpenSearch contract check', () => { checkOpenSearchReadiness: jest.fn().mockResolvedValue(true), getOpenSearchFieldCapabilities: jest.fn().mockResolvedValue(validFieldCapabilities()), searchOpenSearch: jest.fn().mockImplementation((_body, { operation }) => { + if (operation === 'contract-source-projection' || operation === 'contract-cpe-candidate') { + return Promise.resolve(validCandidateResponse()); + } if (operation === 'standard') { return Promise.resolve( validResponse({ @@ -160,24 +284,6 @@ describe('OpenSearch contract check', () => { }) ); } - if (operation === 'contract-source-projection') { - return Promise.resolve( - validResponse({ - hits: { - hits: [ - { - _id: 'CVE-2024-0001', - _source: { - cveMetadata: { - cveId: 'CVE-2024-0001' - } - } - } - ] - } - }) - ); - } return Promise.resolve(validResponse()); }), closeSearchProviderClients: jest.fn().mockResolvedValue(undefined) diff --git a/utils/constants.js b/utils/constants.js index 8a1b1c7..b98fb65 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -8,6 +8,7 @@ const MAX_CVE_IDS = 100; const MIN_CVE_ID_SEQUENCE_DIGITS = 4; const MAX_CVE_ID_SEQUENCE_DIGITS = 19; const MAX_KEYWORD_SEARCH_TOKENS = 64; +const CWE_ID_PATTERN = '^\\s*[Cc][Ww][Ee]-[1-9][0-9]*\\s*$'; const JSON_REQUEST_BODY_LIMIT_KB = 100; const JSON_REQUEST_BODY_LIMIT_BYTES = JSON_REQUEST_BODY_LIMIT_KB * 1024; const DATE_TIME_PATTERN = @@ -31,6 +32,7 @@ const SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE = 250; const SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS = 20 * 1000; const SEARCH_REQUEST_DEADLINE_MS = 30 * 1000; const SEARCH_READINESS_TIMEOUT_MS = 5 * 1000; +const SEARCH_MAX_RESPONSE_BYTES = 10 * 1024 * 1024; const RUNTIME_SHUTDOWN_TIMEOUT_MS = 35 * 1000; const HTTP_HEADERS_TIMEOUT_MS = 10 * 1000; const HTTP_REQUEST_TIMEOUT_MS = 15 * 1000; @@ -141,6 +143,7 @@ module.exports = { MIN_CVE_ID_SEQUENCE_DIGITS, MAX_CVE_ID_SEQUENCE_DIGITS, MAX_KEYWORD_SEARCH_TOKENS, + CWE_ID_PATTERN, JSON_REQUEST_BODY_LIMIT_KB, JSON_REQUEST_BODY_LIMIT_BYTES, DATE_TIME_PATTERN, @@ -155,6 +158,7 @@ module.exports = { SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS, SEARCH_REQUEST_DEADLINE_MS, SEARCH_READINESS_TIMEOUT_MS, + SEARCH_MAX_RESPONSE_BYTES, RUNTIME_SHUTDOWN_TIMEOUT_MS, HTTP_HEADERS_TIMEOUT_MS, HTTP_REQUEST_TIMEOUT_MS, diff --git a/utils/cpe.js b/utils/cpe.js index dd75191..6402b6f 100644 --- a/utils/cpe.js +++ b/utils/cpe.js @@ -34,6 +34,9 @@ function parseCpe23(input) { if (components.length < 4 || components.length > CPE_COMPONENT_COUNT) { return { valid: false, reason: 'component_count' }; } + if (components.some((component) => component.trim() === '')) { + return { valid: false, reason: 'empty_component' }; + } if (!isValidCpePart(components[0], { allowLogicalValues: true })) { return { valid: false, reason: 'part' }; } diff --git a/utils/cpe.test.js b/utils/cpe.test.js index b8317a2..3579723 100644 --- a/utils/cpe.test.js +++ b/utils/cpe.test.js @@ -31,6 +31,19 @@ describe('CPE helpers', () => { ]); }); + test.each([ + 'cpe:2.3::vendor:product:1.0', + 'cpe:2.3:a:vendor::1.0', + 'cpe:2.3:a:vendor:product:1.0:', + 'cpe:2.3:a:vendor:product:1.0::*', + 'cpe:2.3:a:vendor:product:1.0: :*' + ])('parseCpeName rejects explicit empty component in %s', (value) => { + expect(cpe.parseCpeName(value)).toStrictEqual({ + valid: false, + reason: 'empty_component' + }); + }); + test.each([ 'not-a-cpe', 'cpe:2.3:a:vendor:product', diff --git a/utils/cweId.js b/utils/cweId.js new file mode 100644 index 0000000..045965b --- /dev/null +++ b/utils/cweId.js @@ -0,0 +1,20 @@ +const constants = require('./constants'); + +const CWE_ID_PATTERN = new RegExp(constants.CWE_ID_PATTERN); + +function normalizeCweId(value) { + return typeof value === 'string' ? value.trim().toUpperCase() : ''; +} + +function isValidCweId(value) { + return ( + typeof value === 'string' && + value.length <= constants.SEARCH_STRING_LENGTH_LIMITS.cweId && + CWE_ID_PATTERN.test(value) + ); +} + +module.exports = { + isValidCweId, + normalizeCweId +}; diff --git a/utils/cweId.test.js b/utils/cweId.test.js new file mode 100644 index 0000000..0f7e2f1 --- /dev/null +++ b/utils/cweId.test.js @@ -0,0 +1,33 @@ +const constants = require('./constants'); +const cweId = require('./cweId'); + +describe('cweId', () => { + test.each([ + ['canonical identifier', 'CWE-79'], + ['case-insensitive prefix', 'cwe-79'], + ['surrounding whitespace', ' CwE-123 '], + ['one-digit identifier', 'CWE-1'], + ['maximum-length identifier', `CWE-${'1'.repeat(constants.SEARCH_STRING_LENGTH_LIMITS.cweId - 4)}`] + ])('accepts a %s', (_caseName, value) => { + expect(cweId.isValidCweId(value)).toBe(true); + }); + + test.each([ + ['empty value', ''], + ['missing prefix', '79'], + ['zero identifier', 'CWE-0'], + ['leading zero', 'CWE-079'], + ['missing numeric identifier', 'CWE-'], + ['non-numeric identifier', 'CWE-noinfo'], + ['embedded whitespace', 'CWE- 79'], + ['suffix', 'CWE-79-extra'], + ['overlength identifier', `CWE-${'1'.repeat(constants.SEARCH_STRING_LENGTH_LIMITS.cweId - 3)}`], + ['non-string value', ['CWE-79']] + ])('rejects an identifier with %s', (_caseName, value) => { + expect(cweId.isValidCweId(value)).toBe(false); + }); + + test('normalizes surrounding whitespace and prefix case', () => { + expect(cweId.normalizeCweId(' cWe-79 ')).toBe('CWE-79'); + }); +}); diff --git a/utils/errors.js b/utils/errors.js index 9b442d8..3de68cc 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -123,6 +123,13 @@ class errors { ); } + cweIdFormatError() { + return makeError( + 'CWE_ID_FORMAT', + 'The cweId parameter must use the format CWE- without leading zeros.' + ); + } + noRejectedValueError() { return makeError('NO_REJECTED_VALUE', 'The noRejected field must be a boolean.'); } @@ -149,6 +156,13 @@ class errors { ); } + searchResponseTooLargeError() { + return makeError( + 'SEARCH_RESPONSE_TOO_LARGE', + 'Search response exceeds the maximum response size. Reduce resultsPerPage and try again.' + ); + } + searchQueryUnsupportedError() { return makeError( 'SEARCH_QUERY_UNSUPPORTED', diff --git a/utils/errors.test.js b/utils/errors.test.js index 582098e..e3b20fe 100644 --- a/utils/errors.test.js +++ b/utils/errors.test.js @@ -167,6 +167,14 @@ describe('error response helpers', () => { message: `The cveIds parameter cannot contain more than ${constants.MAX_CVE_IDS} CVE IDs.` } ], + [ + 'cweIdFormatError', + [], + { + error: 'CWE_ID_FORMAT', + message: 'The cweId parameter must use the format CWE- without leading zeros.' + } + ], [ 'noRejectedValueError', [], @@ -271,6 +279,14 @@ describe('error response helpers', () => { message: `The JSON request body cannot exceed ${constants.JSON_REQUEST_BODY_LIMIT_KB} KB.` } ], + [ + 'searchResponseTooLargeError', + [], + { + error: 'SEARCH_RESPONSE_TOO_LARGE', + message: 'Search response exceeds the maximum response size. Reduce resultsPerPage and try again.' + } + ], [ 'searchQueryUnsupportedError', [], diff --git a/utils/runtimeConfig.js b/utils/runtimeConfig.js index 77745c4..3558c7f 100644 --- a/utils/runtimeConfig.js +++ b/utils/runtimeConfig.js @@ -21,6 +21,7 @@ const RUNTIME_CONFIG_LIMITS = Object.freeze({ openSearchRequestTimeoutMs: Object.freeze({ min: 1, max: 5 * 60 * 1000 }), requestDeadlineMs: Object.freeze({ min: 1, max: 10 * 60 * 1000 }), readinessTimeoutMs: Object.freeze({ min: 1, max: 60 * 1000 }), + maxResponseBytes: Object.freeze({ min: 64 * 1024, max: 100 * 1024 * 1024 }), shutdownTimeoutMs: Object.freeze({ min: 1, max: 15 * 60 * 1000 }), httpHeadersTimeoutMs: Object.freeze({ min: 1, max: 60 * 1000 }), httpRequestTimeoutMs: Object.freeze({ min: 1, max: 5 * 60 * 1000 }), @@ -289,6 +290,11 @@ function resolveRuntimeConfig({ 'search.readinessTimeoutMs', readConfigValue(appConfig, 'search.readinessTimeoutMs', constants.SEARCH_READINESS_TIMEOUT_MS), RUNTIME_CONFIG_LIMITS.readinessTimeoutMs + ), + maxResponseBytes: parseInteger( + 'search.maxResponseBytes', + readConfigValue(appConfig, 'search.maxResponseBytes', constants.SEARCH_MAX_RESPONSE_BYTES), + RUNTIME_CONFIG_LIMITS.maxResponseBytes ) }, requestLogging: { @@ -398,7 +404,8 @@ function sanitizeRuntimeConfig(config) { postFilterEvaluationChunkSize: config.search.postFilterEvaluationChunkSize, openSearchRequestTimeoutMs: config.search.openSearchRequestTimeoutMs, requestDeadlineMs: config.search.requestDeadlineMs, - readinessTimeoutMs: config.search.readinessTimeoutMs + readinessTimeoutMs: config.search.readinessTimeoutMs, + maxResponseBytes: config.search.maxResponseBytes }, requestLogging: config.requestLogging, runtime: config.runtime, diff --git a/utils/runtimeConfig.test.js b/utils/runtimeConfig.test.js index 90777c4..f13350b 100644 --- a/utils/runtimeConfig.test.js +++ b/utils/runtimeConfig.test.js @@ -56,7 +56,8 @@ describe('runtimeConfig', () => { postFilterEvaluationChunkSize: constants.SEARCH_POST_FILTER_EVALUATION_CHUNK_SIZE, openSearchRequestTimeoutMs: constants.SEARCH_OPEN_SEARCH_REQUEST_TIMEOUT_MS, requestDeadlineMs: constants.SEARCH_REQUEST_DEADLINE_MS, - readinessTimeoutMs: constants.SEARCH_READINESS_TIMEOUT_MS + readinessTimeoutMs: constants.SEARCH_READINESS_TIMEOUT_MS, + maxResponseBytes: constants.SEARCH_MAX_RESPONSE_BYTES }, requestLogging: { successSampleRate: 0.1, @@ -101,6 +102,7 @@ describe('runtimeConfig', () => { 'search.openSearchRequestTimeoutMs': '25000', 'search.requestDeadlineMs': '45000', 'search.readinessTimeoutMs': '7000', + 'search.maxResponseBytes': '20971520', 'requestLogging.successSampleRate': '0.25', 'requestLogging.healthChecks': 'TRUE', 'runtime.shutdownTimeoutMs': '60000', @@ -138,7 +140,8 @@ describe('runtimeConfig', () => { postFilterEvaluationChunkSize: 100, openSearchRequestTimeoutMs: 25000, requestDeadlineMs: 45000, - readinessTimeoutMs: 7000 + readinessTimeoutMs: 7000, + maxResponseBytes: 20971520 }, requestLogging: { successSampleRate: 0.25, @@ -182,6 +185,7 @@ describe('runtimeConfig', () => { ['search.postFilterColdSearchConcurrency', {}, { 'search.postFilterColdSearchConcurrency': '101' }], ['search.postFilterColdSearchMaxQueueDepth', {}, { 'search.postFilterColdSearchMaxQueueDepth': '10001' }], ['search.postFilterEvaluationChunkSize', {}, { 'search.postFilterEvaluationChunkSize': '1.5' }], + ['search.maxResponseBytes', {}, { 'search.maxResponseBytes': '65535' }], ['runtime.http.headersTimeoutMs', {}, { 'runtime.http.headersTimeoutMs': '0' }], ['runtime.http.requestTimeoutMs', {}, { 'runtime.http.requestTimeoutMs': 'invalid' }], ['runtime.http.keepAliveTimeoutMs', {}, { 'runtime.http.keepAliveTimeoutMs': '60001' }], diff --git a/utils/searchCandidateProcessor.test.js b/utils/searchCandidateProcessor.test.js index d3bd3d3..b3ba995 100644 --- a/utils/searchCandidateProcessor.test.js +++ b/utils/searchCandidateProcessor.test.js @@ -155,6 +155,10 @@ describe('searchCandidateProcessor', () => { ['nested-only CPE applicability entry', nestedOnlyCpeCandidateHit()], ['missing CPE criteria', cpeCandidateHit({ criteria: undefined })], ['invalid CPE criteria', cpeCandidateHit({ criteria: 'not-a-cpe-name' })], + [ + 'CPE criteria with an explicit empty component', + cpeCandidateHit({ criteria: 'cpe:2.3:a:vendor:product:*:*::*:*:*:*:*' }) + ], ['missing CPE vulnerability flag', cpeCandidateHit({ vulnerable: undefined })], ['invalid CPE range boundary', cpeCandidateHit({ versionEndExcluding: 2 })], [ diff --git a/utils/searchPerformance.js b/utils/searchPerformance.js index 3e365ae..1b5b1b5 100644 --- a/utils/searchPerformance.js +++ b/utils/searchPerformance.js @@ -4,6 +4,15 @@ const searchProvider = require('./searchProvider'); const EVENT_LOOP_DELAY_RESOLUTION_MS = 20; const EVENT_LOOP_DELAY_WINDOW_MS = 10000; const RESPONSE_JSON_INSTRUMENTED = Symbol('responseJsonInstrumented'); +const SHARED_OPERATION_METRIC_FIELDS = [ + 'coldSearchQueueWaitElapsedMs', + 'coldSearchQueueState', + 'openSearchElapsedMs', + 'openSearchTookMs', + 'pageFetchElapsedMs', + 'pageFetchTookMs', + 'postFilterElapsedMs' +]; let eventLoopDelaySampler; function markTime() { @@ -18,6 +27,50 @@ function roundMs(value) { return value === undefined ? undefined : Math.round(value * 100) / 100; } +async function captureOperationOutcome(run) { + const operationMetrics = {}; + + try { + return { + status: 'fulfilled', + result: await run(operationMetrics), + operationMetrics + }; + } catch (error) { + return { + status: 'rejected', + error, + operationMetrics + }; + } +} + +function mergeOperationMetrics(metrics, operationMetrics) { + for (const field of SHARED_OPERATION_METRIC_FIELDS) { + if (operationMetrics?.[field] !== undefined) { + metrics[field] = operationMetrics[field]; + } + } + + if (operationMetrics?.openSearchOperations?.length) { + metrics.openSearchOperations ??= []; + metrics.openSearchOperations.push(...operationMetrics.openSearchOperations); + } + if (Number.isFinite(operationMetrics?.maxOpenSearchActiveRequests)) { + metrics.maxOpenSearchActiveRequests = Math.max( + metrics.maxOpenSearchActiveRequests ?? 0, + operationMetrics.maxOpenSearchActiveRequests + ); + } +} + +function consumeOperationOutcome(outcome, metrics) { + mergeOperationMetrics(metrics, outcome.operationMetrics); + if (outcome.status === 'rejected') throw outcome.error; + + return outcome.result; +} + function getCacheStatus(metrics, postFilters) { if (!Object.keys(postFilters ?? {}).length) return 'not_applicable'; if (metrics.cacheHit === true) return 'hit'; @@ -102,24 +155,33 @@ function instrumentJsonResponse(res, metrics) { res[RESPONSE_JSON_INSTRUMENTED] = true; res.json = function instrumentedJson(body) { const serializationStart = markTime(); - const result = originalJson.call(this, body); - metrics.responseSerializationElapsedMs = elapsedMs(serializationStart); + try { + return originalJson.call(this, body); + } catch (err) { + if (err?.code === 'SEARCH_RESPONSE_TOO_LARGE') { + metrics.attemptedResponseSerializationElapsedMs = elapsedMs(serializationStart); + } + throw err; + } finally { + metrics.responseSerializationElapsedMs = elapsedMs(serializationStart); - const contentLength = Number(this.getHeader?.('content-length')); - if (Number.isFinite(contentLength) && contentLength >= 0) { - metrics.responseBytes = contentLength; + const contentLength = Number(this.getHeader?.('content-length')); + if (Number.isFinite(contentLength) && contentLength >= 0) { + metrics.responseBytes = contentLength; + } } - - return result; }; } function beginSearchRequest(metrics, res) { - if (!searchProvider.shouldLogSearchPerformance()) return false; + const collectDetailedMetrics = searchProvider.shouldLogSearchPerformance(); + if (collectDetailedMetrics || typeof res?.send === 'function') { + instrumentJsonResponse(res, metrics); + } + if (!collectDetailedMetrics) return false; metrics.heapUsedStartBytes = process.memoryUsage().heapUsed; getEventLoopDelaySampler(); - instrumentJsonResponse(res, metrics); return true; } @@ -159,6 +221,10 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte inFlightState: metrics.inFlightState, coldSearchQueueWaitElapsedMs: roundMs(metrics.coldSearchQueueWaitElapsedMs), coldSearchQueueState: metrics.coldSearchQueueState, + attemptedResponseBytes: metrics.attemptedResponseBytes, + maxResponseBytes: metrics.maxResponseBytes, + attemptedResponseSerializationElapsedMs: roundMs(metrics.attemptedResponseSerializationElapsedMs), + responseSerializationElapsedMs: roundMs(metrics.responseSerializationElapsedMs), totalElapsedMs: roundMs(metrics.totalElapsedMs) }; @@ -179,7 +245,6 @@ function logSearchPerformance(metrics, searchOptions, responseOptions, postFilte cachedResultCveIdCount: metrics.cachedResultCveIdCount, cacheState: metrics.cacheState, responseBytes: metrics.responseBytes, - responseSerializationElapsedMs: roundMs(metrics.responseSerializationElapsedMs), heapUsedBytes: metrics.heapUsedBytes, heapUsedDeltaBytes: metrics.heapUsedDeltaBytes, processEventLoopDelayMeanMs: roundMs(metrics.processEventLoopDelayMeanMs), @@ -209,12 +274,15 @@ function resetForTests() { module.exports = { beginSearchRequest, + captureOperationOutcome, completeSearchRequest, + consumeOperationOutcome, createEventLoopDelaySampler, elapsedMs, getCacheStatus, logSearchPerformance, markTime, + mergeOperationMetrics, resetForTests, roundMs }; diff --git a/utils/searchPerformance.test.js b/utils/searchPerformance.test.js index 8ea90c3..37f470d 100644 --- a/utils/searchPerformance.test.js +++ b/utils/searchPerformance.test.js @@ -133,6 +133,54 @@ describe('searchPerformance', () => { sampler.stop(); }); + test('captures and merges fulfilled shared-operation metrics without copying request state', async () => { + const outcome = await searchPerformance.captureOperationOutcome(async (operationMetrics) => { + operationMetrics.cacheHit = true; + operationMetrics.openSearchOperations = ['cpe-candidate']; + operationMetrics.maxOpenSearchActiveRequests = 3; + operationMetrics.openSearchElapsedMs = 12.5; + operationMetrics.postFilterElapsedMs = 2.5; + return 'candidate-result'; + }); + const metrics = { + cacheHit: false, + openSearchOperations: ['standard'], + maxOpenSearchActiveRequests: 2 + }; + + expect(searchPerformance.consumeOperationOutcome(outcome, metrics)).toBe('candidate-result'); + expect(metrics).toStrictEqual({ + cacheHit: false, + openSearchOperations: ['standard', 'cpe-candidate'], + maxOpenSearchActiveRequests: 3, + openSearchElapsedMs: 12.5, + postFilterElapsedMs: 2.5 + }); + }); + + test('preserves the original shared-operation error after merging available metrics', async () => { + const error = new Error('candidate failed'); + const outcome = await searchPerformance.captureOperationOutcome(async (operationMetrics) => { + operationMetrics.openSearchOperations = ['cpe-candidate']; + operationMetrics.openSearchElapsedMs = 4.5; + throw error; + }); + const metrics = {}; + let caught; + + try { + searchPerformance.consumeOperationOutcome(outcome, metrics); + } catch (err) { + caught = err; + } + + expect(caught).toBe(error); + expect(metrics).toStrictEqual({ + openSearchOperations: ['cpe-candidate'], + openSearchElapsedMs: 4.5 + }); + }); + test('logSearchPerformance emits redacted search timing metadata when enabled', () => { mockShouldLogSearchPerformance.mockReturnValue(true); @@ -263,17 +311,21 @@ describe('searchPerformance', () => { expect(console.log).not.toHaveBeenCalled(); }); - test('does not instrument response or process resources when detailed logging is disabled', () => { + test('captures serialization timing but not process resources when detailed logging is disabled', () => { mockShouldLogSearchPerformance.mockReturnValue(false); const metrics = {}; - const json = jest.fn(); - const res = { json }; + const res = { + getHeader: jest.fn(() => undefined), + json: jest.fn(() => res), + send: jest.fn(() => res) + }; expect(searchPerformance.beginSearchRequest(metrics, res)).toBe(false); + res.json({ status: 'ok' }); searchPerformance.completeSearchRequest(metrics); - expect(res.json).toBe(json); - expect(metrics).toStrictEqual({}); + expect(metrics.responseSerializationElapsedMs).toEqual(expect.any(Number)); + expect(metrics.heapUsedBytes).toBeUndefined(); }); test('captures response size and process resource metrics only when detailed logging is enabled', () => { diff --git a/utils/searchProvider.js b/utils/searchProvider.js index e7ffe6a..3009b28 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -327,6 +327,10 @@ function getSearchRequestDeadlineMs() { return runtimeConfig.getRuntimeConfig().search.requestDeadlineMs; } +function getSearchMaxResponseBytes() { + return runtimeConfig.getRuntimeConfig().search.maxResponseBytes; +} + function resetSearchProviderForTests() { basicSearchManager = undefined; searchReader = undefined; @@ -350,6 +354,7 @@ module.exports = { getPostFilterColdSearchMaxWaitMs, getPostFilterEvaluationChunkSize, getOpenSearchRequestTimeoutMs, + getSearchMaxResponseBytes, getSearchRequestDeadlineMs, getSearchProviderConfig, getSearchReader, diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index 78616c5..922cfac 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -736,6 +736,7 @@ describe('searchProvider', () => { 'search.openSearchRequestTimeoutMs', 'search.requestDeadlineMs', 'search.readinessTimeoutMs', + 'search.maxResponseBytes', 'runtime.shutdownTimeoutMs' ].includes(key) ); @@ -751,6 +752,7 @@ describe('searchProvider', () => { 'search.openSearchRequestTimeoutMs': '25000', 'search.requestDeadlineMs': '45000', 'search.readinessTimeoutMs': '7000', + 'search.maxResponseBytes': '20971520', 'runtime.shutdownTimeoutMs': '60000' }; return values[key]; @@ -766,6 +768,7 @@ describe('searchProvider', () => { expect(searchProvider.getOpenSearchRequestTimeoutMs()).toBe(25000); expect(searchProvider.getSearchRequestDeadlineMs()).toBe(45000); expect(searchProvider.getOpenSearchReadinessTimeoutMs()).toBe(7000); + expect(searchProvider.getSearchMaxResponseBytes()).toBe(20971520); }); test('rejects an invalid explicit CPE post-filter runtime value', () => { diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index f30edc2..9104654 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -1,6 +1,7 @@ const constants = require('./constants'); const cpe = require('./cpe'); const cveId = require('./cveId'); +const cweId = require('./cweId'); const cvss = require('./cvss'); const { MAX_RESULTS_LIMIT, RESULTS_PER_PAGE } = constants; @@ -362,6 +363,13 @@ function getFilterFields(params) { return; } + if (param === 'cweId') { + if (!cweId.isValidCweId(params[param])) throwInvalidQueryValue(param); + + fieldsToReturn.push(buildMatchPhraseAcrossFields(cweId.normalizeCweId(params[param]), fieldSpec)); + return; + } + if (param === 'cpeName') { const filter = buildCpeNameFilter(params[param]); if (filter) fieldsToReturn.push(filter); diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index 79f9c2c..bb04d01 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -238,6 +238,7 @@ describe('searchQueryBuilder buildSearchRequest()', () => { test.each([ ['cpeName', { cpeName: 'not-a-cpe-name' }, 'Invalid cpeName value reached query construction.'], + ['cweId', { cweId: 'CWE-079' }, 'Invalid cweId value reached query construction.'], [ 'virtualMatchString', { virtualMatchString: 'not-a-cpe-match-string' }, @@ -387,7 +388,7 @@ describe('searchQueryBuilder buildSearchRequest()', () => { test('buildOpenSearchBody builds the complete standard search request', () => { const output = queryBuilder.buildSearchRequest({ keywordSearch: 'remote code', - cweId: 'CWE-79', + cweId: ' cwe-79 ', resultsPerPage: 10, pageNumber: 3 }); diff --git a/utils/searchResponseLimiter.js b/utils/searchResponseLimiter.js new file mode 100644 index 0000000..a56da95 --- /dev/null +++ b/utils/searchResponseLimiter.js @@ -0,0 +1,61 @@ +const RESPONSE_SEND_INSTRUMENTED = Symbol('responseSendInstrumented'); + +class SearchResponseTooLargeError extends Error { + constructor(responseBytes, maxResponseBytes) { + super('Search response exceeds the configured maximum response size.'); + this.name = 'SearchResponseTooLargeError'; + this.code = 'SEARCH_RESPONSE_TOO_LARGE'; + this.responseBytes = responseBytes; + this.maxResponseBytes = maxResponseBytes; + } +} + +function getBodyByteLength(body) { + if (Buffer.isBuffer(body)) return body.length; + if (typeof body === 'string') return Buffer.byteLength(body); + + return undefined; +} + +function isSuccessfulResponse(res) { + return res.statusCode >= 200 && res.statusCode < 300; +} + +function instrumentSearchResponse(res, { maxResponseBytes, metrics, checkDeadline }) { + if (typeof res?.send !== 'function' || res[RESPONSE_SEND_INSTRUMENTED]) return; + + const originalSend = res.send; + res[RESPONSE_SEND_INSTRUMENTED] = true; + res.send = function limitedSearchResponse(body) { + if (!isSuccessfulResponse(this)) { + return originalSend.call(this, body); + } + + const responseBytes = getBodyByteLength(body); + if (responseBytes === undefined) { + return originalSend.call(this, body); + } + + checkDeadline(); + + if (responseBytes > maxResponseBytes) { + metrics.attemptedResponseBytes = responseBytes; + metrics.maxResponseBytes = maxResponseBytes; + throw new SearchResponseTooLargeError(responseBytes, maxResponseBytes); + } + + metrics.responseBytes = responseBytes; + return originalSend.call(this, body); + }; +} + +function isSearchResponseTooLargeError(err) { + return err instanceof SearchResponseTooLargeError || err?.code === 'SEARCH_RESPONSE_TOO_LARGE'; +} + +module.exports = { + getBodyByteLength, + instrumentSearchResponse, + isSearchResponseTooLargeError, + SearchResponseTooLargeError +}; diff --git a/utils/searchResponseLimiter.test.js b/utils/searchResponseLimiter.test.js new file mode 100644 index 0000000..b738639 --- /dev/null +++ b/utils/searchResponseLimiter.test.js @@ -0,0 +1,92 @@ +const searchResponseLimiter = require('./searchResponseLimiter'); + +function createResponse(statusCode = 200) { + const originalSend = jest.fn(() => 'sent'); + + return { + originalSend, + res: { + statusCode, + send: originalSend + } + }; +} + +describe('searchResponseLimiter', () => { + test('allows a successful response at the byte limit after checking the deadline', () => { + const { originalSend, res } = createResponse(); + const checkDeadline = jest.fn(); + const metrics = {}; + searchResponseLimiter.instrumentSearchResponse(res, { + maxResponseBytes: 4, + metrics, + checkDeadline + }); + + expect(res.send('test')).toBe('sent'); + expect(checkDeadline).toHaveBeenCalledTimes(1); + expect(originalSend).toHaveBeenCalledWith('test'); + expect(metrics.responseBytes).toBe(4); + }); + + test('counts UTF-8 bytes and rejects an oversized success before sending it', () => { + const { originalSend, res } = createResponse(); + const checkDeadline = jest.fn(); + const metrics = {}; + searchResponseLimiter.instrumentSearchResponse(res, { + maxResponseBytes: 3, + metrics, + checkDeadline + }); + + let thrownError; + try { + res.send('\u00e9\u00e9'); + } catch (err) { + thrownError = err; + } + + expect(thrownError).toMatchObject({ + code: 'SEARCH_RESPONSE_TOO_LARGE', + responseBytes: 4, + maxResponseBytes: 3 + }); + expect(checkDeadline).toHaveBeenCalledTimes(1); + expect(originalSend).not.toHaveBeenCalled(); + expect(metrics).toMatchObject({ + attemptedResponseBytes: 4, + maxResponseBytes: 3 + }); + }); + + test('does not limit or deadline-check an error response', () => { + const { originalSend, res } = createResponse(422); + const checkDeadline = jest.fn(); + const metrics = {}; + searchResponseLimiter.instrumentSearchResponse(res, { + maxResponseBytes: 1, + metrics, + checkDeadline + }); + + expect(res.send('complete error body')).toBe('sent'); + expect(checkDeadline).not.toHaveBeenCalled(); + expect(originalSend).toHaveBeenCalledWith('complete error body'); + expect(metrics).toStrictEqual({}); + }); + + test('propagates a post-serialization deadline failure before sending', () => { + const { originalSend, res } = createResponse(); + const deadlineError = Object.assign(new Error('deadline'), { code: 'SEARCH_TIMEOUT' }); + searchResponseLimiter.instrumentSearchResponse(res, { + maxResponseBytes: 100, + metrics: {}, + checkDeadline: jest.fn(() => { + throw deadlineError; + }) + }); + + expect(() => res.send('test')).toThrow(deadlineError); + expect(originalSend).not.toHaveBeenCalled(); + }); +});