Add default-on PIT traversal for semantic searches

.env.example

@@ -5,6 +5,10 @@ OpenSearchDomainEndpoint=
 SearchOpenSearchRequestTimeoutMs=20000
 SearchRequestDeadlineMs=30000
 SearchReadinessTimeoutMs=5000
+# PIT traversal is enabled by default; set false to restore bounded semantic searches.
+SearchPitEnabled=true
+SearchPitBatchSize=1000
+SearchPitKeepAliveMs=60000
 RequestLogSuccessSampleRate=0.1
 RequestLogHealthChecks=false
 WebSearchEnabled=true

api-docs/openapi.json

@@ -21,7 +21,7 @@
           "CVE Record"
         ],
         "summary": "Retrieves all matching CVE records after applying the JSON request body as filters",
-        "description": "  <h2>Expected Behavior</h2>  <p>Retrieves all matching CVE records after applying the JSON request body as filters</p>  <h2>Result Counts and Pagination</h2>  <p>Direct searches report the exact number of matching records, including totals greater than 10,000, but requested pages must remain within the first 10,000 results.</p>  <p>CVSS vector filters, SSVC filters, date-bounded positive knownExploited searches, cpeName, and virtualMatchString use semantic post-filtering. These searches evaluate at most 10,000 search candidates, so their reported totals and pages include only matches found within that candidate set. A warning is returned when that limit is reached.</p>",
+        "description": "  <h2>Expected Behavior</h2>  <p>Retrieves all matching CVE records after applying the JSON request body as filters</p>  <h2>Result Counts and Pagination</h2>  <p>Direct searches report the exact number of matching records, including totals greater than 10,000, but requested pages must remain within the first 10,000 results.</p>  <p>CVSS vector filters, SSVC filters, date-bounded positive knownExploited searches, cpeName, and virtualMatchString use semantic post-filtering. PIT is enabled by default on the server. With PIT explicitly disabled using SearchPitEnabled=false, these searches evaluate at most 10,000 search candidates, so their reported totals and pages include only matches found within that candidate set. Reaching that limit returns SEMANTIC_CANDIDATE_LIMIT_REACHED and pagination.totalIsExact=false.</p>  <p>With point-in-time (PIT) traversal enabled, semantic searches evaluate all candidate batches before returning exact totals and pagination.totalIsExact=true. Requested pages must still fit entirely within the first 10,000 matching results. Filling the requested page does not stop the scan because the total must also be computed. PIT is a server-side setting, not a request field; clients continue to use pageNumber and resultsPerPage, with no public cursor.</p>  <p>Each PIT semantic request uses one snapshot for its candidates and returned records. Separate page requests use separate snapshots, so changes between requests can still shift results. Incomplete traversal returns a sanitized error, not partial results, and does not restart or fall back to a bounded search. Warnings remain before records in successful responses. Direct searches and /webSearch do not use PIT.</p>",
         "operationId": "search",
         "responses": {
           "200": {
@@ -83,7 +83,7 @@
             }
           },
           "502": {
-            "description": "The search backend returned an invalid or unexpected response",
+            "description": "UPSTREAM_SEARCH_ERROR: The search backend returned an invalid or unexpected response. This includes malformed or incomplete PIT batches or full-page responses; no partial PIT result is returned.",
             "content": {
               "application/json": {
                 "schema": {
@@ -93,7 +93,7 @@
             }
           },
           "503": {
-            "description": "The search service is temporarily unavailable",
+            "description": "SEARCH_UNAVAILABLE: The search service is temporarily unavailable, including a missing or expired PIT search context. PIT traversal is not silently restarted. Retry-After specifies the suggested delay.",
             "content": {
               "application/json": {
                 "schema": {
@@ -111,7 +111,7 @@
             }
           },
           "504": {
-            "description": "Search exceeded its processing deadline",
+            "description": "SEARCH_TIMEOUT: Search exceeded its operation timeout or overall processing deadline, or a PIT search reported a timeout. No partial PIT result is returned. PIT cleanup has a separate bounded timeout and can add up to five seconds before the error response.",
             "content": {
               "application/json": {
                 "schema": {
@@ -275,7 +275,7 @@
           "CVE Record"
         ],
         "summary": "Raw query search",
-        "description": "  <h2>Expected Behavior</h2>  <p>Retrieves matching CVE records using the raw query expression supplied in the JSON request body. Service operators can disable this endpoint through runtime configuration.</p>",
+        "description": "  <h2>Expected Behavior</h2>  <p>Retrieves matching CVE records using the raw query expression supplied in the JSON request body. Service operators can disable this endpoint through runtime configuration. Standalone dates and date ranges search the supported record/container date fields; other expressions use raw query syntax with an AND default operator.</p>",
         "operationId": "webSearch",
         "responses": {
           "200": {
@@ -380,6 +380,7 @@
                 "properties": {
                   "query": {
                     "type": "string",
+                    "description": "Raw search expression. Examples: remote code execution, apache*, 2024-01-01, or 2024-01-01..2024-01-31. Date ranges are inclusive and allow year/month endpoints; standalone years/months and quoted dates remain text queries. Date-query years must be between 1900 and 2100.",
                     "minLength": 1
                   },
                   "sort": {
@@ -455,6 +456,7 @@
       },
       "SearchWarning": {
         "type": "object",
+        "description": "Non-fatal search interpretation information. SEMANTIC_CANDIDATE_LIMIT_REACHED applies only to bounded semantic searches with PIT disabled. BROAD_VIRTUAL_MATCH_STRING can occur in either mode; its message describes the candidate limit when PIT is disabled and increased processing time when PIT is enabled.",
         "additionalProperties": false,
         "required": [
           "code",
@@ -526,16 +528,16 @@
           "totalResults": {
             "type": "integer",
             "minimum": 0,
-            "description": "For direct searches, the exact number of matching records, which may be greater than 10,000. For semantic post-filtered searches, the number of matches found within the bounded 10,000-candidate set."
+            "description": "For direct searches, the exact number of matching records, which may be greater than 10,000. For semantic searches with PIT enabled (the default), the exact number of matches across the completed snapshot traversal. With PIT explicitly disabled, the number of matches found within the bounded 10,000-candidate set. Separate page requests use separate snapshots."
           },
           "totalPages": {
             "type": "integer",
             "minimum": 0,
-            "description": "Pages derived from totalResults and pageSize. Semantic post-filtered searches derive this value from matches found within the bounded 10,000-candidate set."
+            "description": "Pages derived from totalResults and pageSize, using the same bounded or exact total as totalResults. This value can exceed the number of accessible pages: each requested page must still fit entirely within the first 10,000 matching results in either mode."
           },
           "totalIsExact": {
             "type": "boolean",
-            "description": "True when totalResults represents every match. False when a semantic search reaches the 10,000-candidate limit and additional matches may exist."
+            "description": "True for direct searches and successful, fully completed PIT semantic searches (the default mode). With PIT explicitly disabled, false when a semantic search reaches the 10,000-candidate limit and additional matches may exist; true otherwise. A failed PIT traversal returns an error rather than a success with a partial total."
           }
         }
       },
@@ -570,7 +572,7 @@
           },
           "warnings": {
             "type": "array",
-            "description": "Non-fatal conditions that may affect how the results should be interpreted.",
+            "description": "Non-fatal conditions that may affect how the results should be interpreted. This field is emitted before records. Successful PIT traversal does not return a semantic candidate-limit warning, but other interpretation warnings still apply.",
             "items": {
               "$ref": "#/components/schemas/SearchWarning"
             }
@@ -598,6 +600,7 @@
           "searchMetadata": {
             "type": "object",
             "additionalProperties": true,
+            "description": "API-owned raw-query metadata. Includes the response-format version, not dependency package versions.",
             "required": [
               "searchStatus",
               "version",
@@ -608,7 +611,18 @@
                 "type": "string"
               },
               "version": {
-                "type": "string"
+                "type": "string",
+                "description": "Version of the raw-query response format."
+              },
+              "notes": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "queryTime_ms": {
+                "type": "number",
+                "description": "Query processing duration when available."
               },
               "errors": {
                 "type": "array",
@@ -633,7 +647,7 @@
       },
       "SearchRequest": {
         "type": "object",
-        "description": "Optional filters and pagination controls for a CVE record search. Enum-valued fields are accepted case-insensitively and normalized to the documented canonical values. When both boundaries of a date range are supplied, the start must be before or equal to the end. Direct searches report exact totals even when they exceed 10,000, although requested pages must remain inside that window. CVSS vector filters, SSVC filters, date-bounded positive knownExploited searches, cpeName, and virtualMatchString use semantic post-filtering. They evaluate at most 10,000 search candidates, so reported totals and pages include only matches found within that candidate set.",
+        "description": "Optional filters and pagination controls for a CVE record search. Enum-valued fields are accepted case-insensitively and normalized to the documented canonical values. When both boundaries of a date range are supplied, the start must be before or equal to the end. Direct searches report exact totals even when they exceed 10,000, although requested pages must remain inside that window. CVSS vector filters, SSVC filters, date-bounded positive knownExploited searches, cpeName, and virtualMatchString use semantic post-filtering. PIT is enabled by default on the server. All candidate batches must complete before a successful response reports exact totals; traversal failures return an error instead of partial results. With PIT explicitly disabled using SearchPitEnabled=false, these searches evaluate at most 10,000 search candidates, so reported totals and pages include only matches found within that candidate set. Reaching that limit returns SEMANTIC_CANDIDATE_LIMIT_REACHED and totalIsExact=false. Requested pages must still remain within the first 10,000 matching results. PIT is a server-side setting, not a request field.",
         "additionalProperties": false,
         "properties": {
           "scope": {

api-docs/openapi.test.js

@@ -165,6 +165,23 @@ describe('OpenAPI contract', () => {
         expect(operation.responses['410'].description).toContain('disabled by service configuration');
     });
 
+    test('distinguishes bounded semantic searches from internal PIT traversal without adding client controls', () => {
+        const operation = openapi.paths['/search'].post;
+        const request = jsonRequestBodySchema('/search', 'post');
+        for (const description of [operation.description, request.description]) {
+            expect(description).toContain('PIT is enabled by default on the server');
+            expect(description).toContain('With PIT explicitly disabled using SearchPitEnabled=false');
+            expect(description).toContain('SEMANTIC_CANDIDATE_LIMIT_REACHED');
+            expect(description).toContain('first 10,000 matching results');
+            expect(description).toContain('PIT is a server-side setting, not a request field');
+        }
+        expect(operation.description).toContain('Separate page requests use separate snapshots');
+        expect(operation.description).toContain('does not restart or fall back');
+        expect(operation.description).toContain('no public cursor');
+        expect(operation.description).toContain('Filling the requested page does not stop the scan');
+        expect(openapi.components.schemas.SearchWarning.description).toContain('can occur in either mode');
+    });
+
     test('documents cpeName as a CPE Match Criteria filter with required components', () => {
         const cpeName = openapi.components.schemas.SearchRequest.properties.cpeName;
 
@@ -193,7 +210,11 @@ describe('OpenAPI contract', () => {
         expect(responses['400'].description).toContain('Malformed JSON');
         expect(responses['415'].description).toContain('application/json');
         expect(responses['502'].description).toContain('invalid or unexpected response');
+        expect(responses['502'].description).toContain('no partial PIT result');
         expect(responses['503'].description).toContain('search service');
+        expect(responses['503'].description).toContain('missing or expired PIT search context');
+        expect(responses['504'].description).toContain('SEARCH_TIMEOUT');
+        expect(responses['504'].description).toContain('separate bounded timeout');
         expect(responses['503'].headers['Retry-After']).toStrictEqual({
             description: 'Seconds before retrying the unavailable search service',
             schema: {
@@ -244,7 +265,11 @@ describe('OpenAPI contract', () => {
             'exact number of matching records'
         );
         expect(schemas.SearchPagination.properties.totalResults.description).toContain('10,000-candidate set');
+        expect(schemas.SearchPagination.properties.totalResults.description).toContain('completed snapshot traversal');
+        expect(schemas.SearchPagination.properties.totalPages.description).toContain('number of accessible pages');
         expect(schemas.SearchPagination.properties.totalIsExact.description).toContain('additional matches may exist');
+        expect(schemas.SearchPagination.properties.totalIsExact.description).toContain('error rather than a success');
+        expect(searchResponse.properties.warnings.description).toContain('emitted before records');
         expect(schemas.CveRecord.description).toContain('without search-backend transport metadata');
         expect(schemas.ApiError.required).toStrictEqual(['error', 'message']);
         expect(schemas.ApiErrorResponse.oneOf).toHaveLength(2);

config/custom-environment-variables.jsonc

@@ -1,9 +1,6 @@
 {
-    // this file is used by node-config to map a node-config (AppConfig)
-    //  hierarchy of constants to an environment variable
-    // Note that much of the environment variables mapped here existed for some time
-    //  without AppConfig, this file bridges the historical uses of those with the new
-    //  as we transition to AppConfig
+    // Map appConfig settings to environment variables. utils/appConfig.js loads .env
+    // before node-config resolves these overrides; existing process variables take precedence.
     "appConfig": {
         // constants for search capability
         "search": {
@@ -13,22 +10,17 @@
             "allowUnknownSslCerts": "OpenSearchAllowUnknownSslCerts",
             "openSearchRequestTimeoutMs": "SearchOpenSearchRequestTimeoutMs",
             "requestDeadlineMs": "SearchRequestDeadlineMs",
-            "readinessTimeoutMs": "SearchReadinessTimeoutMs"
+            "readinessTimeoutMs": "SearchReadinessTimeoutMs",
+            "pitBatchSize": "SearchPitBatchSize",
+            "pitKeepAliveMs": "SearchPitKeepAliveMs"
         },
         "requestLogging": {
             "successSampleRate": "RequestLogSuccessSampleRate",
             "healthChecks": "RequestLogHealthChecks"
         },
         "features": {
-            "webSearchEnabled": "WebSearchEnabled"
-        },
-        // constants for testing node-config
-        // these values are only used to test node-config in AppConfig.test.int.ts
-        // DO NOT USE THIS FOR ANYTHING ELSE
-        "test": {
-            "appConfigTest": {
-                "test": "JEST_env_config_test"
-            }
+            "webSearchEnabled": "WebSearchEnabled",
+            "searchPitEnabled": "SearchPitEnabled"
         }
     }
 }

config/default.jsonc

@@ -3,8 +3,8 @@
     //  These values are overridable using other *.jsonc (e.g., prod.jsonc)
     //    as well as using environment variables (e.g., in `.env`)
     //    - Each configuration is mapped using custom-environment-variables.jsonc to enable environment variable overrides.
-    //    - For more information, see cve-core/src/adapters/config/AppConfig.ts
-    //  NOTE for consistency, all values need to be strings for proper type when using .env overrides
+    //    - utils/appConfig.js loads .env before reading these files through node-config.
+    //  Values are normalized by their consumers because environment overrides are strings.
     "appConfig": {
         // constants for search capability
         "search": {
@@ -16,7 +16,10 @@
             // Limits total processing time for a /search request.
             "requestDeadlineMs": "30000",
             // Overall /api/ready-check deadline and per-operation timeout for field capabilities and search.
-            "readinessTimeoutMs": "5000"
+            "readinessTimeoutMs": "5000",
+            // Semantic PIT traversal settings; validate capacity for each deployment.
+            "pitBatchSize": "1000",
+            "pitKeepAliveMs": "60000"
         },
         // Access logging settings. Error responses are always logged.
         "requestLogging": {
@@ -25,10 +28,12 @@
             // Set to TRUE to include successful health checks in the same successSampleRate sampling.
             "healthChecks": "FALSE"
         },
-        // Feature flags used for operator-controlled endpoint availability.
+        // Operator-controlled feature flags.
         "features": {
             // Set to FALSE only when an environment must disable POST /webSearch.
-            "webSearchEnabled": "TRUE"
+            "webSearchEnabled": "TRUE",
+            // Set to FALSE to restore bounded semantic searches without PIT.
+            "searchPitEnabled": "TRUE"
         }
     }
 }