From acf465e9f63dee503b69fba170ce6b1e1b7755cc Mon Sep 17 00:00:00 2001 From: Tarek Etman Date: Tue, 17 Nov 2026 18:55:56 +0000 Subject: [PATCH] feat: Add SpecStream as a search provider --- src/api/schemas/index.ts | 2 +- src/services/search.service.spec.ts | 8 +++- src/services/search.service.ts | 57 ++++++++++++++++++++++++++--- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/src/api/schemas/index.ts b/src/api/schemas/index.ts index fd67433..4404447 100644 --- a/src/api/schemas/index.ts +++ b/src/api/schemas/index.ts @@ -191,7 +191,7 @@ export const mapClearCacheSchema = z.object({ export const searchRequestSchema = z.object({ query: z.string().min(1).max(500), limit: z.number().int().positive().max(50).optional(), - provider: z.enum(['duckduckgo', 'searxng', 'serper']).optional(), + provider: z.enum(['duckduckgo', 'searxng', 'serper', 'specstream']).optional(), lang: z.string().max(10).optional(), // When true, scrape each result and attach markdown content. scrapeResults: z.boolean().optional(), diff --git a/src/services/search.service.spec.ts b/src/services/search.service.spec.ts index b37da64..3453333 100644 --- a/src/services/search.service.spec.ts +++ b/src/services/search.service.spec.ts @@ -172,6 +172,11 @@ describe('searchWeb', () => { await expect(searchWeb(' ')).resolves.toEqual([]); }); + it('throws when the \'specstream\' provider is selected but SPECSTREAM_API_KEY is unset', async () => { + delete process.env.SPECSTREAM_API_KEY; + await expect(searchWeb('typescript', { provider: 'specstream' })).rejects.toThrow(/SPECSTREAM_API_KEY/); + }); + it('throws when the \'searxng\' provider is selected but SEARXNG_URL is unset', async () => { delete process.env.SEARXNG_URL; await expect(searchWeb('typescript', { provider: 'searxng' })).rejects.toThrow(/SEARXNG_URL/); @@ -202,8 +207,9 @@ describe('runSearch diagnostics (why a search came back empty)', () => { expect(hint).toMatch(/SEARXNG_URL/); }); - it('gives provider-appropriate empty hints for searxng and serper', () => { + it('gives provider-appropriate empty hints for searxng, serper, and specstream', () => { expect(emptyResultHint('searxng')).toMatch(/SearXNG/); expect(emptyResultHint('serper')).toMatch(/Serper/); + expect(emptyResultHint('specstream')).toMatch(/SpecStream/); }); }); diff --git a/src/services/search.service.ts b/src/services/search.service.ts index b4a32b0..7ddf7fa 100644 --- a/src/services/search.service.ts +++ b/src/services/search.service.ts @@ -42,7 +42,7 @@ export interface SearchOptions { /** Maximum number of results to return (default {@link DEFAULT_LIMIT}). */ limit?: number; /** Which search backend to use. Default: auto (serper if SERPER_API_KEY, else searxng if SEARXNG_URL, else duckduckgo). */ - provider?: 'duckduckgo' | 'searxng' | 'serper'; + provider?: 'duckduckgo' | 'searxng' | 'serper' | 'specstream'; /** * Preferred result language/region. Mapped to DuckDuckGo's `kl` region param * (e.g. `us-en`) and to SearXNG's `language` param (e.g. `en`). @@ -76,7 +76,7 @@ const DESKTOP_USER_AGENT = /** Result of a search, including which provider ran and — when empty — why. */ export interface SearchOutcome { results: SearchResult[]; - provider: 'duckduckgo' | 'searxng' | 'serper'; + provider: 'duckduckgo' | 'searxng' | 'serper' | 'specstream'; /** * Set when `results` is empty for a reason the caller can act on — e.g. the * keyless DuckDuckGo endpoint was anti-bot-challenged, or a provider request @@ -118,6 +118,9 @@ export async function runSearch(query: string, opts: SearchOptions = {}): Promis case 'serper': raw = await fetchSerper(trimmedQuery, limit, opts); break; + case 'specstream': + raw = await fetchSpecStream(trimmedQuery, limit, opts); + break; case 'searxng': // NOTE: a missing SEARXNG_URL throws (config error); request failures do not. raw = await fetchSearxng(trimmedQuery, limit, opts); @@ -147,7 +150,7 @@ export async function searchWeb(query: string, opts: SearchOptions = {}): Promis * DuckDuckGo path this is almost always an anti-bot challenge on server/datacenter * IPs rather than a genuine no-hit query. */ -export function emptyResultHint(provider: 'duckduckgo' | 'searxng' | 'serper'): string { +export function emptyResultHint(provider: 'duckduckgo' | 'searxng' | 'serper' | 'specstream'): string { switch (provider) { case 'duckduckgo': return 'No results from the keyless DuckDuckGo endpoint. It anti-bot-challenges ' + @@ -156,8 +159,11 @@ export function emptyResultHint(provider: 'duckduckgo' | 'searxng' | 'serper'): case 'searxng': return 'No results from the configured SearXNG instance for this query.'; case 'serper': - default: return 'No results from Serper for this query.'; + case 'specstream': + return 'No results from SpecStream for this query.'; + default: + return 'No results for this query.'; } } @@ -336,7 +342,8 @@ async function fetchDuckDuckGo(query: string, limit: number, opts: SearchOptions * logged and yield `[]` instead of throwing. */ /** Pick the most reliable configured provider when the caller didn't specify one. */ -function autoProvider(): 'serper' | 'searxng' | 'duckduckgo' { +function autoProvider(): 'serper' | 'searxng' | 'duckduckgo' | 'specstream' { + if (process.env.SPECSTREAM_API_KEY && process.env.SPECSTREAM_API_KEY.trim()) return 'specstream'; if (process.env.SERPER_API_KEY && process.env.SERPER_API_KEY.trim()) return 'serper'; if (process.env.SEARXNG_URL && process.env.SEARXNG_URL.trim()) return 'searxng'; return 'duckduckgo'; @@ -382,6 +389,46 @@ async function fetchSerper(query: string, limit: number, opts: SearchOptions): P } } +/** + * SpecStream — high-performance search API (specstream.ai). Set + * SPECSTREAM_API_KEY. POST https://api.specstream.ai/v1/web-search with {q, num}. + */ +async function fetchSpecStream(query: string, limit: number, opts: SearchOptions): Promise { + const apiKey = process.env.SPECSTREAM_API_KEY; + if (!apiKey || !apiKey.trim()) { + throw new Error('searchWeb: provider \'specstream\' selected but SPECSTREAM_API_KEY is not set.'); + } + try { + const body: Record = { q: query, num: Math.min(limit, 100) }; + if (opts.lang) body.gl = opts.lang; + const response = await axios.post('https://api.specstream.ai/v1/web-search', body, { + timeout: REQUEST_TIMEOUT_MS, + headers: { 'X-API-KEY': apiKey.trim(), 'Content-Type': 'application/json' }, + }); + const organic = (response.data as { organic?: unknown }).organic; + if (!Array.isArray(organic)) { + logger.warn('SpecStream response had no organic results array'); + return { results: [] }; + } + const results: SearchResult[] = []; + for (const r of organic) { + const item = r as { title?: string; link?: string; snippet?: string }; + if (typeof item.link !== 'string') continue; + results.push({ + title: (item.title ?? '').trim(), + url: item.link, + snippet: (item.snippet ?? '').trim(), + position: results.length + 1, + }); + if (results.length >= limit) break; + } + return { results }; + } catch (error) { + logger.warn('SpecStream search request failed', { query, error: (error as Error).message }); + return { results: [], reason: `SpecStream request failed: ${(error as Error).message}` }; + } +} + async function fetchSearxng(query: string, limit: number, opts: SearchOptions): Promise { const base = process.env.SEARXNG_URL; if (!base || !base.trim()) {