diff --git a/bun.lock b/bun.lock index d6dd7f4e..fe154e92 100644 --- a/bun.lock +++ b/bun.lock @@ -12,9 +12,9 @@ "@profullstack/profile": "0.1.6", "@profullstack/redeem-credits": "^0.2.0", "@profullstack/stack": "^0.5.0", - "@profullstack/throttle": "^0.4.0", + "@profullstack/throttle": "^0.4.1", "@profullstack/webhooks": "0.2.1", - "@profullstack/x402-gateway": "^0.15.3", + "@profullstack/x402-gateway": "^0.16.0", "@stripe/stripe-js": "^10.1.0", "@supabase/ssr": "^0.12.7", "@supabase/supabase-js": "^2.118.0", @@ -350,11 +350,11 @@ "@profullstack/stack": ["@profullstack/stack@0.5.0", "", { "dependencies": { "@profullstack/emailer": "^1.0.1", "@profullstack/form-guard": "^0.1.1", "@profullstack/referrals": "^0.1.0" }, "peerDependencies": { "@supabase/ssr": ">=0.5.0", "next": ">=13.0.0", "react": ">=18.0.0" }, "optionalPeers": ["@supabase/ssr", "next", "react"] }, "sha512-LbXovIhyzLEq+a+y3q8KWC+jN1tJjvSoUiFEhpiEBisQn5U+iutIGHDgg16Lul74kQaFZeFlbp7CMPAVJ8KL6g=="], - "@profullstack/throttle": ["@profullstack/throttle@0.4.0", "", { "dependencies": { "@profullstack/x402-gateway": "^0.6.0 || ^0.7.0 || ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^0.14.0 || ^0.15.0" } }, "sha512-Lj+XHt51OxyaJ2FqosZDyr9Z5RFB8z+ARixpoUfrDu3YlvFlct5GbbZM/C0qLZkUgzeuFSDvtYt9f0NNfBkFqA=="], + "@profullstack/throttle": ["@profullstack/throttle@0.4.1", "", { "dependencies": { "@profullstack/x402-gateway": "^0.6.0 || ^0.7.0 || ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^0.14.0 || ^0.15.0 || ^0.16.0" } }, "sha512-ffY49nQ20/ZXyp/LVh/LxXTKJozqxyggBhjt9rRGz/JCC30IKFaqN2UmNwThm721r0rgQnhikpHNOb5EqzU9kw=="], "@profullstack/webhooks": ["@profullstack/webhooks@0.2.1", "", {}, "sha512-QHeSkpu4O44oJ620qhQqxqFlD3u/OhupmXRnBHw4MGKuDTa0R+Y/727nnF7psSHxxSfnbWfNuVl15z3IXuaZiw=="], - "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.15.3", "", { "dependencies": { "@profullstack/keys": "^0.1.0" } }, "sha512-YtxaHuiNdnEmwQBkNJXGOEvFUb+5k54GPeZw5RiPKqYcKrA/tRTRVTMbyuX4XSGfmaOEo1BCGpa3w8S46+YQUg=="], + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.16.0", "", { "dependencies": { "@profullstack/keys": "^0.1.0" } }, "sha512-3xN7vlGXmoqdexR7NNAIExZLtYvE5M8nR9fqr7kwLK/xks06lUiymEsjZOc2wCbowfnCVNiZBOe98rvsCE9Mgw=="], "@rolldown/binding-android-arm-eabi": ["@rolldown/binding-android-arm-eabi@1.2.15", "", { "os": "android", "cpu": "arm" }, "sha512-aLcgK9gIJ3YRvrrtfSBbRa27F5WrpiTaPulnFpsHPxpUHgFlSgnU4854MHwoRGEBhfRJDWCS2WQ7lIjskE9Olw=="], diff --git a/package.json b/package.json index fb9c87c9..a391e35e 100644 --- a/package.json +++ b/package.json @@ -45,9 +45,9 @@ "@profullstack/profile": "0.1.6", "@profullstack/redeem-credits": "^0.2.0", "@profullstack/stack": "^0.5.0", - "@profullstack/throttle": "^0.4.0", + "@profullstack/throttle": "^0.4.1", "@profullstack/webhooks": "0.2.1", - "@profullstack/x402-gateway": "^0.15.3", + "@profullstack/x402-gateway": "^0.16.0", "@stripe/stripe-js": "^10.1.0", "@supabase/ssr": "^0.12.7", "@supabase/supabase-js": "^2.118.0", diff --git a/src/lib/crawl-auth.test.ts b/src/lib/crawl-auth.test.ts new file mode 100644 index 00000000..4af9e3c4 --- /dev/null +++ b/src/lib/crawl-auth.test.ts @@ -0,0 +1,66 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const authenticateApiKey = vi.fn(); +const authenticateWithToken = vi.fn(); +const authenticateAgentPass = vi.fn(); +vi.mock("@/lib/auth/api-key", () => ({ authenticateApiKey: (...a: unknown[]) => authenticateApiKey(...a) })); +vi.mock("@/lib/supabase/service", () => ({ authenticateWithToken: (...a: unknown[]) => authenticateWithToken(...a) })); +vi.mock("@/lib/auth/agentpass", () => ({ authenticateAgentPass: (...a: unknown[]) => authenticateAgentPass(...a) })); + +import { authenticateCrawl, clearCrawlAuthCache, credentialKind } from "./crawl-auth"; + +const req = (headers: Record) => new Request("https://ugig.net/api/gigs/1", { headers }); +const KEY = `ugig_live_${crypto.randomUUID().replace(/-/g, "")}`; + +beforeEach(() => { + vi.clearAllMocks(); + clearCrawlAuthCache(); +}); + +describe("credentialKind", () => { + it("reads ugig keys from every scheme the routes accept", () => { + expect(credentialKind(req({ authorization: `Bearer ${KEY}` }))?.kind).toBe("api_key"); + expect(credentialKind(req({ authorization: `ApiKey ${KEY}` }))?.kind).toBe("api_key"); + expect(credentialKind(req({ "x-api-key": KEY }))?.kind).toBe("api_key"); + expect(credentialKind(req({ authorization: "Bearer aaa.bbb.ccc" }))?.kind).toBe("jwt"); + expect(credentialKind(req({ authorization: "AgentPass p:s:1" }))?.kind).toBe("agentpass"); + expect(credentialKind(req({ authorization: "Bearer junk" }))?.kind).toBe("unknown_bearer"); + }); + + it("leaves the gateway's own credentials and other schemes alone", () => { + expect(credentialKind(req({}))).toBeNull(); + expect(credentialKind(req({ authorization: "Bearer cp_abc.def" }))).toBeNull(); + expect(credentialKind(req({ authorization: "Basic dXNlcjpwYXNz" }))).toBeNull(); + }); +}); + +describe("authenticateCrawl", () => { + it("passes a verified API key and remembers it", async () => { + authenticateApiKey.mockResolvedValue({ userId: "u", keyId: "k", scope: "full" }); + expect(await authenticateCrawl(req({ authorization: `Bearer ${KEY}` }))).toBe(true); + expect(await authenticateCrawl(req({ authorization: `Bearer ${KEY}` }))).toBe(true); + expect(authenticateApiKey).toHaveBeenCalledTimes(1); + }); + + it("refuses a ugig key the database does not know, and does not remember the refusal", async () => { + authenticateApiKey.mockResolvedValue(null); + expect(await authenticateCrawl(req({ authorization: `Bearer ${KEY}` }))).toBe(false); + expect(await authenticateCrawl(req({ authorization: `Bearer ${KEY}` }))).toBe(false); + expect(authenticateApiKey).toHaveBeenCalledTimes(2); + }); + + it("refuses a bogus bearer without a lookup, so it gets 401 rather than a crawl offer", async () => { + expect(await authenticateCrawl(req({ authorization: "Bearer junk" }))).toBe(false); + expect(authenticateApiKey).not.toHaveBeenCalled(); + }); + + it("verifies a Supabase access token with GoTrue", async () => { + authenticateWithToken.mockResolvedValueOnce({ user: { id: "u" } }).mockResolvedValueOnce(null); + expect(await authenticateCrawl(req({ authorization: "Bearer aaa.bbb.ccc" }))).toBe(true); + expect(await authenticateCrawl(req({ authorization: "Bearer ddd.eee.fff" }))).toBe(false); + }); + + it("has no opinion without a credential", async () => { + expect(await authenticateCrawl(req({}))).toBeNull(); + }); +}); diff --git a/src/lib/crawl-auth.ts b/src/lib/crawl-auth.ts new file mode 100644 index 00000000..ab592c5f --- /dev/null +++ b/src/lib/crawl-auth.ts @@ -0,0 +1,101 @@ +/** + * The crawl gateway's `authenticate` hook: does this request carry a ugig + * credential, and is it a real one? + * + * WHY. An agent seller reading its own listing, views or orders with its API + * key was answered with the crawl paywall (2026-10-28: GET /api/gigs/ + * with Chrome/141 and no Sec-Fetch got 402) because the gate judged the user + * agent before anything knew who was asking. A bogus bearer got the same 402, + * where "your key is wrong" is the honest answer. + * + * It VERIFIES. Exempting a request because an Authorization header is present + * would let every scraper bolt a fake one on, so a key is checked against the + * database (an API key), GoTrue (a Supabase access token) or the AgentPass + * signature, the same checks getAuthContext runs in the route. + * + * true a valid ugig credential: never charged + * false a credential that is ours and is wrong: answered 401 + * null no credential, or one that is not ours (a crawl pass `cp_...`, a + * Power Key, Basic auth): the gate's ordinary rules decide + * + * A verified credential is kept for a minute so an integration polling with one key does + * not pay a bcrypt compare twice per request (here and in the route). + */ + +const TTL_MS = 60_000; +const MAX_ENTRIES = 5_000; +const verdicts = new Map(); + +const UGIG_KEY = /^ugig_[a-z]+_/i; +/** A Profullstack Power Key (@profullstack/keys `isPowerKey`; its types are not exported). */ +const isPowerKey = (token: string) => token.startsWith("pfs_") && token.indexOf(".") > 4; +const JWT = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/; + +type Kind = "api_key" | "jwt" | "agentpass" | "unknown_bearer"; + +/** Which of our credentials the request presents, if any. Pure: no I/O. */ +export function credentialKind(request: Request): { kind: Kind; value: string } | null { + const apiKey = request.headers.get("x-api-key")?.trim(); + if (apiKey && UGIG_KEY.test(apiKey)) return { kind: "api_key", value: apiKey }; + + const auth = request.headers.get("authorization")?.trim(); + if (!auth) return null; + if (/^AgentPass\s/i.test(auth)) return { kind: "agentpass", value: auth }; + + const scheme = /^(Bearer|ApiKey|Token)\s+(\S+)$/i.exec(auth); + if (!scheme) return null; + const token = scheme[2]; + if (UGIG_KEY.test(token)) return { kind: "api_key", value: token }; + if (scheme[1].toLowerCase() !== "bearer") return null; + // The gateway's own credentials: a crawl pass or a Profullstack Power Key. + if (token.startsWith("cp_") || isPowerKey(token)) return null; + if (JWT.test(token)) return { kind: "jwt", value: token }; + return { kind: "unknown_bearer", value: token }; +} + +async function verify(kind: Kind, value: string): Promise { + switch (kind) { + case "api_key": { + const { authenticateApiKey } = await import("@/lib/auth/api-key"); + return Boolean(await authenticateApiKey(null, value)); + } + case "jwt": { + const { authenticateWithToken } = await import("@/lib/supabase/service"); + return Boolean(await authenticateWithToken(`Bearer ${value}`)); + } + case "agentpass": { + const { authenticateAgentPass } = await import("@/lib/auth/agentpass"); + return Boolean(await authenticateAgentPass(value)); + } + default: + // A bearer that is none of ours. The routes would refuse it too. + return false; + } +} + +export async function authenticateCrawl(request: Request): Promise { + const presented = credentialKind(request); + if (!presented) return null; + + const cacheKey = `${presented.kind}:${presented.value}`; + const now = Date.now(); + const hit = verdicts.get(cacheKey); + if (hit && hit.expiresAt > now) return hit.ok; + + // A failed lookup throws out of here and the gateway reads it as "no + // opinion", never as a pass; only real verdicts are remembered. + const ok = await verify(presented.kind, presented.value); + // Only a pass is remembered: a refusal can be a database blip, and a key + // that was refused once must not stay refused for a minute. AgentPass signs + // a timestamp, so its header is single-use by design. + if (ok && presented.kind !== "agentpass") { + verdicts.set(cacheKey, { ok, expiresAt: now + TTL_MS }); + if (verdicts.size > MAX_ENTRIES) verdicts.delete(verdicts.keys().next().value as string); + } + return ok; +} + +/** Test seam. */ +export function clearCrawlAuthCache() { + verdicts.clear(); +} diff --git a/src/lib/crawl-gateway.test.ts b/src/lib/crawl-gateway.test.ts index 65aa4064..cb8761e7 100644 --- a/src/lib/crawl-gateway.test.ts +++ b/src/lib/crawl-gateway.test.ts @@ -145,11 +145,20 @@ describe("crawlers that do not say who they are", () => { expect(res).toBeUndefined(); }); - it("charges a Chrome UA that lacks Sec-Fetch-Mode", async () => { - const res = await gate(request("/gigs/x", CHROME)); + it("lets a Chrome UA without Sec-Fetch-Mode read a few pages, then charges it", async () => { + // gateway 0.16: one fetch reads the page (an agent reading its own + // listing, 2026-10-28); a bulk crawl still pays past 20 an hour. + const from = { "x-real-ip": "203.0.113.77" }; + for (let i = 0; i < 20; i++) expect(await gate(request(`/gigs/${i}`, CHROME, from))).toBeUndefined(); + const res = await gate(request("/gigs/x", CHROME, from)); expect(res?.status).toBe(402); }); + it("answers a bogus bearer 401, not with a crawl offer", async () => { + const res = await gate(request("/api/gigs/x", CHROME, { authorization: "Bearer not-a-real-key" })); + expect(res?.status).toBe(401); + }); + it("passes a Chrome UA that sends Sec-Fetch-Mode", async () => { expect(await gate(request("/gigs/x", CHROME, { "sec-fetch-mode": "navigate" }))).toBeUndefined(); }); diff --git a/src/lib/crawl-gateway.ts b/src/lib/crawl-gateway.ts index 555d260f..11209c31 100644 --- a/src/lib/crawl-gateway.ts +++ b/src/lib/crawl-gateway.ts @@ -1,6 +1,7 @@ import { createGateway } from "@profullstack/x402-gateway"; import { x402Proxy } from "@profullstack/x402-gateway/next"; import { getAppUrl } from "@/lib/app-url"; +import { authenticateCrawl } from "@/lib/crawl-auth"; /** * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, @@ -63,6 +64,9 @@ export const gateway = createGateway({ // are judged by the lists instead, never by this. chargeSpoofedBrowsers: true, exempt: hasSupabaseSession, + // A VERIFIED ugig API key, Supabase access token or AgentPass signature is + // never charged; one that is ours and wrong is answered 401. See crawl-auth.ts. + authenticate: authenticateCrawl, }); /** Resolves to a Response for a refused crawler, or undefined to carry on. */