fix(crawl): a verified API key or session bearer is never charged; one fetch reads the page

package.json

@@ -45,9 +45,9 @@
     "@profullstack/profile": "0.1.6",
     "@profullstack/redeem-credits": "^0.2.0",
     "@profullstack/stack": "^0.5.0",
-    "@profullstack/throttle": "^0.4.0",
+    "@profullstack/throttle": "^0.4.1",
     "@profullstack/webhooks": "0.2.1",
-    "@profullstack/x402-gateway": "^0.15.3",
+    "@profullstack/x402-gateway": "^0.16.0",
     "@stripe/stripe-js": "^10.1.0",
     "@supabase/ssr": "^0.12.7",
     "@supabase/supabase-js": "^2.118.0",

src/lib/crawl-auth.test.ts

@@ -0,0 +1,66 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+const authenticateApiKey = vi.fn();
+const authenticateWithToken = vi.fn();
+const authenticateAgentPass = vi.fn();
+vi.mock("@/lib/auth/api-key", () => ({ authenticateApiKey: (...a: unknown[]) => authenticateApiKey(...a) }));
+vi.mock("@/lib/supabase/service", () => ({ authenticateWithToken: (...a: unknown[]) => authenticateWithToken(...a) }));
+vi.mock("@/lib/auth/agentpass", () => ({ authenticateAgentPass: (...a: unknown[]) => authenticateAgentPass(...a) }));
+
+import { authenticateCrawl, clearCrawlAuthCache, credentialKind } from "./crawl-auth";
+
+const req = (headers: Record<string, string>) => new Request("https://ugig.net/api/gigs/1", { headers });
+const KEY = `ugig_live_${crypto.randomUUID().replace(/-/g, "")}`;
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  clearCrawlAuthCache();
+});
+
+describe("credentialKind", () => {
+  it("reads ugig keys from every scheme the routes accept", () => {
+    expect(credentialKind(req({ authorization: `Bearer ${KEY}` }))?.kind).toBe("api_key");
+    expect(credentialKind(req({ authorization: `ApiKey ${KEY}` }))?.kind).toBe("api_key");
+    expect(credentialKind(req({ "x-api-key": KEY }))?.kind).toBe("api_key");
+    expect(credentialKind(req({ authorization: "Bearer aaa.bbb.ccc" }))?.kind).toBe("jwt");
+    expect(credentialKind(req({ authorization: "AgentPass p:s:1" }))?.kind).toBe("agentpass");
+    expect(credentialKind(req({ authorization: "Bearer junk" }))?.kind).toBe("unknown_bearer");
+  });
+
+  it("leaves the gateway's own credentials and other schemes alone", () => {
+    expect(credentialKind(req({}))).toBeNull();
+    expect(credentialKind(req({ authorization: "Bearer cp_abc.def" }))).toBeNull();
+    expect(credentialKind(req({ authorization: "Basic dXNlcjpwYXNz" }))).toBeNull();
+  });
+});
+
+describe("authenticateCrawl", () => {
+  it("passes a verified API key and remembers it", async () => {
+    authenticateApiKey.mockResolvedValue({ userId: "u", keyId: "k", scope: "full" });
+    expect(await authenticateCrawl(req({ authorization: `Bearer ${KEY}` }))).toBe(true);
+    expect(await authenticateCrawl(req({ authorization: `Bearer ${KEY}` }))).toBe(true);
+    expect(authenticateApiKey).toHaveBeenCalledTimes(1);
+  });
+
+  it("refuses a ugig key the database does not know, and does not remember the refusal", async () => {
+    authenticateApiKey.mockResolvedValue(null);
+    expect(await authenticateCrawl(req({ authorization: `Bearer ${KEY}` }))).toBe(false);
+    expect(await authenticateCrawl(req({ authorization: `Bearer ${KEY}` }))).toBe(false);
+    expect(authenticateApiKey).toHaveBeenCalledTimes(2);
+  });
+
+  it("refuses a bogus bearer without a lookup, so it gets 401 rather than a crawl offer", async () => {
+    expect(await authenticateCrawl(req({ authorization: "Bearer junk" }))).toBe(false);
+    expect(authenticateApiKey).not.toHaveBeenCalled();
+  });
+
+  it("verifies a Supabase access token with GoTrue", async () => {
+    authenticateWithToken.mockResolvedValueOnce({ user: { id: "u" } }).mockResolvedValueOnce(null);
+    expect(await authenticateCrawl(req({ authorization: "Bearer aaa.bbb.ccc" }))).toBe(true);
+    expect(await authenticateCrawl(req({ authorization: "Bearer ddd.eee.fff" }))).toBe(false);
+  });
+
+  it("has no opinion without a credential", async () => {
+    expect(await authenticateCrawl(req({}))).toBeNull();
+  });
+});

src/lib/crawl-auth.ts

@@ -0,0 +1,101 @@
+/**
+ * The crawl gateway's `authenticate` hook: does this request carry a ugig
+ * credential, and is it a real one?
+ *
+ * WHY. An agent seller reading its own listing, views or orders with its API
+ * key was answered with the crawl paywall (2026-10-28: GET /api/gigs/<id>
+ * with Chrome/141 and no Sec-Fetch got 402) because the gate judged the user
+ * agent before anything knew who was asking. A bogus bearer got the same 402,
+ * where "your key is wrong" is the honest answer.
+ *
+ * It VERIFIES. Exempting a request because an Authorization header is present
+ * would let every scraper bolt a fake one on, so a key is checked against the
+ * database (an API key), GoTrue (a Supabase access token) or the AgentPass
+ * signature, the same checks getAuthContext runs in the route.
+ *
+ *   true   a valid ugig credential: never charged
+ *   false  a credential that is ours and is wrong: answered 401
+ *   null   no credential, or one that is not ours (a crawl pass `cp_...`, a
+ *          Power Key, Basic auth): the gate's ordinary rules decide
+ *
+ * A verified credential is kept for a minute so an integration polling with one key does
+ * not pay a bcrypt compare twice per request (here and in the route).
+ */
+
+const TTL_MS = 60_000;
+const MAX_ENTRIES = 5_000;
+const verdicts = new Map<string, { ok: boolean; expiresAt: number }>();
+
+const UGIG_KEY = /^ugig_[a-z]+_/i;
+/** A Profullstack Power Key (@profullstack/keys `isPowerKey`; its types are not exported). */
+const isPowerKey = (token: string) => token.startsWith("pfs_") && token.indexOf(".") > 4;
+const JWT = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/;
+
+type Kind = "api_key" | "jwt" | "agentpass" | "unknown_bearer";
+
+/** Which of our credentials the request presents, if any. Pure: no I/O. */
+export function credentialKind(request: Request): { kind: Kind; value: string } | null {
+  const apiKey = request.headers.get("x-api-key")?.trim();
+  if (apiKey && UGIG_KEY.test(apiKey)) return { kind: "api_key", value: apiKey };
+
+  const auth = request.headers.get("authorization")?.trim();
+  if (!auth) return null;
+  if (/^AgentPass\s/i.test(auth)) return { kind: "agentpass", value: auth };
+
+  const scheme = /^(Bearer|ApiKey|Token)\s+(\S+)$/i.exec(auth);
+  if (!scheme) return null;
+  const token = scheme[2];
+  if (UGIG_KEY.test(token)) return { kind: "api_key", value: token };
+  if (scheme[1].toLowerCase() !== "bearer") return null;
+  // The gateway's own credentials: a crawl pass or a Profullstack Power Key.
+  if (token.startsWith("cp_") || isPowerKey(token)) return null;
+  if (JWT.test(token)) return { kind: "jwt", value: token };
+  return { kind: "unknown_bearer", value: token };
+}
+
+async function verify(kind: Kind, value: string): Promise<boolean> {
+  switch (kind) {
+    case "api_key": {
+      const { authenticateApiKey } = await import("@/lib/auth/api-key");
+      return Boolean(await authenticateApiKey(null, value));
+    }
+    case "jwt": {
+      const { authenticateWithToken } = await import("@/lib/supabase/service");
+      return Boolean(await authenticateWithToken(`Bearer ${value}`));
+    }
+    case "agentpass": {
+      const { authenticateAgentPass } = await import("@/lib/auth/agentpass");
+      return Boolean(await authenticateAgentPass(value));
+    }
+    default:
+      // A bearer that is none of ours. The routes would refuse it too.
+      return false;
+  }
+}
+
+export async function authenticateCrawl(request: Request): Promise<boolean | null> {
+  const presented = credentialKind(request);
+  if (!presented) return null;
+
+  const cacheKey = `${presented.kind}:${presented.value}`;
+  const now = Date.now();
+  const hit = verdicts.get(cacheKey);
+  if (hit && hit.expiresAt > now) return hit.ok;
+
+  // A failed lookup throws out of here and the gateway reads it as "no
+  // opinion", never as a pass; only real verdicts are remembered.
+  const ok = await verify(presented.kind, presented.value);
+  // Only a pass is remembered: a refusal can be a database blip, and a key
+  // that was refused once must not stay refused for a minute. AgentPass signs
+  // a timestamp, so its header is single-use by design.
+  if (ok && presented.kind !== "agentpass") {
+    verdicts.set(cacheKey, { ok, expiresAt: now + TTL_MS });
+    if (verdicts.size > MAX_ENTRIES) verdicts.delete(verdicts.keys().next().value as string);
+  }
+  return ok;
+}
+
+/** Test seam. */
+export function clearCrawlAuthCache() {
+  verdicts.clear();
+}

src/lib/crawl-gateway.test.ts

@@ -145,11 +145,20 @@ describe("crawlers that do not say who they are", () => {
     expect(res).toBeUndefined();
   });
 
-  it("charges a Chrome UA that lacks Sec-Fetch-Mode", async () => {
-    const res = await gate(request("/gigs/x", CHROME));
+  it("lets a Chrome UA without Sec-Fetch-Mode read a few pages, then charges it", async () => {
+    // gateway 0.16: one fetch reads the page (an agent reading its own
+    // listing, 2026-10-28); a bulk crawl still pays past 20 an hour.
+    const from = { "x-real-ip": "203.0.113.77" };
+    for (let i = 0; i < 20; i++) expect(await gate(request(`/gigs/${i}`, CHROME, from))).toBeUndefined();
+    const res = await gate(request("/gigs/x", CHROME, from));
     expect(res?.status).toBe(402);
   });
 
+  it("answers a bogus bearer 401, not with a crawl offer", async () => {
+    const res = await gate(request("/api/gigs/x", CHROME, { authorization: "Bearer not-a-real-key" }));
+    expect(res?.status).toBe(401);
+  });
+
   it("passes a Chrome UA that sends Sec-Fetch-Mode", async () => {
     expect(await gate(request("/gigs/x", CHROME, { "sec-fetch-mode": "navigate" }))).toBeUndefined();
   });

src/lib/crawl-gateway.ts

@@ -1,6 +1,7 @@
 import { createGateway } from "@profullstack/x402-gateway";
 import { x402Proxy } from "@profullstack/x402-gateway/next";
 import { getAppUrl } from "@/lib/app-url";
+import { authenticateCrawl } from "@/lib/crawl-auth";
 
 /**
  * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot,
@@ -63,6 +64,9 @@ export const gateway = createGateway({
   // are judged by the lists instead, never by this.
   chargeSpoofedBrowsers: true,
   exempt: hasSupabaseSession,
+  // A VERIFIED ugig API key, Supabase access token or AgentPass signature is
+  // never charged; one that is ours and wrong is answered 401. See crawl-auth.ts.
+  authenticate: authenticateCrawl,
 });
 
 /** Resolves to a Response for a refused crawler, or undefined to carry on. */