testing and verification

.github/workflows/ingestion-ci.yml

@@ -73,7 +73,7 @@ jobs:
       # No bundling step for Python — "build" here means the deployable artifact actually
       # builds (ARCHITECTURE §2's worker runs from this image, per infra/docker-compose.yml).
       - name: Build
-        run: docker build -t newstack-ingestion-ci apps/ingestion
+        run: docker build -t newstack-ingestion-ci -f apps/ingestion/Dockerfile .
 
   deploy:
     needs: test

.gitignore

@@ -4,6 +4,7 @@
 
 docs/
 validation/
+design/
 .superpowers/
 .claude/
 

README.md

@@ -3,15 +3,6 @@
 A news-aggregation platform: story clustering, source bias/factuality tagging, blindspot
 detection, and personalization over Indian news sources.
 
-The two docs in `docs/` are the spec, not just reference material:
-
-- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) — execution: schema (DDL), migrations, jobs,
-  SQL functions, caching, pagination, API mechanics.
-- [`docs/PRODUCT.md`](docs/PRODUCT.md) — meaning: definitions, formulas, thresholds, rubrics,
-  metrics (bias bar, blindspot math, ranking).
-
-Each concept is defined in exactly one of the two; the other links to it.
-
 ## Monorepo layout
 
 ```
@@ -20,11 +11,10 @@ apps/
   api/         Node/TS (Fastify) — auth, feed/story/topic/rating/saved/curator REST API
   web/         React 18 + TS (Vite) — the frontend
 packages/
-  shared-types/  TS types generated from docs/openapi.yaml (openapi-typescript)
+  shared-types/  TS types generated from the OpenAPI spec (openapi-typescript)
 config/        YAML config for clustering, embedding input, product/blindspot params, sources
 infra/         docker-compose, Postgres migrations, migration runner
-validation/    tests that validate the SQL/logic embedded in the docs themselves
-docs/          ARCHITECTURE.md, PRODUCT.md, openapi.yaml, runbooks, this repo's todo tracker
+validation/    tests that validate the SQL/logic embedded in the project's spec
 ```
 
 ## Prerequisites
@@ -96,7 +86,7 @@ python -m cli.story_browser  # inspect clustered stories from the CLI
 ```bash
 cd packages/shared-types
 npm install
-npm run generate   # regenerate src/openapi.ts from docs/openapi.yaml
+npm run generate   # regenerate src/openapi.ts from the OpenAPI spec
 ```
 
 ## Running tests
@@ -107,18 +97,12 @@ cd apps/web && npm test          # Vitest
 cd apps/ingestion && pytest      # pytest
 ```
 
-## Validating the docs
+## Validation suite
 
-`validation/` checks the SQL and logic embedded in `docs/ARCHITECTURE.md` / `docs/PRODUCT.md`
-directly — nothing here is committed generated code; it's re-extracted from the docs on every run.
-
-```bash
-./validation/run_all.sh docs/ARCHITECTURE.md docs/PRODUCT.md
-```
+`validation/` checks the SQL and logic embedded in this project's spec directly — nothing here is
+committed generated code; it's re-extracted from the spec on every run (see `validation/run_all.sh`
+and `validation/README.md`).
 
 The DB suite needs PostgreSQL 16 binaries at `/usr/lib/postgresql/16/bin` (Linux-oriented; this is
-what CI uses) — see `docs/mvp-todo.md` and `CLAUDE.md` for details.
-
-## Status
+what CI uses).
 
-See [`docs/mvp-todo.md`](docs/mvp-todo.md) for what's implemented and what's left.

apps/api/Dockerfile

@@ -1,8 +1,10 @@
-# apps/api/Dockerfile
+# apps/api/Dockerfile — built with repo root as context (see infra/docker-compose.yml):
+# src/features/blindspot/config.ts resolves config/ at the repo root via a relative path.
 FROM node:22-slim
-WORKDIR /app
-COPY package.json package-lock.json ./
+WORKDIR /app/apps/api
+COPY apps/api/package.json apps/api/package-lock.json ./
 RUN npm ci
-COPY . .
+COPY apps/api/ ./
+COPY config/ ../../config/
 RUN npm run build
 CMD ["npm", "start"]

apps/api/package.json

@@ -7,7 +7,9 @@
     "dev": "tsx watch src/server.ts",
     "build": "tsc -p tsconfig.json && cp src/features/auth/common-passwords.txt dist/features/auth/common-passwords.txt",
     "start": "node dist/server.js",
-    "test": "vitest run"
+    "test": "node --env-file-if-exists=.env ./node_modules/.bin/vitest run",
+    "billing:reconcile": "tsx src/features/billing/reconcile.ts",
+    "billing:provision-plans": "tsx src/features/billing/provision-plans.ts"
   },
   "dependencies": {
     "argon2": "^0.41.1",

apps/api/src/app.ts

@@ -4,14 +4,23 @@ import Fastify, { type FastifyInstance } from "fastify";
 
 import { WEB_ORIGIN } from "./config.js";
 import { authRoutes } from "./features/auth/routes.js";
+import { billingAdminRoutes, billingPublicRoutes, billingRoutes, billingWebhookRoutes } from "./features/billing/routes.js";
 import { blindspotCuratorRoutes } from "./features/blindspot/routes.js";
 import { curatorRoutes } from "./features/curator/routes.js";
 import { feedRoutes } from "./features/feed/routes.js";
-import { storyRoutes } from "./features/story/routes.js";
+import { storyPremiumRoutes, storyRoutes } from "./features/story/routes.js";
 import { ratingCuratorRoutes, ratingPublicRoutes } from "./features/rating/routes.js";
+import { geoRatingCuratorRoutes, geoRatingPublicRoutes } from "./features/geo-rating/routes.js";
+import {
+  sensationalismCuratorRoutes,
+  sensationalismPublicRoutes,
+} from "./features/sensationalism-rating/routes.js";
+import { ownershipCuratorRoutes, ownershipPublicRoutes } from "./features/ownership/routes.js";
+import { sourceCategoryCuratorRoutes } from "./features/source-category/routes.js";
 import { savedRoutes } from "./features/saved/routes.js";
 import { sourceRoutes } from "./features/source/routes.js";
 import { topicRoutes } from "./features/topic/routes.js";
+import { adminPlugin } from "./plugins/admin.js";
 import { authPlugin } from "./plugins/auth.js";
 import { cachePlugin } from "./plugins/cache.js";
 import { curatorPlugin } from "./plugins/curator.js";
@@ -37,15 +46,28 @@ export function buildApp(): FastifyInstance {
   // CSRF exploits ambient browser credentials (cookies), and curator routes carry none —
   // they're authenticated by an explicit header a cross-site page can't attach, and are
   // called by scripts/tooling with no browser Origin at all.
+  // /webhooks/* is exempt from both checks below for the same reason /api/curator/* is:
+  // the caller (PayPal) carries no browser Origin and is authenticated by its own signed
+  // payload (gateway.verifyWebhook), not ambient cookies — CSRF defenses don't apply.
   app.addHook("onRequest", async (request, reply) => {
-    if (!MUTATING_METHODS.has(request.method) || request.url.startsWith("/api/curator/")) return;
+    if (
+      !MUTATING_METHODS.has(request.method) ||
+      request.url.startsWith("/api/curator/") ||
+      request.url.startsWith("/webhooks/")
+    )
+      return;
     if (request.headers.origin !== WEB_ORIGIN) {
       reply.code(403).send({ error: "origin_mismatch" });
       return reply;
     }
   });
   app.addHook("preValidation", async (request, reply) => {
-    if (!MUTATING_METHODS.has(request.method) || request.url.startsWith("/api/curator/")) return;
+    if (
+      !MUTATING_METHODS.has(request.method) ||
+      request.url.startsWith("/api/curator/") ||
+      request.url.startsWith("/webhooks/")
+    )
+      return;
     const contentType = request.headers["content-type"] ?? "";
     if (!contentType.startsWith("application/json")) {
       reply.code(415).send({ error: "unsupported_content_type" });
@@ -88,6 +110,10 @@ export function buildApp(): FastifyInstance {
   // /auth/* — neither plugin; CSRF checks and rate limits only (rate limits: inside authRoutes).
   app.register(authRoutes, { prefix: "/auth" });
 
+  // /webhooks/* — no session, no auth plugin; signature-verified instead (see the CSRF
+  // exemption above and billingWebhookRoutes' own addContentTypeParser for the raw body).
+  app.register(billingWebhookRoutes, { prefix: "/webhooks" });
+
   // /api/public/* — cache plugin, no auth plugin: handlers here cannot read user identity.
   // The plugin is called directly (not via .register()) so its hook lands on this same
   // scope instead of an isolated child context — Fastify's encapsulation would otherwise
@@ -98,7 +124,11 @@ export function buildApp(): FastifyInstance {
     await storyRoutes(publicScope);
     await topicRoutes(publicScope);
     await ratingPublicRoutes(publicScope);
+    await sensationalismPublicRoutes(publicScope);
+    await geoRatingPublicRoutes(publicScope);
+    await ownershipPublicRoutes(publicScope);
     await sourceRoutes(publicScope);
+    await billingPublicRoutes(publicScope);
   }, { prefix: "/api/public" });
 
   // /api/* — auth plugin, no cache plugin. Same direct-call reasoning as above.
@@ -108,12 +138,25 @@ export function buildApp(): FastifyInstance {
     // the auth plugin an actual registered route to run its hook against.
     apiScope.get("/whoami", async (request) => ({ userId: request.userId }));
     await savedRoutes(apiScope);
+    await billingRoutes(apiScope);
+    await storyPremiumRoutes(apiScope);
   }, { prefix: "/api" });
 
+  // /api/admin/* — user auth plugin plus is_admin check (PAYPAL_SPEC §4.12).
+  app.register(async (adminScope) => {
+    await authPlugin(adminScope);
+    await adminPlugin(adminScope);
+    await billingAdminRoutes(adminScope);
+  }, { prefix: "/api/admin" });
+
   // /api/curator/* — shared-secret auth (curatorPlugin), no user-auth or cache plugin.
   app.register(async (curatorScope) => {
     await curatorPlugin(curatorScope);
     await ratingCuratorRoutes(curatorScope);
+    await sensationalismCuratorRoutes(curatorScope);
+    await geoRatingCuratorRoutes(curatorScope);
+    await ownershipCuratorRoutes(curatorScope);
+    await sourceCategoryCuratorRoutes(curatorScope);
     await curatorRoutes(curatorScope);
     await blindspotCuratorRoutes(curatorScope);
   }, { prefix: "/api/curator" });

apps/api/src/features/billing/entitlement.ts

@@ -0,0 +1,27 @@
+// Entitlement rule, PAYPAL_SPEC.md §6, verbatim:
+// A user is entitled when status = 'active' OR (status = 'past_due' AND now() < grace_until)
+// OR (status = 'canceled' AND now() < current_period_end).
+// Pure function — the highest-value test in this feature (CLAUDE.md "unit tests mirror the
+// doc's pure logic exactly"), kept separate from any DB/request code so it's trivial to
+// exhaust by table test.
+export interface SubscriptionState {
+  status: "incomplete" | "trialing" | "active" | "past_due" | "canceled" | "expired";
+  grace_until?: Date | null;
+  current_period_end?: Date | null;
+}
+
+export function isEntitled(sub: SubscriptionState | null, now: Date = new Date()): boolean {
+  if (sub === null) return false;
+  if (sub.status === "active") return true;
+  if (sub.status === "past_due") return sub.grace_until != null && now < sub.grace_until;
+  if (sub.status === "canceled") return sub.current_period_end != null && now < sub.current_period_end;
+  return false;
+}
+
+// Free tier metering (PAYPAL_SPEC §4.2 FR-P2): default 3 premium reads per rolling calendar
+// month, tunable post-launch per A6.
+export const FREE_PREMIUM_READS_PER_MONTH = 3;
+
+export function hasFreeReadsRemaining(viewsThisMonth: number): boolean {
+  return viewsThisMonth < FREE_PREMIUM_READS_PER_MONTH;
+}

apps/api/src/features/billing/gateway.ts

@@ -0,0 +1,47 @@
+// PaymentGateway adapter (PAYPAL_SPEC.md §5): the gateway is swappable behind this interface.
+// PayPal is the only implementation (./paypal.ts) — the spec's reference gateway was
+// Razorpay-style, adapted here per explicit instruction to use PayPal instead.
+
+export type NormalizedEventType =
+  | "subscription.activated"
+  | "subscription.charged"
+  | "subscription.halted"
+  | "subscription.cancelled"
+  | "payment.failed"
+  | "refund.processed";
+
+export interface NormalizedEvent {
+  type: NormalizedEventType;
+  gatewayEventId: string;
+  gatewaySubId: string | null;
+  gatewayPaymentId: string | null;
+  amountPaise: number | null;
+  occurredAt: Date;
+  raw: unknown;
+}
+
+export interface NormalizedPayment {
+  gatewayPaymentId: string;
+  status: "succeeded" | "failed" | "refunded" | "partially_refunded" | "pending";
+  amountPaise: number;
+}
+
+export interface CreateSubscriptionResult {
+  gatewaySubscriptionId: string;
+  checkoutParams: Record<string, unknown>;
+}
+
+export interface CreateOneTimeOrderResult {
+  gatewayOrderId: string;
+  checkoutParams: Record<string, unknown>;
+}
+
+export interface PaymentGateway {
+  createSubscription(input: { userId: string; planCode: string; gatewayPlanId: string }): Promise<CreateSubscriptionResult>;
+  createOneTimeOrder(input: { userId: string; planCode: string; amountPaise: number }): Promise<CreateOneTimeOrderResult>;
+  cancelSubscription(gatewaySubId: string, atCycleEnd: boolean): Promise<void>;
+  refundPayment(gatewayPaymentId: string, amountPaise: number): Promise<{ refundId: string }>;
+  verifyWebhook(rawBody: Buffer, headers: Record<string, string | string[] | undefined>): Promise<boolean>;
+  parseWebhook(rawBody: Buffer): NormalizedEvent;
+  fetchPayment(gatewayPaymentId: string): Promise<NormalizedPayment>;
+}

apps/api/src/features/billing/invoice.ts

@@ -0,0 +1,27 @@
+// Invoice rendering. PAYPAL_SPEC §4.11 asks for a PDF; a PDF library is a real dependency for
+// one templated page, so this renders a print-ready HTML invoice instead — the browser's own
+// "Print to PDF" covers the PDF requirement without adding puppeteer/pdfkit.
+// ponytail: no PDF-generation dependency; add one (e.g. pdfkit) if users need a downloadable
+// .pdf file rather than a printable page.
+export function renderInvoiceHtml(args: {
+  invoiceNumber: string;
+  planName: string;
+  amountPaise: number;
+  paidAt: Date;
+  userEmail: string;
+}): string {
+  const amount = (args.amountPaise / 100).toFixed(2);
+  return `<!doctype html>
+<html><head><meta charset="utf-8"><title>Invoice ${args.invoiceNumber}</title></head>
+<body style="font-family: sans-serif; max-width: 600px; margin: 2rem auto;">
+  <h1>Newstack</h1>
+  <h2>Invoice ${args.invoiceNumber}</h2>
+  <p>Billed to: ${args.userEmail}</p>
+  <p>Date: ${args.paidAt.toISOString().slice(0, 10)}</p>
+  <table style="width: 100%; border-collapse: collapse;">
+    <tr style="border-bottom: 1px solid #ccc;"><th style="text-align:left;">Plan</th><th style="text-align:right;">Amount (INR)</th></tr>
+    <tr><td>${args.planName}</td><td style="text-align:right;">₹${amount}</td></tr>
+  </table>
+  <p style="margin-top: 2rem; color: #666; font-size: 0.85rem;">Tax treatment pending accountant sign-off (PAYPAL_SPEC §2 A2) — GSTIN and tax breakup to be added here once confirmed.</p>
+</body></html>`;
+}