Review and Fix

README.md

@@ -0,0 +1,124 @@
+# Newstack
+
+A news-aggregation platform: story clustering, source bias/factuality tagging, blindspot
+detection, and personalization over Indian news sources.
+
+The two docs in `docs/` are the spec, not just reference material:
+
+- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) — execution: schema (DDL), migrations, jobs,
+  SQL functions, caching, pagination, API mechanics.
+- [`docs/PRODUCT.md`](docs/PRODUCT.md) — meaning: definitions, formulas, thresholds, rubrics,
+  metrics (bias bar, blindspot math, ranking).
+
+Each concept is defined in exactly one of the two; the other links to it.
+
+## Monorepo layout
+
+```
+apps/
+  ingestion/   Python — fetch, canonicalize, embed, cluster, baseline/ratings jobs (APScheduler)
+  api/         Node/TS (Fastify) — auth, feed/story/topic/rating/saved/curator REST API
+  web/         React 18 + TS (Vite) — the frontend
+packages/
+  shared-types/  TS types generated from docs/openapi.yaml (openapi-typescript)
+config/        YAML config for clustering, embedding input, product/blindspot params, sources
+infra/         docker-compose, Postgres migrations, migration runner
+validation/    tests that validate the SQL/logic embedded in the docs themselves
+docs/          ARCHITECTURE.md, PRODUCT.md, openapi.yaml, runbooks, this repo's todo tracker
+```
+
+## Prerequisites
+
+- Docker + Docker Compose (easiest path — runs everything below for you), **or**
+- Node.js 20+, Python 3.11+, PostgreSQL 16 with `pgvector`, Redis 7, for running apps locally
+  without containers.
+
+## Install & run (Docker Compose — recommended)
+
+```bash
+cd infra
+docker compose up --build
+```
+
+This starts Postgres (pgvector), Redis, the ingestion worker, the API, the mailer process, and
+the web frontend. Apply migrations once the db is healthy:
+
+```bash
+DATABASE_URL=postgresql://newstack:newstack@localhost:5432/newstack python infra/migrate.py
+```
+
+Then:
+
+- Web app: http://localhost:5173
+- API: http://localhost:3000 (health check at `/health`)
+
+All services use insecure dev-default secrets (JWT key, curator API key, cursor-signing key) when
+their env vars are unset — fine for local dev, **never for production**. See `apps/api/src/config.ts`,
+`features/auth/jwt.ts`, `pagination.ts`, and `plugins/curator.ts` for the env vars to override
+(`JWT_KEYS`, `CURATOR_API_KEY`, `CURSOR_KEYS`, `RESEND_API_KEY`/`MAIL_FROM` for real email).
+
+## Install & run (without Docker)
+
+**Database**
+
+```bash
+# Postgres 16 with pgvector running locally, then:
+DATABASE_URL=postgresql://newstack:newstack@localhost:5432/newstack python infra/migrate.py
+```
+
+**`apps/api`**
+
+```bash
+cd apps/api
+npm install
+npm run dev     # tsx watch, http://localhost:3000
+```
+
+**`apps/web`**
+
+```bash
+cd apps/web
+npm install
+npm run dev      # Vite dev server, http://localhost:5173
+```
+
+**`apps/ingestion`**
+
+```bash
+cd apps/ingestion
+pip install -r requirements.txt
+python -m service         # scheduler/orchestration entrypoint
+python -m cli.story_browser  # inspect clustered stories from the CLI
+```
+
+**`packages/shared-types`**
+
+```bash
+cd packages/shared-types
+npm install
+npm run generate   # regenerate src/openapi.ts from docs/openapi.yaml
+```
+
+## Running tests
+
+```bash
+cd apps/api && npm test          # Vitest
+cd apps/web && npm test          # Vitest
+cd apps/ingestion && pytest      # pytest
+```
+
+## Validating the docs
+
+`validation/` checks the SQL and logic embedded in `docs/ARCHITECTURE.md` / `docs/PRODUCT.md`
+directly — nothing here is committed generated code; it's re-extracted from the docs on every run.
+
+```bash
+./validation/run_all.sh docs/ARCHITECTURE.md docs/PRODUCT.md
+```
+
+The DB suite needs PostgreSQL 16 binaries at `/usr/lib/postgresql/16/bin` (Linux-oriented; this is
+what CI uses) — see `docs/mvp-todo.md` and `CLAUDE.md` for details.
+
+## Status
+
+See [`docs/mvp-todo.md`](docs/mvp-todo.md) for what's implemented and what's left.

apps/api/src/features/blindspot/config.ts

@@ -0,0 +1,33 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+// config/product/blindspot.yaml (PRODUCT §4.4) is 5 flat numeric keys with trailing comments --
+// simple enough that a YAML dependency isn't worth adding for it.
+export interface BlindspotParams {
+  nMin: number;
+  materiality: number;
+  z: number;
+  baselineWindowDays: number;
+  poolCutoverAppearances: number;
+}
+
+const CONFIG_PATH = fileURLToPath(new URL("../../../../../config/product/blindspot.yaml", import.meta.url));
+
+export function loadBlindspotConfig(path: string = CONFIG_PATH): BlindspotParams {
+  const text = readFileSync(path, "utf-8");
+  const values: Record<string, number> = {};
+  for (const line of text.split("\n")) {
+    const match = /^(\w+):\s*([0-9.]+)/.exec(line);
+    if (match) values[match[1]] = Number(match[2]);
+  }
+  for (const key of ["n_min", "materiality", "z", "baseline_window_days", "pool_cutover_appearances"]) {
+    if (!(key in values)) throw new Error(`blindspot.yaml missing required key: ${key}`);
+  }
+  return {
+    nMin: values.n_min,
+    materiality: values.materiality,
+    z: values.z,
+    baselineWindowDays: values.baseline_window_days,
+    poolCutoverAppearances: values.pool_cutover_appearances,
+  };
+}

apps/api/src/features/blindspot/repo.ts

@@ -1,5 +1,7 @@
 import type { Pool } from "pg";
 
+import type { BlindspotParams } from "./config.js";
+
 export interface Baseline {
   p_left: number;
   p_center: number;
@@ -30,7 +32,13 @@ export interface BlindspotRow {
 // binds them): stories active in the last 7 days, newest coverage first.
 export async function fetchBlindspotPage(
   pool: Pool,
-  args: { asOf: Date; cursorCoverageAt: Date | null; cursorId: string | null; limit: number }
+  args: {
+    asOf: Date;
+    cursorCoverageAt: Date | null;
+    cursorId: string | null;
+    limit: number;
+    params: BlindspotParams;
+  }
 ): Promise<BlindspotRow[]> {
   const result = await pool.query<{
     id: string;
@@ -60,15 +68,17 @@ export async function fetchBlindspotPage(
        GROUP BY a.story_id),
      flagged AS (
        SELECT c.*, blindspot_side(c.k_left::int, c.k_center::int, c.k_right::int,
-                                  b.p_left, b.p_center, b.p_right) AS side
+                                  b.p_left, b.p_center, b.p_right,
+                                  $5::int, $6::float8, $7::float8) AS side
        FROM counts c CROSS JOIN base b)
      SELECT s.id, s.title, s.last_coverage_at, f.side, f.k_left, f.k_center, f.k_right
      FROM flagged f JOIN stories s ON s.id = f.story_id
      WHERE f.side IS NOT NULL
        AND ($2::timestamptz IS NULL OR (s.last_coverage_at, s.id) < ($2, $3))
      ORDER BY s.last_coverage_at DESC, s.id DESC
      LIMIT $4`,
-    [args.asOf, args.cursorCoverageAt, args.cursorId, args.limit]
+    [args.asOf, args.cursorCoverageAt, args.cursorId, args.limit,
+     args.params.nMin, args.params.materiality, args.params.z]
   );
   return result.rows.map((r) => ({
     ...r,

apps/api/src/features/blindspot/service.ts

@@ -1,10 +1,14 @@
 import type { Pool } from "pg";
 
 import { buildPage, decodeCursor } from "../../pagination.js";
+import { loadBlindspotConfig } from "./config.js";
 import { fetchBlindspotPage, type BlindspotRow } from "./repo.js";
 
 const PAGE_SIZE = 20;
 const FEED_NAME = "blindspot";
+// Loaded once per process, not per request -- config/product/blindspot.yaml (PRODUCT §4.4)
+// changes only through a deploy, same as every other config/*.yaml this codebase reads.
+const BLINDSPOT_PARAMS = loadBlindspotConfig();
 
 export interface BlindspotItem {
   id: string;
@@ -32,7 +36,13 @@ export async function getBlindspotFeed(pool: Pool, cursorRaw: string | undefined
   const cursorCoverageAt = cursor === null ? null : new Date(cursor.c);
   const cursorId = cursor === null ? null : cursor.id;
 
-  const rows = await fetchBlindspotPage(pool, { asOf, cursorCoverageAt, cursorId, limit: PAGE_SIZE + 1 });
+  const rows = await fetchBlindspotPage(pool, {
+    asOf,
+    cursorCoverageAt,
+    cursorId,
+    limit: PAGE_SIZE + 1,
+    params: BLINDSPOT_PARAMS,
+  });
   const page = buildPage(rows, PAGE_SIZE, FEED_NAME, asOf);
 
   const items: BlindspotItem[] = page.items.map((row: BlindspotRow) => ({

apps/api/test/blindspot-config.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "vitest";
+
+import { loadBlindspotConfig } from "../src/features/blindspot/config.js";
+
+describe("loadBlindspotConfig (PRODUCT §4.4)", () => {
+  it("reads the real config/product/blindspot.yaml, not hardcoded defaults", () => {
+    const params = loadBlindspotConfig();
+    expect(params).toEqual({
+      nMin: 6,
+      materiality: 0.5,
+      z: 1.645,
+      baselineWindowDays: 30,
+      poolCutoverAppearances: 500,
+    });
+  });
+
+  it("throws if a required key is missing, rather than silently defaulting", async () => {
+    const fs = await import("node:fs/promises");
+    const os = await import("node:os");
+    const path = await import("node:path");
+    const tmpFile = path.join(await fs.mkdtemp(path.join(os.tmpdir(), "blindspot-")), "blindspot.yaml");
+    await fs.writeFile(tmpFile, "n_min: 6\nmateriality: 0.5\n");
+    expect(() => loadBlindspotConfig(tmpFile)).toThrow(/missing required key/);
+  });
+});

apps/api/test/blindspot.test.ts

@@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest";
 
 import { buildApp } from "../src/app.js";
 import { getPool } from "../src/db.js";
+import { loadBlindspotConfig } from "../src/features/blindspot/config.js";
+import { fetchBlindspotPage } from "../src/features/blindspot/repo.js";
 import { getBlindspotFeed } from "../src/features/blindspot/service.js";
 import { addCoverage, createTestSource, createTestStory, rateSource } from "./feed-helpers.js";
 
@@ -45,6 +47,30 @@ describe("getBlindspotFeed (PRODUCT §4)", () => {
     expect(result.items.find((i) => i.id === storyId)).toBeUndefined();
   });
 
+  it("n_min/materiality/z are actually read from config/product/blindspot.yaml, not a silently "
+    + "ignored argument or a hardcoded SQL default", async () => {
+    await insertBaseline();
+    // Same minimal 6-source flag as the first test above, which passes at the real n_min (6).
+    const sources = await Promise.all(Array.from({ length: 6 }, () => createTestSource("blindspot-cfgwire")));
+    await Promise.all(sources.map((s) => rateSource(s, { biasRating: "left", confidence: "high" })));
+    const { storyId } = await createTestStory(sources[0]);
+    await Promise.all(sources.slice(1).map((s) => addCoverage(storyId, s)));
+
+    const real = loadBlindspotConfig();
+    const flagsAtRealNMin = await fetchBlindspotPage(getPool(), {
+      asOf: new Date(), cursorCoverageAt: null, cursorId: null, limit: 50, params: real,
+    });
+    expect(flagsAtRealNMin.some((r) => r.id === storyId)).toBe(true);
+
+    // Raising n_min by 1 past this story's count must suppress the flag -- if the SQL were
+    // still using its own DEFAULT (6) instead of this bound parameter, it would stay flagged.
+    const stricter = { ...real, nMin: real.nMin + 1 };
+    const flagsAtStricterNMin = await fetchBlindspotPage(getPool(), {
+      asOf: new Date(), cursorCoverageAt: null, cursorId: null, limit: 50, params: stricter,
+    });
+    expect(flagsAtStricterNMin.some((r) => r.id === storyId)).toBe(false);
+  });
+
   it("does not flag balanced coverage (no side is one-sided)", async () => {
     await insertBaseline();
     const left = await Promise.all(Array.from({ length: 3 }, () => createTestSource("blindspot-bal-left")));

apps/ingestion/config.py

@@ -9,6 +9,7 @@
 CLUSTERING_DIR = CONFIG_ROOT / "clustering"
 EMBED_INPUT_DIR = CONFIG_ROOT / "embed_input"
 SOURCES_DIR = CONFIG_ROOT / "sources"
+PRODUCT_DIR = CONFIG_ROOT / "product"
 
 
 def load_clustering_config(version: str = "v1") -> dict:
@@ -32,3 +33,10 @@ def load_source_adapter(adapter: str) -> dict:
             overrides = yaml.safe_load(f) or {}
         merged.update(overrides)
     return merged
+
+
+def load_blindspot_config() -> dict:
+    """PRODUCT §4.4's pinned decision-rule parameters. A change ships only after beating the
+    §4.9 calibration set with the new values -- never hand-edit code to tune these."""
+    with open(PRODUCT_DIR / "blindspot.yaml", encoding="utf-8") as f:
+        return yaml.safe_load(f)

apps/ingestion/jobs/baseline.py

@@ -1,9 +1,12 @@
 """Daily blindspot baseline (PRODUCT §4.3). Verbatim from the doc — the cold-start
-pool/appearances split and the 500-appearance cutover are policy, not reimplemented here."""
+pool/appearances split and the 500-appearance cutover are policy, read from
+config/product/blindspot.yaml, not hardcoded here."""
 from __future__ import annotations
 
 import psycopg
 
+from config import load_blindspot_config
+
 BASELINE_SQL = """
 INSERT INTO bias_baselines (spectrum, method, appearances, p_left, p_center, p_right)
 WITH app AS (
@@ -14,15 +17,16 @@
   JOIN stories s ON s.id = a.story_id AND s.status = 'active' AND s.hidden_at IS NULL
   JOIN source_ratings r ON r.source_id = a.source_id AND r.superseded_at IS NULL
   WHERE a.hidden_at IS NULL AND r.spectrum = 'IN' AND r.confidence <> 'low'
-    AND a.coverage_at >= now() - interval '30 days'),
+    AND a.coverage_at >= now() - (%(baseline_window_days)s * interval '1 day')),
 pool AS (
   SELECT CASE WHEN r.bias_rating IN ('left','center-left') THEN 'L'
               WHEN r.bias_rating = 'center' THEN 'C' ELSE 'R' END AS side
   FROM sources src
   JOIN source_ratings r ON r.source_id = src.id AND r.superseded_at IS NULL
   WHERE src.status = 'active' AND r.spectrum = 'IN' AND r.confidence <> 'low'),
 pick AS (
-  SELECT CASE WHEN (SELECT count(*) FROM app) >= 500 THEN 'appearances' ELSE 'pool' END AS method),
+  SELECT CASE WHEN (SELECT count(*) FROM app) >= %(pool_cutover_appearances)s THEN 'appearances'
+              ELSE 'pool' END AS method),
 chosen AS (
   SELECT side FROM app  WHERE (SELECT method FROM pick) = 'appearances'
   UNION ALL
@@ -35,9 +39,12 @@
 """
 
 
-def run(conn: psycopg.Connection) -> dict | None:
+def run(conn: psycopg.Connection, config: dict | None = None) -> dict | None:
+    config = config or load_blindspot_config()
+    params = {"baseline_window_days": config["baseline_window_days"],
+              "pool_cutover_appearances": config["pool_cutover_appearances"]}
     with conn.transaction():
-        row = conn.execute(BASELINE_SQL).fetchone()
+        row = conn.execute(BASELINE_SQL, params).fetchone()
     if row is None:
         return None
     method, appearances, p_left, p_center, p_right = row

apps/ingestion/tests/test_baseline.py

@@ -36,3 +36,30 @@ def test_run_uses_pool_method_under_500_appearances():
         result = baseline.run(conn)
     assert result["method"] == "pool"  # far under the 500-appearances cutover
     assert result["appearances"] == 0  # no article coverage yet, only the rated-source pool
+
+
+def test_run_passes_config_values_as_query_params_not_hardcoded():
+    """Proves baseline_window_days/pool_cutover_appearances are read from the passed config and
+    bound as query parameters, not hardcoded SQL literals -- catches a regression to the old
+    `interval '30 days'` / `>= 500` literals even without a full article/story fixture."""
+    assert "%(baseline_window_days)s" in baseline.BASELINE_SQL
+    assert "%(pool_cutover_appearances)s" in baseline.BASELINE_SQL
+    assert "30 days" not in baseline.BASELINE_SQL
+    assert ">= 500" not in baseline.BASELINE_SQL
+
+    captured = {}
+    real_execute = psycopg.Connection.execute
+
+    def spy_execute(self, query, params=None, **kwargs):
+        if params is not None:
+            captured.update(params)
+        return real_execute(self, query, params, **kwargs)
+
+    with psycopg.connect(DSN, autocommit=True) as conn:
+        conn.execute("DELETE FROM bias_baselines")
+        conn.execute("DELETE FROM source_ratings")
+    with psycopg.connect(DSN) as conn:
+        conn.execute = spy_execute.__get__(conn, psycopg.Connection)
+        baseline.run(conn, config={"baseline_window_days": 7, "pool_cutover_appearances": 42})
+    assert captured["baseline_window_days"] == 7
+    assert captured["pool_cutover_appearances"] == 42