From 695371ad338a343958e271e88d10a1e28cfaccd1 Mon Sep 17 00:00:00 2001 From: saran Date: Mon, 12 Oct 2026 09:29:58 +0530 Subject: [PATCH 1/4] built missing product event schema --- infra/migrations/0003_product_events.sql | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 infra/migrations/0003_product_events.sql diff --git a/infra/migrations/0003_product_events.sql b/infra/migrations/0003_product_events.sql new file mode 100644 index 0000000..6290de6 --- /dev/null +++ b/infra/migrations/0003_product_events.sql @@ -0,0 +1,26 @@ +-- ARCHITECTURE.md §4 (Data Model), PRODUCT.md §8.1. Copied verbatim from the doc's schema block. + +-- First-party analytics (PRODUCT §8.1). Raw rows kept 90 days (§11); daily aggregates are out +-- of scope until something reads them (complexity budget, §1). Deleting an account deletes its +-- events (PRODUCT §8.1 rule), so user_id cascades instead of SET NULL like auth_events. +CREATE TABLE product_events ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + event TEXT NOT NULL CHECK (event IN ( + 'feed_view','story_impression','story_open','article_click', + 'methodology_view','rating_panel_view','save','follow_source', + 'follow_topic','signup','cluster_report','rating_concern')), + occurred_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- server receive time; client clocks ignored + session_id UUID NOT NULL, + user_id UUID REFERENCES users(id) ON DELETE CASCADE, + client_event_id UUID NOT NULL, -- sendBeacon retry dedup key (§8.1 rule 2) + story_id UUID REFERENCES stories(id), + source_id UUID REFERENCES sources(id), + topic_id UUID REFERENCES topics(id), + feed TEXT, + position INT, + sides_available INT, + source_side TEXT CHECK (source_side IN ('L','C','R','unrated')) +); +CREATE UNIQUE INDEX idx_product_events_client_event_id ON product_events(client_event_id); +CREATE INDEX idx_product_events_session_story ON product_events(session_id, story_id, event, occurred_at); +CREATE INDEX idx_product_events_occurred_at ON product_events(occurred_at); From 4cd7611c8585f7ea7d73e2dffb925deb570598aa Mon Sep 17 00:00:00 2001 From: saran Date: Mon, 12 Oct 2026 09:50:43 +0530 Subject: [PATCH 2/4] fixed openapi wiring issue --- apps/web/src/features/auth/types.ts | 7 +- apps/web/src/features/curator/types.ts | 22 ++-- apps/web/src/features/feed/types.ts | 70 ++--------- apps/web/src/features/topics/types.ts | 9 +- packages/shared-types/src/openapi.ts | 163 +++++++++++++------------ 5 files changed, 110 insertions(+), 161 deletions(-) diff --git a/apps/web/src/features/auth/types.ts b/apps/web/src/features/auth/types.ts index 0f0e6bd..e7e29e5 100644 --- a/apps/web/src/features/auth/types.ts +++ b/apps/web/src/features/auth/types.ts @@ -1,3 +1,4 @@ -export interface LoginResponse { - accessToken: string -} +// Re-exported from the OpenAPI spec (docs/openapi.yaml, generated into packages/shared-types). +import type { components } from '../../../../../packages/shared-types/src/openapi' + +export type LoginResponse = components['schemas']['LoginResponse'] diff --git a/apps/web/src/features/curator/types.ts b/apps/web/src/features/curator/types.ts index 86e2238..1b2ab93 100644 --- a/apps/web/src/features/curator/types.ts +++ b/apps/web/src/features/curator/types.ts @@ -39,18 +39,12 @@ export interface NewClusterReport { reportedBy: string } -export interface BlindspotItem { - id: string - title: string - lastCoverageAt: string - side: 'L' | 'R' - countedLeft: number - countedCenter: number - countedRight: number -} +// Re-exported from the OpenAPI spec (docs/openapi.yaml, generated into packages/shared-types) +// rather than hand-duplicated. NewSource/NewRating/NewClusterReport above stay hand-written: +// NewRating.rubricScores is genuinely free-form JSON, which openapi-typescript renders as +// Record (an object that can hold no properties) instead of the usable +// Record this feature actually needs. +import type { components } from '../../../../../packages/shared-types/src/openapi' -export interface BlindspotFeedResponse { - items: BlindspotItem[] - asOf: string - nextCursor: string | null -} +export type BlindspotItem = components['schemas']['BlindspotItem'] +export type BlindspotFeedResponse = components['schemas']['BlindspotFeedResponse'] diff --git a/apps/web/src/features/feed/types.ts b/apps/web/src/features/feed/types.ts index 4ed09fc..1817aa3 100644 --- a/apps/web/src/features/feed/types.ts +++ b/apps/web/src/features/feed/types.ts @@ -1,58 +1,12 @@ -export interface BiasBarSegment { - count: number - lowConfidenceCount: number -} - -export interface BiasBar { - left: BiasBarSegment - centerLeft: BiasBarSegment - center: BiasBarSegment - centerRight: BiasBarSegment - right: BiasBarSegment - notRated: BiasBarSegment -} - -export interface FeedItem { - id: string - title: string - summary: string | null - lastCoverageAt: string - ratedSourceCount: number - unratedSourceCount: number - biasBar: BiasBar -} - -export interface FeedResponse { - items: FeedItem[] - asOf: string - nextCursor: string | null - updatedCount: number -} - -export interface SourceCard { - sourceId: string - sourceName: string - primaryArticle: { id: string; title: string; url: string; publishedAt: string | null } - moreCount: number -} - -export interface BiasColumns { - left: SourceCard[] - centerLeft: SourceCard[] - center: SourceCard[] - centerRight: SourceCard[] - right: SourceCard[] - notRated: SourceCard[] -} - -export interface StoryDetail { - id: string - title: string - summary: string | null - status: 'building' | 'active' | 'tombstoned' - redirectTo: { id: string; title: string } | null - splitNotices: { storyId: string; title: string; articleCount: number }[] - ratedSourceCount: number - unratedSourceCount: number - biasColumns: BiasColumns -} +// Re-exported from the OpenAPI spec (docs/openapi.yaml, generated into +// packages/shared-types) rather than hand-duplicated, so a schema change can't silently +// drift from what this feature assumes the API returns. +import type { components } from '../../../../../packages/shared-types/src/openapi' + +export type BiasBarSegment = components['schemas']['BiasBarSegment'] +export type BiasBar = components['schemas']['BiasBar'] +export type FeedItem = components['schemas']['FeedItem'] +export type FeedResponse = components['schemas']['FeedResponse'] +export type SourceCard = components['schemas']['SourceCard'] +export type BiasColumns = components['schemas']['BiasColumns'] +export type StoryDetail = components['schemas']['StoryDetailResponse'] diff --git a/apps/web/src/features/topics/types.ts b/apps/web/src/features/topics/types.ts index 783563b..8ce504d 100644 --- a/apps/web/src/features/topics/types.ts +++ b/apps/web/src/features/topics/types.ts @@ -1,5 +1,4 @@ -export interface Topic { - id: string - name: string - keywords: string[] -} +// Re-exported from the OpenAPI spec (docs/openapi.yaml, generated into packages/shared-types). +import type { components } from '../../../../../packages/shared-types/src/openapi' + +export type Topic = components['schemas']['Topic'] diff --git a/packages/shared-types/src/openapi.ts b/packages/shared-types/src/openapi.ts index 276a483..77aebb5 100644 --- a/packages/shared-types/src/openapi.ts +++ b/packages/shared-types/src/openapi.ts @@ -172,10 +172,7 @@ export interface paths { [name: string]: unknown; }; content: { - "application/json": { - /** @description 15-minute JWT carrying `sub` (user id), `av` (auth_version), `kid`. */ - accessToken?: string; - }; + "application/json": components["schemas"]["LoginResponse"]; }; }; 401: components["responses"]["AuthError"]; @@ -1276,185 +1273,189 @@ export interface components { Error: { error?: string; }; + LoginResponse: { + /** @description 15-minute JWT carrying `sub` (user id), `av` (auth_version), `kid`. */ + accessToken: string; + }; BiasBarSegment: { /** @description Distinct rated sources in this bias category that covered the story with a visible article. */ - count?: number; + count: number; /** @description Subset of `count` whose current rating has low confidence — shown hatched on the bar, never excluded from `count` (ARCHITECTURE §10 bias bar). */ - lowConfidenceCount?: number; + lowConfidenceCount: number; }; /** @description 6 buckets — the 5 bias categories plus `notRated` (PRODUCT §3.3). */ BiasBar: { - left?: components["schemas"]["BiasBarSegment"]; - centerLeft?: components["schemas"]["BiasBarSegment"]; - center?: components["schemas"]["BiasBarSegment"]; - centerRight?: components["schemas"]["BiasBarSegment"]; - right?: components["schemas"]["BiasBarSegment"]; - notRated?: components["schemas"]["BiasBarSegment"]; + left: components["schemas"]["BiasBarSegment"]; + centerLeft: components["schemas"]["BiasBarSegment"]; + center: components["schemas"]["BiasBarSegment"]; + centerRight: components["schemas"]["BiasBarSegment"]; + right: components["schemas"]["BiasBarSegment"]; + notRated: components["schemas"]["BiasBarSegment"]; }; FeedItem: { /** Format: uuid */ - id?: string; - title?: string; - summary?: string | null; + id: string; + title: string; + summary: string | null; /** * Format: date-time * @description When fresh coverage last joined the story — not publication or event time (ARCHITECTURE §10 pinned meanings). */ - lastCoverageAt?: string; + lastCoverageAt: string; /** @description Distinct sources with a current rating and a visible article in the story. */ - ratedSourceCount?: number; + ratedSourceCount: number; /** @description Distinct sources with a visible article and no current rating. */ - unratedSourceCount?: number; - biasBar?: components["schemas"]["BiasBar"]; + unratedSourceCount: number; + biasBar: components["schemas"]["BiasBar"]; }; FeedResponse: { - items?: components["schemas"]["FeedItem"][]; + items: components["schemas"]["FeedItem"][]; /** * Format: date-time * @description The frozen upper bound this page was read against. Clients never compute or adjust this themselves — it round-trips inside `nextCursor` and is only used to phrase the "N updated" banner. */ - asOf?: string; + asOf: string; /** @description Opaque cursor for the next page, or null at the end of the feed. */ - nextCursor?: string | null; + nextCursor: string | null; /** @description Stories with coverage newer than `asOf` that exist right now — drives the "N stories updated, refresh" banner (ARCHITECTURE §10). Not a count of items on this page. */ - updatedCount?: number; + updatedCount: number; }; SourceCard: { /** Format: uuid */ - sourceId?: string; - sourceName?: string; - primaryArticle?: { + sourceId: string; + sourceName: string; + primaryArticle: { /** Format: uuid */ - id?: string; - title?: string; + id: string; + title: string; /** Format: uri */ - url?: string; + url: string; /** * Format: date-time * @description The article's effective time — the publisher's date, clamped to when we first saw it (ARCHITECTURE §10 pinned meanings). */ - publishedAt?: string | null; + publishedAt: string | null; }; /** @description Additional visible articles from this source beyond the primary card, after collapsing duplicates that share a `content_hash` (ARCHITECTURE §10 story detail). */ - moreCount?: number; + moreCount: number; }; /** @description Per-source cards grouped into the 5 bias categories plus `notRated`. */ BiasColumns: { - left?: components["schemas"]["SourceCard"][]; - centerLeft?: components["schemas"]["SourceCard"][]; - center?: components["schemas"]["SourceCard"][]; - centerRight?: components["schemas"]["SourceCard"][]; - right?: components["schemas"]["SourceCard"][]; - notRated?: components["schemas"]["SourceCard"][]; + left: components["schemas"]["SourceCard"][]; + centerLeft: components["schemas"]["SourceCard"][]; + center: components["schemas"]["SourceCard"][]; + centerRight: components["schemas"]["SourceCard"][]; + right: components["schemas"]["SourceCard"][]; + notRated: components["schemas"]["SourceCard"][]; }; StoryDetailResponse: { /** Format: uuid */ - id?: string; - title?: string; - summary?: string | null; + id: string; + title: string; + summary: string | null; /** @enum {string} */ - status?: "building" | "active" | "tombstoned"; + status: "building" | "active" | "tombstoned"; /** @description Set only when `status` is `tombstoned` and the story was merged into a successor. */ - redirectTo?: { + redirectTo: { /** Format: uuid */ - id?: string; - title?: string; + id: string; + title: string; } | null; - splitNotices?: { + splitNotices: { /** Format: uuid */ - storyId?: string; - title?: string; + storyId: string; + title: string; /** @description Number of articles that moved to this split-off story. */ - articleCount?: number; + articleCount: number; }[]; /** @description Distinct sources with a current rating and a visible article in the story. */ - ratedSourceCount?: number; + ratedSourceCount: number; /** @description Distinct sources with a visible article and no current rating. */ - unratedSourceCount?: number; - biasColumns?: components["schemas"]["BiasColumns"]; + unratedSourceCount: number; + biasColumns: components["schemas"]["BiasColumns"]; }; Source: { /** Format: uuid */ - id?: string; - name?: string; - domain?: string; + id: string; + name: string; + domain: string; /** @enum {string} */ - status?: "active" | "deprecated"; + status: "active" | "deprecated"; /** * Format: date-time * @description When this source stopped being polled — its past stories still match follows (PRODUCT §7.2 "no longer updated" badge). */ - deprecatedAt?: string | null; - logoUrl?: string | null; + deprecatedAt: string | null; + logoUrl: string | null; }; BlindspotItem: { /** Format: uuid */ - id?: string; - title?: string; + id: string; + title: string; /** * Format: date-time * @description When fresh coverage last joined the story — not publication or event time (ARCHITECTURE §10 pinned meanings). */ - lastCoverageAt?: string; + lastCoverageAt: string; /** * @description The side with little coverage (PRODUCT §4.7 — never phrased as the covering side's fault). * @enum {string} */ - side?: "L" | "R"; + side: "L" | "R"; /** @description Distinct left-leaning sources counted toward this story's blindspot test (PRODUCT §4.2). */ - countedLeft?: number; + countedLeft: number; /** @description Distinct center sources counted toward this story's blindspot test (PRODUCT §4.2). */ - countedCenter?: number; + countedCenter: number; /** @description Distinct right-leaning sources counted toward this story's blindspot test (PRODUCT §4.2). */ - countedRight?: number; + countedRight: number; }; BlindspotFeedResponse: { - items?: components["schemas"]["BlindspotItem"][]; + items: components["schemas"]["BlindspotItem"][]; /** * Format: date-time * @description The frozen upper bound this page was read against (ARCHITECTURE §10 cursor mechanics). */ - asOf?: string; + asOf: string; /** @description Opaque cursor for the next page, or null at the end of the feed. */ - nextCursor?: string | null; + nextCursor: string | null; }; Topic: { /** Format: uuid */ - id?: string; - name?: string; + id: string; + name: string; /** @description Already normalized (NFC, casefolded, deduped) — see POST /api/curator/topics. */ - keywords?: string[]; + keywords: string[]; }; Rating: { /** Format: uuid */ - id?: string; + id: string; /** Format: uuid */ - sourceId?: string; + sourceId: string; /** @enum {string|null} */ - spectrum?: "IN" | null; + spectrum: "IN" | null; /** @enum {string|null} */ - biasRating?: "left" | "center-left" | "center" | "center-right" | "right" | null; + biasRating: "left" | "center-left" | "center" | "center-right" | "right" | null; /** @enum {string} */ - factuality?: "high" | "mixed" | "low"; + factuality: "high" | "mixed" | "low"; /** @enum {string|null} */ - confidence?: "high" | "medium" | "low" | null; - method?: string; - ratedBy?: string; + confidence: "high" | "medium" | "low" | null; + method: string; + ratedBy: string; /** * Format: date-time * @description When this rating row was inserted — ratings are append-only, never mutated in place. */ - ratedAt?: string; + ratedAt: string; /** * Format: uuid * @description Id of the rating row this one replaced, or null if this is the source's first rating. */ - supersedes?: string | null; + supersedes: string | null; /** * Format: date-time * @description When a later rating replaced this one; null while this is the current rating. */ - supersededAt?: string | null; + supersededAt: string | null; }; NewRating: { /** From 34e438e2f8b85c2ae5b10f2ea8bc331a520975f6 Mon Sep 17 00:00:00 2001 From: saran Date: Mon, 12 Oct 2026 13:25:23 +0530 Subject: [PATCH 3/4] wired both config/product/blindspot.yaml into apps/api and apps/ingestion --- apps/api/src/features/blindspot/config.ts | 33 ++++++++++++++++++++++ apps/api/src/features/blindspot/repo.ts | 16 +++++++++-- apps/api/src/features/blindspot/service.ts | 12 +++++++- apps/api/test/blindspot-config.test.ts | 25 ++++++++++++++++ apps/api/test/blindspot.test.ts | 26 +++++++++++++++++ apps/ingestion/config.py | 8 ++++++ apps/ingestion/jobs/baseline.py | 17 +++++++---- apps/ingestion/tests/test_baseline.py | 27 ++++++++++++++++++ apps/ingestion/tests/test_config.py | 9 ++++++ 9 files changed, 164 insertions(+), 9 deletions(-) create mode 100644 apps/api/src/features/blindspot/config.ts create mode 100644 apps/api/test/blindspot-config.test.ts diff --git a/apps/api/src/features/blindspot/config.ts b/apps/api/src/features/blindspot/config.ts new file mode 100644 index 0000000..7977b71 --- /dev/null +++ b/apps/api/src/features/blindspot/config.ts @@ -0,0 +1,33 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +// config/product/blindspot.yaml (PRODUCT §4.4) is 5 flat numeric keys with trailing comments -- +// simple enough that a YAML dependency isn't worth adding for it. +export interface BlindspotParams { + nMin: number; + materiality: number; + z: number; + baselineWindowDays: number; + poolCutoverAppearances: number; +} + +const CONFIG_PATH = fileURLToPath(new URL("../../../../../config/product/blindspot.yaml", import.meta.url)); + +export function loadBlindspotConfig(path: string = CONFIG_PATH): BlindspotParams { + const text = readFileSync(path, "utf-8"); + const values: Record = {}; + for (const line of text.split("\n")) { + const match = /^(\w+):\s*([0-9.]+)/.exec(line); + if (match) values[match[1]] = Number(match[2]); + } + for (const key of ["n_min", "materiality", "z", "baseline_window_days", "pool_cutover_appearances"]) { + if (!(key in values)) throw new Error(`blindspot.yaml missing required key: ${key}`); + } + return { + nMin: values.n_min, + materiality: values.materiality, + z: values.z, + baselineWindowDays: values.baseline_window_days, + poolCutoverAppearances: values.pool_cutover_appearances, + }; +} diff --git a/apps/api/src/features/blindspot/repo.ts b/apps/api/src/features/blindspot/repo.ts index fc37a67..f9c227f 100644 --- a/apps/api/src/features/blindspot/repo.ts +++ b/apps/api/src/features/blindspot/repo.ts @@ -1,5 +1,7 @@ import type { Pool } from "pg"; +import type { BlindspotParams } from "./config.js"; + export interface Baseline { p_left: number; p_center: number; @@ -30,7 +32,13 @@ export interface BlindspotRow { // binds them): stories active in the last 7 days, newest coverage first. export async function fetchBlindspotPage( pool: Pool, - args: { asOf: Date; cursorCoverageAt: Date | null; cursorId: string | null; limit: number } + args: { + asOf: Date; + cursorCoverageAt: Date | null; + cursorId: string | null; + limit: number; + params: BlindspotParams; + } ): Promise { const result = await pool.query<{ id: string; @@ -60,7 +68,8 @@ export async function fetchBlindspotPage( GROUP BY a.story_id), flagged AS ( SELECT c.*, blindspot_side(c.k_left::int, c.k_center::int, c.k_right::int, - b.p_left, b.p_center, b.p_right) AS side + b.p_left, b.p_center, b.p_right, + $5::int, $6::float8, $7::float8) AS side FROM counts c CROSS JOIN base b) SELECT s.id, s.title, s.last_coverage_at, f.side, f.k_left, f.k_center, f.k_right FROM flagged f JOIN stories s ON s.id = f.story_id @@ -68,7 +77,8 @@ export async function fetchBlindspotPage( AND ($2::timestamptz IS NULL OR (s.last_coverage_at, s.id) < ($2, $3)) ORDER BY s.last_coverage_at DESC, s.id DESC LIMIT $4`, - [args.asOf, args.cursorCoverageAt, args.cursorId, args.limit] + [args.asOf, args.cursorCoverageAt, args.cursorId, args.limit, + args.params.nMin, args.params.materiality, args.params.z] ); return result.rows.map((r) => ({ ...r, diff --git a/apps/api/src/features/blindspot/service.ts b/apps/api/src/features/blindspot/service.ts index 68387df..1b508a9 100644 --- a/apps/api/src/features/blindspot/service.ts +++ b/apps/api/src/features/blindspot/service.ts @@ -1,10 +1,14 @@ import type { Pool } from "pg"; import { buildPage, decodeCursor } from "../../pagination.js"; +import { loadBlindspotConfig } from "./config.js"; import { fetchBlindspotPage, type BlindspotRow } from "./repo.js"; const PAGE_SIZE = 20; const FEED_NAME = "blindspot"; +// Loaded once per process, not per request -- config/product/blindspot.yaml (PRODUCT §4.4) +// changes only through a deploy, same as every other config/*.yaml this codebase reads. +const BLINDSPOT_PARAMS = loadBlindspotConfig(); export interface BlindspotItem { id: string; @@ -32,7 +36,13 @@ export async function getBlindspotFeed(pool: Pool, cursorRaw: string | undefined const cursorCoverageAt = cursor === null ? null : new Date(cursor.c); const cursorId = cursor === null ? null : cursor.id; - const rows = await fetchBlindspotPage(pool, { asOf, cursorCoverageAt, cursorId, limit: PAGE_SIZE + 1 }); + const rows = await fetchBlindspotPage(pool, { + asOf, + cursorCoverageAt, + cursorId, + limit: PAGE_SIZE + 1, + params: BLINDSPOT_PARAMS, + }); const page = buildPage(rows, PAGE_SIZE, FEED_NAME, asOf); const items: BlindspotItem[] = page.items.map((row: BlindspotRow) => ({ diff --git a/apps/api/test/blindspot-config.test.ts b/apps/api/test/blindspot-config.test.ts new file mode 100644 index 0000000..b81030a --- /dev/null +++ b/apps/api/test/blindspot-config.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; + +import { loadBlindspotConfig } from "../src/features/blindspot/config.js"; + +describe("loadBlindspotConfig (PRODUCT §4.4)", () => { + it("reads the real config/product/blindspot.yaml, not hardcoded defaults", () => { + const params = loadBlindspotConfig(); + expect(params).toEqual({ + nMin: 6, + materiality: 0.5, + z: 1.645, + baselineWindowDays: 30, + poolCutoverAppearances: 500, + }); + }); + + it("throws if a required key is missing, rather than silently defaulting", async () => { + const fs = await import("node:fs/promises"); + const os = await import("node:os"); + const path = await import("node:path"); + const tmpFile = path.join(await fs.mkdtemp(path.join(os.tmpdir(), "blindspot-")), "blindspot.yaml"); + await fs.writeFile(tmpFile, "n_min: 6\nmateriality: 0.5\n"); + expect(() => loadBlindspotConfig(tmpFile)).toThrow(/missing required key/); + }); +}); diff --git a/apps/api/test/blindspot.test.ts b/apps/api/test/blindspot.test.ts index 05dca58..3b91a9b 100644 --- a/apps/api/test/blindspot.test.ts +++ b/apps/api/test/blindspot.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest"; import { buildApp } from "../src/app.js"; import { getPool } from "../src/db.js"; +import { loadBlindspotConfig } from "../src/features/blindspot/config.js"; +import { fetchBlindspotPage } from "../src/features/blindspot/repo.js"; import { getBlindspotFeed } from "../src/features/blindspot/service.js"; import { addCoverage, createTestSource, createTestStory, rateSource } from "./feed-helpers.js"; @@ -45,6 +47,30 @@ describe("getBlindspotFeed (PRODUCT §4)", () => { expect(result.items.find((i) => i.id === storyId)).toBeUndefined(); }); + it("n_min/materiality/z are actually read from config/product/blindspot.yaml, not a silently " + + "ignored argument or a hardcoded SQL default", async () => { + await insertBaseline(); + // Same minimal 6-source flag as the first test above, which passes at the real n_min (6). + const sources = await Promise.all(Array.from({ length: 6 }, () => createTestSource("blindspot-cfgwire"))); + await Promise.all(sources.map((s) => rateSource(s, { biasRating: "left", confidence: "high" }))); + const { storyId } = await createTestStory(sources[0]); + await Promise.all(sources.slice(1).map((s) => addCoverage(storyId, s))); + + const real = loadBlindspotConfig(); + const flagsAtRealNMin = await fetchBlindspotPage(getPool(), { + asOf: new Date(), cursorCoverageAt: null, cursorId: null, limit: 50, params: real, + }); + expect(flagsAtRealNMin.some((r) => r.id === storyId)).toBe(true); + + // Raising n_min by 1 past this story's count must suppress the flag -- if the SQL were + // still using its own DEFAULT (6) instead of this bound parameter, it would stay flagged. + const stricter = { ...real, nMin: real.nMin + 1 }; + const flagsAtStricterNMin = await fetchBlindspotPage(getPool(), { + asOf: new Date(), cursorCoverageAt: null, cursorId: null, limit: 50, params: stricter, + }); + expect(flagsAtStricterNMin.some((r) => r.id === storyId)).toBe(false); + }); + it("does not flag balanced coverage (no side is one-sided)", async () => { await insertBaseline(); const left = await Promise.all(Array.from({ length: 3 }, () => createTestSource("blindspot-bal-left"))); diff --git a/apps/ingestion/config.py b/apps/ingestion/config.py index a0296f8..4eb77a5 100644 --- a/apps/ingestion/config.py +++ b/apps/ingestion/config.py @@ -9,6 +9,7 @@ CLUSTERING_DIR = CONFIG_ROOT / "clustering" EMBED_INPUT_DIR = CONFIG_ROOT / "embed_input" SOURCES_DIR = CONFIG_ROOT / "sources" +PRODUCT_DIR = CONFIG_ROOT / "product" def load_clustering_config(version: str = "v1") -> dict: @@ -32,3 +33,10 @@ def load_source_adapter(adapter: str) -> dict: overrides = yaml.safe_load(f) or {} merged.update(overrides) return merged + + +def load_blindspot_config() -> dict: + """PRODUCT §4.4's pinned decision-rule parameters. A change ships only after beating the + §4.9 calibration set with the new values -- never hand-edit code to tune these.""" + with open(PRODUCT_DIR / "blindspot.yaml", encoding="utf-8") as f: + return yaml.safe_load(f) diff --git a/apps/ingestion/jobs/baseline.py b/apps/ingestion/jobs/baseline.py index cda5d9b..dc3a88e 100644 --- a/apps/ingestion/jobs/baseline.py +++ b/apps/ingestion/jobs/baseline.py @@ -1,9 +1,12 @@ """Daily blindspot baseline (PRODUCT §4.3). Verbatim from the doc — the cold-start -pool/appearances split and the 500-appearance cutover are policy, not reimplemented here.""" +pool/appearances split and the 500-appearance cutover are policy, read from +config/product/blindspot.yaml, not hardcoded here.""" from __future__ import annotations import psycopg +from config import load_blindspot_config + BASELINE_SQL = """ INSERT INTO bias_baselines (spectrum, method, appearances, p_left, p_center, p_right) WITH app AS ( @@ -14,7 +17,7 @@ JOIN stories s ON s.id = a.story_id AND s.status = 'active' AND s.hidden_at IS NULL JOIN source_ratings r ON r.source_id = a.source_id AND r.superseded_at IS NULL WHERE a.hidden_at IS NULL AND r.spectrum = 'IN' AND r.confidence <> 'low' - AND a.coverage_at >= now() - interval '30 days'), + AND a.coverage_at >= now() - (%(baseline_window_days)s * interval '1 day')), pool AS ( SELECT CASE WHEN r.bias_rating IN ('left','center-left') THEN 'L' WHEN r.bias_rating = 'center' THEN 'C' ELSE 'R' END AS side @@ -22,7 +25,8 @@ JOIN source_ratings r ON r.source_id = src.id AND r.superseded_at IS NULL WHERE src.status = 'active' AND r.spectrum = 'IN' AND r.confidence <> 'low'), pick AS ( - SELECT CASE WHEN (SELECT count(*) FROM app) >= 500 THEN 'appearances' ELSE 'pool' END AS method), + SELECT CASE WHEN (SELECT count(*) FROM app) >= %(pool_cutover_appearances)s THEN 'appearances' + ELSE 'pool' END AS method), chosen AS ( SELECT side FROM app WHERE (SELECT method FROM pick) = 'appearances' UNION ALL @@ -35,9 +39,12 @@ """ -def run(conn: psycopg.Connection) -> dict | None: +def run(conn: psycopg.Connection, config: dict | None = None) -> dict | None: + config = config or load_blindspot_config() + params = {"baseline_window_days": config["baseline_window_days"], + "pool_cutover_appearances": config["pool_cutover_appearances"]} with conn.transaction(): - row = conn.execute(BASELINE_SQL).fetchone() + row = conn.execute(BASELINE_SQL, params).fetchone() if row is None: return None method, appearances, p_left, p_center, p_right = row diff --git a/apps/ingestion/tests/test_baseline.py b/apps/ingestion/tests/test_baseline.py index 40c1a86..0ce78f6 100644 --- a/apps/ingestion/tests/test_baseline.py +++ b/apps/ingestion/tests/test_baseline.py @@ -36,3 +36,30 @@ def test_run_uses_pool_method_under_500_appearances(): result = baseline.run(conn) assert result["method"] == "pool" # far under the 500-appearances cutover assert result["appearances"] == 0 # no article coverage yet, only the rated-source pool + + +def test_run_passes_config_values_as_query_params_not_hardcoded(): + """Proves baseline_window_days/pool_cutover_appearances are read from the passed config and + bound as query parameters, not hardcoded SQL literals -- catches a regression to the old + `interval '30 days'` / `>= 500` literals even without a full article/story fixture.""" + assert "%(baseline_window_days)s" in baseline.BASELINE_SQL + assert "%(pool_cutover_appearances)s" in baseline.BASELINE_SQL + assert "30 days" not in baseline.BASELINE_SQL + assert ">= 500" not in baseline.BASELINE_SQL + + captured = {} + real_execute = psycopg.Connection.execute + + def spy_execute(self, query, params=None, **kwargs): + if params is not None: + captured.update(params) + return real_execute(self, query, params, **kwargs) + + with psycopg.connect(DSN, autocommit=True) as conn: + conn.execute("DELETE FROM bias_baselines") + conn.execute("DELETE FROM source_ratings") + with psycopg.connect(DSN) as conn: + conn.execute = spy_execute.__get__(conn, psycopg.Connection) + baseline.run(conn, config={"baseline_window_days": 7, "pool_cutover_appearances": 42}) + assert captured["baseline_window_days"] == 7 + assert captured["pool_cutover_appearances"] == 42 diff --git a/apps/ingestion/tests/test_config.py b/apps/ingestion/tests/test_config.py index bf4b241..0a13ef2 100644 --- a/apps/ingestion/tests/test_config.py +++ b/apps/ingestion/tests/test_config.py @@ -21,6 +21,15 @@ def test_load_source_adapter_falls_back_to_generic(): assert cfg["date_fields"] == ["published", "updated"] +def test_load_blindspot_config_has_required_keys(): + cfg = config.load_blindspot_config() + assert cfg["n_min"] == 6 + assert cfg["materiality"] == 0.5 + assert cfg["z"] == 1.645 + assert cfg["baseline_window_days"] == 30 + assert cfg["pool_cutover_appearances"] == 500 + + def test_load_source_adapter_merges_override(tmp_path, monkeypatch): monkeypatch.setattr(config, "SOURCES_DIR", tmp_path) (tmp_path / "generic.yaml").write_text( From b18eac6911ae7a0a0bcb2825a880fea392b7056e Mon Sep 17 00:00:00 2001 From: saran Date: Mon, 12 Oct 2026 13:51:39 +0530 Subject: [PATCH 4/4] created repo readme explaining the product --- README.md | 124 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..adfbcee --- /dev/null +++ b/README.md @@ -0,0 +1,124 @@ +# Newstack + +A news-aggregation platform: story clustering, source bias/factuality tagging, blindspot +detection, and personalization over Indian news sources. + +The two docs in `docs/` are the spec, not just reference material: + +- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) — execution: schema (DDL), migrations, jobs, + SQL functions, caching, pagination, API mechanics. +- [`docs/PRODUCT.md`](docs/PRODUCT.md) — meaning: definitions, formulas, thresholds, rubrics, + metrics (bias bar, blindspot math, ranking). + +Each concept is defined in exactly one of the two; the other links to it. + +## Monorepo layout + +``` +apps/ + ingestion/ Python — fetch, canonicalize, embed, cluster, baseline/ratings jobs (APScheduler) + api/ Node/TS (Fastify) — auth, feed/story/topic/rating/saved/curator REST API + web/ React 18 + TS (Vite) — the frontend +packages/ + shared-types/ TS types generated from docs/openapi.yaml (openapi-typescript) +config/ YAML config for clustering, embedding input, product/blindspot params, sources +infra/ docker-compose, Postgres migrations, migration runner +validation/ tests that validate the SQL/logic embedded in the docs themselves +docs/ ARCHITECTURE.md, PRODUCT.md, openapi.yaml, runbooks, this repo's todo tracker +``` + +## Prerequisites + +- Docker + Docker Compose (easiest path — runs everything below for you), **or** +- Node.js 20+, Python 3.11+, PostgreSQL 16 with `pgvector`, Redis 7, for running apps locally + without containers. + +## Install & run (Docker Compose — recommended) + +```bash +cd infra +docker compose up --build +``` + +This starts Postgres (pgvector), Redis, the ingestion worker, the API, the mailer process, and +the web frontend. Apply migrations once the db is healthy: + +```bash +DATABASE_URL=postgresql://newstack:newstack@localhost:5432/newstack python infra/migrate.py +``` + +Then: + +- Web app: http://localhost:5173 +- API: http://localhost:3000 (health check at `/health`) + +All services use insecure dev-default secrets (JWT key, curator API key, cursor-signing key) when +their env vars are unset — fine for local dev, **never for production**. See `apps/api/src/config.ts`, +`features/auth/jwt.ts`, `pagination.ts`, and `plugins/curator.ts` for the env vars to override +(`JWT_KEYS`, `CURATOR_API_KEY`, `CURSOR_KEYS`, `RESEND_API_KEY`/`MAIL_FROM` for real email). + +## Install & run (without Docker) + +**Database** + +```bash +# Postgres 16 with pgvector running locally, then: +DATABASE_URL=postgresql://newstack:newstack@localhost:5432/newstack python infra/migrate.py +``` + +**`apps/api`** + +```bash +cd apps/api +npm install +npm run dev # tsx watch, http://localhost:3000 +``` + +**`apps/web`** + +```bash +cd apps/web +npm install +npm run dev # Vite dev server, http://localhost:5173 +``` + +**`apps/ingestion`** + +```bash +cd apps/ingestion +pip install -r requirements.txt +python -m service # scheduler/orchestration entrypoint +python -m cli.story_browser # inspect clustered stories from the CLI +``` + +**`packages/shared-types`** + +```bash +cd packages/shared-types +npm install +npm run generate # regenerate src/openapi.ts from docs/openapi.yaml +``` + +## Running tests + +```bash +cd apps/api && npm test # Vitest +cd apps/web && npm test # Vitest +cd apps/ingestion && pytest # pytest +``` + +## Validating the docs + +`validation/` checks the SQL and logic embedded in `docs/ARCHITECTURE.md` / `docs/PRODUCT.md` +directly — nothing here is committed generated code; it's re-extracted from the docs on every run. + +```bash +./validation/run_all.sh docs/ARCHITECTURE.md docs/PRODUCT.md +``` + +The DB suite needs PostgreSQL 16 binaries at `/usr/lib/postgresql/16/bin` (Linux-oriented; this is +what CI uses) — see `docs/mvp-todo.md` and `CLAUDE.md` for details. + +## Status + +See [`docs/mvp-todo.md`](docs/mvp-todo.md) for what's implemented and what's left.