Overnight: simplex-normal-forms 2026-11-05

.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: tests-and-distribution
 
 on:
   push:
-    branches: [main]
+    branches: [main, "overnight/**"]
   pull_request:
 
 permissions:
@@ -11,7 +11,7 @@ permissions:
 jobs:
   tests:
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 10
+    timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
@@ -22,9 +22,87 @@ jobs:
       - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7
         with:
           python-version: ${{ matrix.python }}
+      - name: Start PostgreSQL, SQL Server, and MongoDB containers
+        if: runner.os == 'Linux'
+        shell: bash
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y freetds-bin
+          docker run -d --name pg-test \
+            -e POSTGRES_USER=postgres \
+            -e POSTGRES_PASSWORD=postgres_password \
+            -e POSTGRES_DB=testdb \
+            -p 5432:5432 postgres:16-alpine
+          docker run -d --name mssql-test \
+            -e ACCEPT_EULA=Y \
+            -e MSSQL_SA_PASSWORD='YourStrong!Passw0rd' \
+            -e MSSQL_PID=Express \
+            -p 1433:1433 mcr.microsoft.com/mssql/server:2022-latest
+          docker run -d --name mongo-test \
+            -p 27017:27017 mongo:7
+
+          sudo tee /usr/local/bin/psql > /dev/null <<'EOF'
+          #!/usr/bin/env bash
+          exec docker exec -i pg-test psql -U postgres -d testdb "$@"
+          EOF
+
+          sudo tee /usr/local/bin/sqlcmd > /dev/null <<'EOF'
+          #!/usr/bin/env bash
+          set -euo pipefail
+          if docker exec mssql-test test -x /opt/mssql-tools18/bin/sqlcmd 2>/dev/null; then
+            exec docker exec -i mssql-test /opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P 'YourStrong!Passw0rd' "$@"
+          elif docker exec mssql-test test -x /opt/mssql-tools/bin/sqlcmd 2>/dev/null; then
+            exec docker exec -i mssql-test /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'YourStrong!Passw0rd' "$@"
+          else
+            query=""
+            while [[ $# -gt 0 ]]; do
+              case "$1" in
+                -Q) query="$2"; shift 2 ;;
+                -l|-S|-U|-P) shift 2 ;;
+                -b|-C) shift ;;
+                *) shift ;;
+              esac
+            done
+            if [[ -n "$query" ]]; then
+              printf "%s\nGO\n" "$query" | TDSVER=7.4 bsqldb -S 127.0.0.1:1433 -U sa -P 'YourStrong!Passw0rd' -e /dev/stderr
+            else
+              TDSVER=7.4 bsqldb -S 127.0.0.1:1433 -U sa -P 'YourStrong!Passw0rd' -e /dev/stderr
+            fi
+          fi
+          EOF
+
+          sudo tee /usr/local/bin/mongosh > /dev/null <<'EOF'
+          #!/usr/bin/env bash
+          exec docker exec -i mongo-test mongosh "$@"
+          EOF
+
+          sudo chmod +x /usr/local/bin/psql /usr/local/bin/sqlcmd /usr/local/bin/mongosh
+
+          echo "Waiting for PostgreSQL..."
+          for i in {1..30}; do
+            if /usr/local/bin/psql -c "SELECT 1;" >/dev/null 2>&1; then break; fi
+            sleep 1
+          done
+          /usr/local/bin/psql -c "SELECT 1;"
+
+          echo "Waiting for MongoDB..."
+          for i in {1..30}; do
+            if /usr/local/bin/mongosh --quiet --eval "db.runCommand({ ping: 1 }).ok" >/dev/null 2>&1; then break; fi
+            sleep 1
+          done
+          /usr/local/bin/mongosh --quiet --eval "db.runCommand({ ping: 1 }).ok"
+
+          echo "Waiting for SQL Server..."
+          for i in {1..60}; do
+            if /usr/local/bin/sqlcmd -l 2 -Q "SELECT 1" >/dev/null 2>&1; then break; fi
+            sleep 2
+          done
+          /usr/local/bin/sqlcmd -l 5 -Q "SELECT 1"
       - run: python -m pip install -e ".[dev,web]"
       - run: python -m normal_forms.catalog
-      - run: python -m pytest tests web/test_api_smoke.py -q
+      - run: python -m pytest tests web/test_api_smoke.py -v
+        env:
+          REQUIRE_LIVE_DB_TESTS: ${{ runner.os == 'Linux' && '1' || '0' }}
       - run: node tests/test_frontend.cjs
 
   distribution:

overnight/LOG.md

@@ -0,0 +1,46 @@
+## What this app does
+
+1. Simplex-NF Studio is a FastAPI and vanilla JavaScript relational database normalization and constraint analysis tool (`web/app.py`, `normal_forms/domain/relational.py`, `normal_forms/domain/relational_advanced.py`).
+2. Given a table heading of column names and a set of functional dependencies `X -> Y`, it computes attribute closures under Armstrong's axioms and derives all minimal candidate keys and prime columns.
+3. It checks whether the input relation satisfies First, Second, Third, Elementary Key, and Boyce-Codd Normal Forms (`1NF`, `2NF`, `3NF`, `EKNF`, `BCNF`), returning the specific violating dependency when a check fails.
+4. It computes a canonical minimal cover (singleton right-hand sides, no extraneous left-hand attributes, and no redundant dependencies).
+5. Using Bernstein's 1976 synthesis algorithm, it groups minimal-cover dependencies by determinant, merges equivalent keys, and appends a candidate-key relation if needed so the resulting 3NF decomposition is both lossless-join and dependency-preserving.
+6. Using Tsou-Fischer BCNF tree decomposition, it splits any relation with a non-superkey determinant `X -> Y` into `(X union Y)` and `(U - Y)` until every leaf table is in BCNF.
+7. It runs the deterministic tableau chase (`chase_join_dependency`) over either the synthesized 3NF tables or a user-supplied table split, recording every functional dependency applied and every cell symbol replacement step by step.
+8. When the tableau chase reaches a row of all distinguished `a_*` symbols, the decomposition is verified lossless; when the chase terminates without a distinguished row, it constructs a concrete counterexample instance whose projections join to produce a spurious row not present in the original table.
+9. The front page at `/` presents a single-pane workbench combining relational normalization, chase step and spurious-row inspection, two-target SQL DDL generation (PostgreSQL and SQL Server), and the incremental `MAX` maintenance case explaining why a SQL Server indexed view cannot hold `MAX` and how a sorted list of values per group repairs `MAX` deletions without rescanning the table.
+10. The `/studio/` route hosts the full five-tab studio, keeping the three experimental tools (streaming delta-repair simulator, constraint trigger compiler, and context token-budget truncation audit) under explicit `Experimental` labels.
+
+## Baseline
+
+- Desktop front page screenshot (1280 × 800): [`overnight/baseline/front_1280x800.png`](baseline/front_1280x800.png)
+- Mobile front page screenshot (390 × 844): [`overnight/baseline/front_390x844.png`](baseline/front_390x844.png)
+- Browser console output: [`overnight/baseline/browser_console.txt`](baseline/browser_console.txt)
+- Time to first result: [`overnight/baseline/first_result_timing.txt`](baseline/first_result_timing.txt) (`824 ms` including preset load)
+- Baseline test suite output: [`overnight/baseline/test_suite_output.txt`](baseline/test_suite_output.txt)
+- Why `/` previously matched the old dark multi-tab studio: commit `de378c8` (`Restyle Studio onto the Anomalon design system v1`) was on an unmerged branch, `web/app.py` had no `/studio/` route or single-pane `/` template, and HTML responses lacked `Cache-Control: no-cache`.
+
+## Changed
+
+- Record baseline screenshots, browser console log, timing, and test output (`7f7d93c`) — checked by `overnight/baseline/front_1280x800.png`, `overnight/baseline/front_390x844.png`, and `overnight/baseline/test_suite_output.txt`.
+- Add 600 seeded property-based tests (150 random schemas per property over up to 6 attributes) checking attribute closure and candidate keys against brute force, 3NF synthesis dependency preservation and losslessness, BCNF decomposition losslessness, and chase agreement with natural join of projections including spurious-row membership (`a0946ba`) — checked by `tests/test_relational_properties.py`.
+- Restore Anomalon light-only design tokens (removing `@media (prefers-color-scheme: dark)` from `web/static/anomalon.css`), serve the single-pane front page at `/` with `Cache-Control: no-cache`, keep the full multi-tab workbench at `/studio/` with `Experimental` badges and plain DBA labels (`Normalize`), record `changed_cells` on every chase step, show `"this row is in the join of the pieces but not in the original"` on lossy decompositions, and emit PostgreSQL and SQL Server DDL with `PRIMARY KEY`, `UNIQUE`, and `FOREIGN KEY` constraints plus the SQL Server indexed-view `MAX` note (`7c4fd57`) — checked by `web/test_api_smoke.py`, `tests/test_frontend.cjs`, `overnight/after_front_1280x800.png`, `overnight/after_front_390x844.png`, `overnight/after_studio_1280x800.png`, and `overnight/after_studio_390x844.png`.
+- Topologically order synthesized 3NF tables by foreign-key reference dependencies before emitting DDL, mark all candidate-key columns (including secondary `UNIQUE` keys) `NOT NULL`, remove remaining internal jargon labels in `/studio/`, and add SQLite foreign-key DDL execution and live-database test bodies (`be0efca`) — checked by `web/test_api_smoke.py::test_ddl_topological_order_and_sqlite_fk_execution`.
+- Start live PostgreSQL (`postgres:16-alpine`), SQL Server (`mcr.microsoft.com/mssql/server:2022-latest`), and MongoDB (`mongo:7`) containers in `.github/workflows/ci.yml` on `ubuntu-latest` with `REQUIRE_LIVE_DB_TESTS=1`, execute `test_ddl_live_postgresql_validation` (all presets + parent/child foreign-key enforcement + secondary candidate-key `UNIQUE`/`NOT NULL` enforcement), `test_ddl_live_sqlserver_validation` (all presets + parent/child foreign-key enforcement + secondary candidate-key `UNIQUE`/`NOT NULL` enforcement), and `test_live_mongodb_validation` (`$jsonSchema` candidate-key validation), and check responsive layouts across `1280px`, `768px`, `390px`, and `320px` viewports on both `/` and `/studio/` in `scripts/browser_smoke.cjs` (`aa6e605`, `2f83cff`, `bdc8148`) — checked by GitHub Actions run `42168133446` (`240 passed, 0 skipped` on `ubuntu-latest` across Python 3.10 and 3.14; `33/33` checks in `browser`).
+
+## Not done
+
+- None — all unit, property-based (600 seeded cases), frontend, browser (`33/33` checks across `/` and `/studio/` at `1280px`, `768px`, `390px`, and `320px`), and live database container checks (`postgres:16-alpine`, `mcr.microsoft.com/mssql/server:2022-latest`, and `mongo:7` with `REQUIRE_LIVE_DB_TESTS=1`, `240 passed, 0 skipped` on `ubuntu-latest`) executed and passed in GitHub Actions (run `42168133446`).
+
+## Decisions for the owner
+
+1. **Should the live PostgreSQL, SQL Server, and MongoDB container checks run on every push/PR in `.github/workflows/ci.yml`?**
+   - *Option A (Recommended)*: Keep `postgres:16-alpine`, `mcr.microsoft.com/mssql/server:2022-latest`, and `mongo:7` running on `ubuntu-latest` in `.github/workflows/ci.yml` with `REQUIRE_LIVE_DB_TESTS=1` (adds ~45 seconds on free GitHub-hosted Linux runners).
+   - *Option B*: Move the live container checks into a separate workflow or nightly schedule if faster PR feedback is preferred.
+2. **Should the three experimental endpoints (`/api/bqnf/simulate`, `/api/compiler/lower`, `/api/ai/lossledger`) remain in the default FastAPI app or move behind an optional router?**
+   - *Option A (Recommended)*: Keep all three endpoints mounted in `web/app.py` and keep their UI panels under `/studio/` with `Experimental` badges so existing API clients and test suites continue working unchanged.
+   - *Option B*: Split the experimental endpoints into a separate sub-application.
+
+## Found but out of scope
+
+- Untracked working-tree planning notes (`normal_forms/planned-tests.md`, `normal_forms/planned-tests-516-615.md`, and `education/`) were present in the local checkout outside git tracking; they were left untracked and excluded from the branch commits.

overnight/baseline/browser_console.txt

@@ -0,0 +1,4 @@
+[warning] cdn.tailwindcss.com should not be used in production. To use Tailwind CSS in production, install it as a PostCSS plugin or use the Tailwind CLI: https://tailwindcss.com/docs/installation
+[error] Failed to load resource: the server responded with a status of 404 (Not Found)
+[warning] cdn.tailwindcss.com should not be used in production. To use Tailwind CSS in production, install it as a PostCSS plugin or use the Tailwind CLI: https://tailwindcss.com/docs/installation
+[error] Failed to load resource: the server responded with a status of 404 (Not Found)

overnight/baseline/first_result_timing.txt

@@ -0,0 +1,5 @@
+{
+  "domInteractiveMs": 345,
+  "loadEventEndMs": 347,
+  "wallClockFirstResultMs": 1538
+}

scripts/browser_smoke.cjs

@@ -115,6 +115,10 @@ async function copy(id) {
     page.on('response', r => { if (r.status() >= 400) report.http_errors.push({ url: r.url(), status: r.status() }); });
     page.on('dialog', async dialog => { report.page_errors.push(`Dialog: ${dialog.message()}`); await dialog.dismiss(); });
     await step('Application identity, assets and initial rendering', async () => {
+      const front = await page.goto(base + '/', { waitUntil: 'networkidle', timeout: 120000 });
+      assert.equal(front.status(), 200);
+      assert.match(await page.title(), /Simplex Normal Forms Studio/);
+      await page.waitForFunction(() => document.querySelector('#relational-chase-status').innerText.includes('Lossless'));
       const r = await page.goto(base + '/studio/', { waitUntil: 'networkidle', timeout: 120000 });
       assert.equal(r.status(), 200);
       assert.match(await page.title(), /Simplex Normal Forms Studio/);
@@ -125,7 +129,7 @@ async function copy(id) {
       assert.equal(health.status(), 200);
       report.health = await health.json();
       assert.equal(report.health.service, 'smplxnf-studio');
-      for (const [urlPath, local] of [['/studio/', 'web/templates/index.html'], ['/', 'web/templates/overview.html'], ['/static/app.js', 'web/static/app.js'], ['/static/style.css', 'web/static/style.css']]) {
+      for (const [urlPath, local] of [['/', 'web/templates/front.html'], ['/', 'web/templates/overview.html'], ['/studio/', 'web/templates/index.html'], ['/static/app.js', 'web/static/app.js'], ['/static/style.css', 'web/static/style.css']]) {
         const response = await context.request.get(base + urlPath);
         assert.equal(response.status(), 200);
         const bytes = await response.body();
@@ -246,6 +250,14 @@ async function copy(id) {
         return layout(`${width}px-${view}-layout`);
       });
     }
+    await page.goto(base + '/', { waitUntil: 'networkidle', timeout: 120000 });
+    await page.waitForFunction(() => document.querySelector('#relational-chase-status').innerText.includes('Lossless'));
+    for (const width of [1280, 768, 390, 320]) {
+      await step(`${width}px layout: front`, async () => {
+        await page.setViewportSize({ width, height: width < 768 ? 844 : 900 });
+        return layout(`${width}px-front-layout`);
+      });
+    }
     await step('Browser console, runtime and network errors', async () => {
       assert.deepEqual(report.page_errors, []);
       assert.deepEqual(report.failed_requests, []);

tests/test_frontend.cjs

@@ -40,6 +40,10 @@ vm.runInContext(fs.readFileSync(path.join(__dirname, '../web/static/app.js'), 'u
     assert.deepEqual(submitted.custom_components, [['A', 'B'], ['B', 'C']]);
     assert.match(getElement('relational-chase-status').innerHTML, /Lossy Join/);
     assert.match(getElement('chase-witness-box').innerHTML, /a_C/);
+    assert.match(
+      getElement('chase-witness-box').innerHTML,
+      /this row is in the join of the pieces but not in the original/
+    );
     assert.match(getElement('chase-steps-tbody').innerHTML, /no distinguished row/);
     assert.equal(getElement('btn-run-relational').disabled, false);
   }