diff --git a/.changeset/pg-enums-unique-constraints.md b/.changeset/pg-enums-unique-constraints.md new file mode 100644 index 0000000..15ed6c9 --- /dev/null +++ b/.changeset/pg-enums-unique-constraints.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +Postgres enum types and unique constraints are first-class. `PG.enumType(name, labels, { schema? })` is a column type (rows decode to its labels; inserts and comparisons take only them; `nullable` and `array` wrap it, and enum arrays decode from the text form drivers send) and a schema object: `generate` creates the type before the tables that use it, adds labels with `ADD VALUE ... BEFORE`, renames one behind a `confirm_rename` hint, asks before dropping the type, and reports removing or reordering labels, or using a label in the migration that adds it, as unsupported. `PG.unique(name, columns, { nullsNotDistinct? })` in a table's `uniqueConstraints`, or `unique` in `PG.column` (named `__unique` by default), declares an `ADD CONSTRAINT ... UNIQUE` constraint, diffed as a constraint rather than an index. `verify` reads enums from `pg_enum` and unique constraints from `pg_constraint`, and resolves types of the current schema during its scratch rebuild, so `PG.custom("public.x", ...)` enums no longer report false drift. `generate --baseline --from-drizzle` converts drizzle-kit enums, enum columns and unique constraints. diff --git a/.changeset/runnable-typescript-5.md b/.changeset/runnable-typescript-5.md new file mode 100644 index 0000000..6225f6c --- /dev/null +++ b/.changeset/runnable-typescript-5.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": patch +--- + +A concrete insert is assignable to `Runnable` (from `/database`) and accepted by `compile` under TypeScript 5.x again. Both took `CHInsert`, which TypeScript 5 compares method by method, and the generic `onConflictDoUpdate` failed that comparison; they now take an insert by its tag, state and phantom (`AnyInsert`). A workaround such as `type Runnable = Orm.Runnable | { readonly _tag: "CHInsert" }` can be removed. diff --git a/design/migrations.md b/design/migrations.md index 19656fd..f8f05b1 100644 --- a/design/migrations.md +++ b/design/migrations.md @@ -518,6 +518,15 @@ legacy: `check` accepts them, `generate` refuses to diff until `--baseline` (fro definitions, or `--from-drizzle` from drizzle-kit's last snapshot) starts the history. `Migrate.baseline` records already-applied migrations without running them. -**Not done.** Check and unique constraints, enums, views, sequences beyond identity defaults, -non-`public` schemas, `CREATE INDEX CONCURRENTLY` (needs a migration outside a transaction), -rename detection, and `pull` / `push`. +**Enums and unique constraints.** `PG.enumType` is a column type and a schema object; its +`enum` entity sorts before tables, `create_enum` runs first and `drop_enum` last. Labels are +added in place (`ADD VALUE ... BEFORE`), a label replaced in place is renamed (`RENAME VALUE`) +behind a `confirm_rename` hint, and removing or reordering labels is unsupported. A migration is +one transaction, so a label it adds cannot be used in it; the diff reports a default or +predicate that would. Unique constraints are `unique` entities, kept apart from unique indexes +all the way to `verify`, which reads them from `pg_constraint`. `verify` searches the current +schema after the scratch one, so types of the current schema deparse unqualified on both sides. + +**Not done.** Check constraints, views, sequences beyond identity defaults, tables in non-`public` +schemas, removing enum labels (a type rebuild), `CREATE INDEX CONCURRENTLY` (needs a migration +outside a transaction), rename detection for tables and columns, and `pull` / `push`. diff --git a/docs/migrations.md b/docs/migrations.md index 1066837..6d731fc 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -285,17 +285,59 @@ with `where` for a partial index and `using` for the access method. A foreign ke `name` gets drizzle-orm's, `
____fk`, shortened with drizzle-kit's hash to `
__fk` when it would pass 63 characters. Types are stored as Postgres names them (`int4` is `integer`), so snapshots compare with the catalog and -with drizzle-kit. Check and unique constraints, generated columns, enums, views, sequences and other schemas -are not modeled yet; write them in a `--custom` migration, and declare a view you query as an +with drizzle-kit. + +**Unique constraints.** `uniqueConstraints: [PG.unique(name, columns, { nullsNotDistinct? })]` +on a table, or `unique` in `PG.column` for one column, declares a `UNIQUE` constraint: `ALTER +TABLE ... ADD CONSTRAINT name UNIQUE (columns)`, which is what drizzle-kit writes for `unique()` +and `.unique()`. It is not a unique index: Postgres backs it with an index of the same name, but +drops it with `DROP CONSTRAINT`, and a foreign key may reference its columns. `unique: true` +names the constraint `
__unique`, drizzle-orm's name; `unique: "name"` or `unique: +{ name, nullsNotDistinct }` gives another (drizzle-kit v1 names a column's unnamed `.unique()` +`
__key`, so say so when adopting one). `nullsNotDistinct` needs Postgres 15. +`onConflictDoNothing` / `onConflictDoUpdate` take the constraint's columns as `target`, or `{ +constraint: name }`. + +**Enums.** `PG.enumType(name, labels)` is a column type and a schema object; see +[Enums](./postgres.md#enums). Its `CREATE TYPE` comes before every table, and is skipped when +the type already exists (Postgres has no `IF NOT EXISTS` for it), so an enum created by hand +behind `PG.custom` can be declared with `enumType` later. + +Check constraints, generated columns, views, sequences and tables in other schemas are not +modeled yet; write them in a `--custom` migration, and declare a view you query as an [external table](./tables-and-types.md#external-tables). **Generating.** Postgres changes a column's type, nullability, default or identity in place -(`ALTER COLUMN`), and a primary key, an index or a foreign key by dropping and re-creating it, -so `generate` reports nothing as unsupported. A type change is labeled `rewrite`: Postgres -rewrites the table under an exclusive lock. Drops still need confirmation, and renames still -read as a drop and an add. Generated files carry `"dialect": "postgres"`. With `emit: "sql"` in -the config, `generate` writes the rendered statements to `migration.sql` instead, each under a -comment naming its label, so a tool that applies drizzle-kit folders applies the migration too. +(`ALTER COLUMN`), and a primary key, an index, a unique constraint or a foreign key by dropping +and re-creating it. A type change is labeled `rewrite`: Postgres rewrites the table under an +exclusive lock. Drops need confirmation, dropping an enum type included (`"kind": "enum"`), and +renames of tables and columns still read as a drop and an add. Enum types are created first and +dropped last, unique constraints are added before the foreign keys that reference them, and +turning a `uniqueIndex` into a `unique` of the same name drops the index before adding the +constraint. Generated files carry `"dialect": "postgres"`. With `emit: "sql"` in the config, +`generate` writes the rendered statements to `migration.sql` instead, each under a comment +naming its label, so a tool that applies drizzle-kit folders applies the migration too. + +Enum labels change as far as Postgres can change them in place: + +- **A new label** is `ALTER TYPE ... ADD VALUE`, with `BEFORE` the label it precedes, so the + order matches the definition. +- **A renamed label**, the same labels in the same order but one replaced in place, is `ALTER + TYPE ... RENAME VALUE`. Rows holding the old label then hold the new one, so it asks for a + hint first: `{"type":"confirm_rename","kind":"enum_value","entity":"channel_type.direct","to":"dm"}`. +- **Removing or reordering labels** needs a new type and every column moved to it. `generate` + reports it as unsupported and writes nothing; write that rebuild with `--custom`. + +A migration is one transaction, and Postgres does not let a transaction use a label it added +(`unsafe use of new value`). So `generate` refuses a migration that adds a label and also +uses it, in a default or a partial index's predicate: add the label, generate, then use it and +generate again. Before Postgres 12, `ADD VALUE` cannot run in a transaction at all, so a +migration that adds a label needs Postgres 12 or later. + +Moving an enum from `PG.custom("public.channel_type", ...)` to `PG.enumType("channel_type", +...)` generates a `create_enum`, skipped since the type exists, and an `alter_column` per column +from `public.channel_type` to `channel_type`, the same type under its unqualified name, which +leaves the data as it is. **Applying.** Each migration runs in one transaction with its ledger row, under a transaction-scoped advisory lock, so concurrent deploys wait for each other rather than @@ -307,10 +349,15 @@ CONCURRENTLY`, `ALTER TYPE ... ADD VALUE` before Postgres 12) cannot be in a mig **Drift.** `verify` builds the expected schema in a scratch schema, inside a transaction it always rolls back, and reads both catalogs with the same queries, so Postgres deparses both -sides and `'open'` matches the stored `'open'::text`. It checks tables, columns (type, `NOT NULL`, -default, identity), primary keys, indexes (uniqueness, method, keys, predicate) and foreign keys -in the current schema. `Migrate.verify(migrations, { ignoreTables })` skips tables another tool -owns. +sides and `'open'` matches the stored `'open'::text`. The scratch schema is searched before the +current one, so a type of the current schema resolves in both, and a column of it reads +`channel_type`, its default `'public'::channel_type`, on both sides, however the definition +qualified it. It checks enum types (labels, in order, read from `pg_enum`), tables, columns +(type, `NOT NULL`, default, identity), primary keys, unique constraints (columns, `NULLS NOT +DISTINCT`), indexes (uniqueness, method, keys, predicate) and foreign keys in the current +schema. A unique constraint is never read as an index, nor an index as a constraint. An enum +the snapshot does not declare is reported only if no column uses it, so a `PG.custom` enum still +verifies. `Migrate.verify(migrations, { ignoreTables })` skips tables another tool owns. ### Adopting a drizzle-kit folder @@ -319,8 +366,10 @@ split on `--> statement-breakpoint`. Its `snapshot.json` files are recognized as and set aside, so the folder runs as it is. Adoption is two steps: 1. `effect-orm generate --baseline --from-drizzle` writes a migration that runs nothing, whose - snapshot is drizzle-kit's last one converted to entities. Anything the conversion cannot model - is listed and nothing is written. Without `--from-drizzle` the snapshot comes from your + snapshot is drizzle-kit's last one converted to entities: enums, tables, columns (enum columns + included), keys, indexes, unique constraints and foreign keys. Anything the conversion cannot + model (sequences, views, check constraints, policies, tables in other schemas) is listed and + nothing is written. Without `--from-drizzle` the snapshot comes from your `PG.table` definitions instead. Migrations before the baseline are legacy: they run, but nothing diffs against them, and a plain `generate` refuses to run until a baseline exists. 2. On a database drizzle-kit (or anything else) already migrated, `effect-orm baseline ` diff --git a/docs/postgres.md b/docs/postgres.md index ecfeef8..8964dcb 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -3,7 +3,8 @@ The same builder writes Postgres SQL. Queries, params, tenant scoping and row decoding work as they do for ClickHouse. `@maple-dev/effect-orm/postgres` is the one import for it: the builder, column types whose codecs read what Postgres drivers send, functions spelled the -Postgres way, `table` with its keys, indexes and foreign keys, and a `compile` that defaults to +Postgres way, `table` with its keys, indexes, unique constraints and foreign keys, `enumType`, +and a `compile` that defaults to the Postgres dialect. Nothing from `/clickhouse` is needed. ```ts title="postgres-quickstart.ts" @@ -59,7 +60,8 @@ await db.close() `@effect/sql-pg`'s `unsafe`. The builder never runs the query. `Id` is an identity column, so an insert may leave it out: `PG.InsertRowOf` makes it optional. See [Tables and column types](./tables-and-types.md) for column options and external tables, and -[Schema and migrations](./migrations.md#postgres) for indexes, foreign keys, and `generate`. +[Schema and migrations](./migrations.md#postgres) for indexes, unique constraints, foreign keys, +and `generate`; [Enums](#enums) for enum types. ## What the dialect changes @@ -98,6 +100,7 @@ literal that still contained it would fail the compile with `InvalidLiteral`. | `timestamptz` | `timestamptz` | `DateTime.Utc` | `Date`, or text such as `2026-01-01 00:00:00+00` | | `timestamptzMillis` | `timestamptz` | epoch milliseconds (`number`) | the same | | `jsonb(schema?)` | `jsonb` | the schema's type (`unknown` by default) | a parsed value | +| `enumType(name, labels, { schema? })` | the enum type | one of its labels | string | | `array(type)` | `type[]` | `ReadonlyArray` | array | | `nullable(type)` | the same type | `T \| null` | the same, or `null` | | `custom(sql, schema, literalSchema?)` | anything | the schema's type | whatever the schema reads | @@ -115,6 +118,70 @@ Comparisons with a literal-union column take only its members: with `status` typ `"open" | "closed"`, `$.status.eq("opne")` is a type error. A param of the primitive (`param.string`) still compares, for a value known only at run time. +## Enums + +`PG.enumType(name, labels)` declares a Postgres enum type. The value is a column type, so rows +decode to the union of its labels and inserts and comparisons take only those, and it is a +schema object, so `generate` writes `CREATE TYPE ... AS ENUM` before any table that uses it and +alters the type when its labels change. Use it bare, in `PG.column` with a `default`, or inside +`PG.nullable` and `PG.array`: + +```ts title="postgres-enums.ts" +import { Effect } from "effect" +import * as PG from "@maple-dev/effect-orm/postgres" +import * as S from "@maple-dev/effect-orm/schema" + +export const ChannelType = PG.enumType("channel_type", ["public", "private", "direct"]) + +export const Channels = PG.table("channels", { + columns: { + id: PG.text, + org_id: PG.text, + slug: PG.column(PG.text, { unique: true }), + type: PG.column(ChannelType, { default: "public" }), + allowed: PG.nullable(PG.array(ChannelType)), + }, + primaryKey: ["id"], + uniqueConstraints: [PG.unique("channels_org_slug_unique", ["org_id", "slug"])], +}) + +export type Channel = PG.SelectRowOf +// { ...; readonly type: "public" | "private" | "direct"; readonly allowed: ReadonlyArray<...> | null } + +export const shared = PG.compileUnsafe( + PG.from(Channels) + .select("id", "type") + .where(($) => [$.type.in_("public", "private")]), + {}, +) + +export const upsert = PG.compileUnsafe( + PG.insertInto(Channels) + .values({ id: "c1", org_id: "o", slug: "general", type: "private" }) + .onConflictDoUpdate({ target: ["slug"], set: { type: "private" } }), + {}, +) + +export const ddl = Effect.map(S.pgEntitiesOf([ChannelType, Channels]), (entities) => S.renderPgSchema(entities)) +// CREATE TYPE "channel_type" AS ENUM ('public', 'private', 'direct') (skipped if it exists) +// CREATE TABLE IF NOT EXISTS "channels" (... "type" channel_type NOT NULL DEFAULT 'public', ...) +// ALTER TABLE "channels" ADD CONSTRAINT "channels_org_slug_unique" UNIQUE ("org_id", "slug") +// ALTER TABLE "channels" ADD CONSTRAINT "channels_slug_unique" UNIQUE ("slug") +``` + +`ChannelType.values` lists the labels for application code. Exporting the enum from a schema +module is optional: the tables that use it bring it along. Pass `{ schema: "auth" }` for a type +in another schema; leave it out for the schema migrations run in (usually `public`), which is +how columns of the type are spelled there. + +Drivers parse arrays of built-in types but not of an enum, whose array type they do not know: +node-postgres and PGlite send `{public,private}` as text. An `array(enumType)` column reads +either form, and writes its value as array text, which Postgres casts to the column's type. + +`PG.brand(PG.text, Schema.Literals([...]))` and `PG.custom("public.channel_type", ...)` still +work for an enum `generate` does not manage; see [migrations](./migrations.md#postgres) for +moving one to `enumType`. + ## Functions | Function | SQL | Notes | diff --git a/docs/reference.md b/docs/reference.md index 3fef497..66e108a 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -410,15 +410,20 @@ following. See [Postgres](./postgres.md). ### Tables and DDL ```ts +const Role = PG.enumType("user_role", ["owner", "member"]) + const Users = PG.table("users", { columns: { id: PG.column(PG.int8, { identity: "always" }), orgId: PG.text, email: PG.text, + handle: PG.column(PG.text, { unique: true }), + role: PG.column(Role, { default: "member" }), createdAt: PG.column(PG.timestamptz, { defaultExpr: "now()" }), }, primaryKey: ["id"], indexes: [PG.uniqueIndex("users_org_email_idx", ["orgId", "email"])], + uniqueConstraints: [PG.unique("users_org_handle_unique", ["orgId", "handle"])], tenantColumn: "orgId", }) ``` @@ -429,22 +434,33 @@ const Users = PG.table("users", { | `column` | `(type, ColumnOptions) => ColumnSpec` | | `index` | `(name, on, IndexOptions?) => IndexSpec`: columns, or a callback building expressions | | `uniqueIndex` | The same, `UNIQUE`; with `where`, a partial unique index | +| `unique` | `(name, columns, UniqueOptions?) => UniqueSpec`: a `UNIQUE` constraint, for `uniqueConstraints` | +| `enumType` | `(name, labels, EnumOptions?) => PgEnum`: an enum type, both a column type and a schema object | | `foreignKey` | `({ columns, references, foreignColumns, onDelete?, onUpdate?, name? }) => ForeignKeySpec`; `references` is a `Table` (its columns are checked) or a name | | `defaultForeignKeyName` | `(table, columns, foreignTable, foreignColumns) => string`: the name drizzle-kit gives, shortened past 63 characters as drizzle-kit does | **`TableDefinition`** — `columns` (required), `primaryKey` (column names, or -`{ columns, name? }`), `indexes`, `foreignKeys`, `tenantColumn`. +`{ columns, name? }`), `indexes`, `uniqueConstraints`, `foreignKeys`, `tenantColumn`. **`ExternalTableDefinition`** — `{ external: true, columns, tenantColumn? }`, as on `/clickhouse`: a view, a catalog table, one another tool migrates. No DDL. **`ColumnOptions`** — one of `default`, `defaultExpr`, `identity` (`"always"` or -`"by default"`); any of them lets an insert leave the column out (`DefaultedColumnsOf`). +`"by default"`); any of them lets an insert leave the column out (`DefaultedColumnsOf`). Also +`name` (the column's name in SQL) and `unique` (`true`, a constraint name, or +`UniqueColumnOptions`: `{ name?, nullsNotDistinct? }`), which names a one-column constraint +`
__unique` by default. + +**`UniqueOptions`** — `nullsNotDistinct` (Postgres 15+). + +**`PgEnum`** — a `PgType` whose rows decode to its labels, with `name`, `values` (the labels), +and the `ddl` and `problems` `generate` reads. **`EnumOptions`** — `schema`, for a type outside +the schema migrations run in. **`IndexOptions`** — `where` (a `DdlPredicate`: SQL text or a callback returning a condition), `using` (the access method; default `btree`). -**Types** — `ColumnInput`, `ColumnSpec`, `ColumnsOf`, `IndexSpec`, `ForeignKeySpec`, +**Types** — `ColumnInput`, `ColumnSpec`, `ColumnsOf`, `IndexSpec`, `UniqueSpec`, `ForeignKeySpec`, `ReferentialAction` (drizzle's lowercase spelling or the catalog's), `TableDdl`, `PgSchemaTable`, `DdlExpr`, `DdlKey`. A definition that cannot render records a problem; `S.pgEntitiesOf` fails with it. @@ -452,12 +468,13 @@ const Users = PG.table("users", { `text`, `uuid`, `bool`, `int2`, `int4`, `int8`, `float4`, `float8`, `numeric`, `timestamptz`, `timestamptzMillis` (a timestamptz read as epoch milliseconds), `jsonb(schema?)`, `array(type)`, `nullable(type)`, `custom(sql, schema, literalSchema?)`, -`brand(type, schema)`. See [Postgres column types](./postgres.md#column-types). +`brand(type, schema)`, and `enumType(name, labels)` above. See [Postgres column types](./postgres.md#column-types). Types: `PgType` (a Postgres column type; a `CHType`), `PgArray`, `PgNullable`. Codecs: `PgNumber` (a number, numeric string or `bigint`), `PgTimestampLiteral` (an instant written as ISO-8601), `timestampLiteral(format)` (the same with another format), and `pgTimestampToIso`, -which normalizes Postgres timestamp text to ISO-8601. +which normalizes Postgres timestamp text to ISO-8601. `parsePgArrayLiteral` reads a +one-dimensional array literal (`{a,"b c",NULL}`), the form enum arrays arrive in. ### Functions @@ -485,7 +502,7 @@ placeholders, double-quoted identifiers. | `SqlQuery`, `compileQuery` | `/sql` | | `ClickHouseStatement`, `parseStatement`, `renderStatement`, `withSettings`, `withFormat` | `/sql` | | `ClickHouseStatementFromString`, `splitTerminalClauses`, `maskLiteralsAndComments` | `/sql` | -| Schema tooling: `isSchemaObject`, `makeSnapshot`, `renderSchema`, `diffSchemas`, `diffPgSchemas`, `fromDrizzleSnapshot`, snapshot and entity schemas | `/schema` | +| Schema tooling: `isSchemaObject`, `makeSnapshot`, `renderSchema`, `diffSchemas`, `diffPgSchemas`, `fromDrizzleSnapshot`, snapshot and entity schemas (`PgEnumEntity`, `PgUniqueEntity`, …), `Hint` | `/schema` | | `run`, `status`, `verify`, `baseline`, `MigrationDriver` and the migrate errors | `/migrate` | | `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | | `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | diff --git a/docs/tables-and-types.md b/docs/tables-and-types.md index 9249e11..23b0795 100644 --- a/docs/tables-and-types.md +++ b/docs/tables-and-types.md @@ -60,9 +60,10 @@ A column's key is its name in SQL unless `PG.column` gives it another: `orgId: P { name: "org_id" })` is stored as `org_id` and read, written, filtered and indexed as `orgId` everywhere else, as drizzle's `text("org_id")` is. Rows decode under the key. -A Postgres column is `NOT NULL` unless its type is `PG.nullable(...)`. See -[Postgres](./postgres.md) for its types and [migrations](./migrations.md#postgres) for its -indexes and foreign keys. +A Postgres column is `NOT NULL` unless its type is `PG.nullable(...)`. An enum column's type is +a `PG.enumType("status", ["open", "closed"])`, which `generate` creates as a type of its own. See +[Postgres](./postgres.md) for its types and enums, and [migrations](./migrations.md#postgres) for +its indexes, unique constraints and foreign keys. ## Keys, defaults, and tenancy @@ -78,7 +79,9 @@ decide the insert row type, so it is derived from the DDL rather than declared t | `codec`, `comment` | `CODEC(…)`, `COMMENT` | unchanged | Postgres columns take `default`, `defaultExpr`, and `identity` (`"always"` or `"by default"`); -each makes the column optional on insert. Postgres generated columns are not modeled yet. +each makes the column optional on insert. They also take `unique` (`true`, a constraint name, or +`{ name, nullsNotDistinct }`), a one-column `UNIQUE` constraint, which leaves the insert type +alone. Postgres generated columns are not modeled yet. ```ts const Requests = CH.table("requests", { @@ -134,7 +137,8 @@ const RoutesHourlyMv = CH.materializedView("routes_hourly_mv", { ``` Postgres indexes are `PG.index` and `PG.uniqueIndex`, over column names or expressions, with -`where` for a partial index; foreign keys are `PG.foreignKey`. Both are covered in +`where` for a partial index; unique constraints are `PG.unique` in `uniqueConstraints`, and +foreign keys are `PG.foreignKey`. All are covered in [Schema and migrations](./migrations.md#postgres). ## External tables @@ -339,7 +343,8 @@ A brand is strict everywhere it is written or compared: param value that fails them is a `QueryBuilderError` from `compile`. A literal union (`PG.brand(PG.text, Schema.Literals(["open", "closed"]))`) is not a brand: it -compares against any string, and the database checks the value. +compares against its members, or a param of the primitive (`param.string`), which the database +checks. A Postgres enum column, `PG.enumType(...)`, behaves the same way. `CH.custom("String", OrgId)` brands the same way, but replaces the wire codec with `OrgId` itself; prefer `brand` over a built-in type whose codec does work (numbers, timestamps). diff --git a/scripts/check-doc-examples.mjs b/scripts/check-doc-examples.mjs index f9ba68e..d8b190a 100644 --- a/scripts/check-doc-examples.mjs +++ b/scripts/check-doc-examples.mjs @@ -109,6 +109,14 @@ assert.deepEqual(postgres.rows, [ { route: "/checkout", count: 2, slow: 1, p50: 510 }, { route: "/search", count: 1, slow: 0, p50: 40 }, ]) +const enums = await import("./postgres-enums") +const enumDdl = await Effect.runPromise(enums.ddl) +assert.match(enumDdl[0], /CREATE TYPE "channel_type" AS ENUM \\('public', 'private', 'direct'\\)/) +assert.equal(enumDdl.at(-1), 'ALTER TABLE "channels" ADD CONSTRAINT "channels_slug_unique" UNIQUE ("slug")') +assert.deepEqual(enums.shared.parameters, ["public", "private"]) +assert.deepEqual(await Effect.runPromise(enums.shared.decodeRows([{ id: "c", type: "public" }])), [{ id: "c", type: "public" }]) +await assert.rejects(() => Effect.runPromise(enums.shared.decodeRows([{ id: "c", type: "secret" }]))) +assert.match(sql(enums.upsert), /ON CONFLICT \\("slug"\\) DO UPDATE SET/) const database = await import("./database-transaction") assert.deepEqual(database.balances, { from: 70, to: 30, refused: "InsufficientFunds" }) console.log("Markdown example behavior checks passed") diff --git a/scripts/check-package.ts b/scripts/check-package.ts index 45a3ae4..651b2be 100644 --- a/scripts/check-package.ts +++ b/scripts/check-package.ts @@ -53,6 +53,8 @@ const program = Effect.gen(function* () { }, devDependencies: { typescript: yield* version("typescript"), + // Consumers still on TypeScript 5 see the built types too; see the consumer's Runnable check. + "typescript-5": "npm:typescript@~5.9.3", "@types/node": yield* version("@types/node"), }, }), @@ -78,6 +80,16 @@ const program = Effect.gen(function* () { }), ) yield* runCommand(path.join(temporary, "node_modules/.bin/tsc"), ["-p", "tsconfig.json"], temporary) + // The TypeScript 5 checks, alone in their own program (see the file's header), on both compilers. + yield* fs.copyFile(path.join(root, "tests/package-consumer-ts5.mts"), path.join(temporary, "consumer-ts5.mts")) + const ts5Config = path.join(temporary, "tsconfig.ts5.json") + yield* fs.writeFileString( + ts5Config, + yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))({ extends: "./tsconfig.json", include: ["consumer-ts5.mts"] }), + ) + yield* runCommand(path.join(temporary, "node_modules/.bin/tsc"), ["-p", ts5Config], temporary) + yield* runCommand("node", [path.join(temporary, "node_modules/typescript-5/bin/tsc"), "-p", ts5Config], temporary) + yield* runCommand("node", ["consumer-ts5.mts"], temporary) // The script runs on Bun; the published library must also work in Node. yield* runCommand("node", ["consumer.mts"], temporary) yield* runCommand( diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 323a570..5709864 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -11,7 +11,7 @@ import { compiled, compiledUnsafe, fail } from "./failure" import { columnSqlName, custom, dateTime, dateTime64, type CHType, type ColumnDefs } from "./types" import type { CHQuery, CHQueryState, NeedsSelect } from "./query" import type { CHUnionQuery } from "./union" -import { isInsert, type CHInsert } from "./insert" +import { isInsert, type AnyInsert, type CHInsert } from "./insert" import { isDelete, isUpdate, type CHDelete, type CHUpdate } from "./update" import type { Table } from "./table" import { createColumnAccessor, createQualifiedColumnAccessor, createJoinedColumnAccessor, sourceAlias } from "./query" @@ -606,7 +606,7 @@ export function compileCH( /** A write statement: what `compile` takes besides a query. */ export type CHWrite = - | CHInsert + | AnyInsert | CHUpdate | CHDelete diff --git a/src/ch/insert.ts b/src/ch/insert.ts index 0de70c2..bf6adc0 100644 --- a/src/ch/insert.ts +++ b/src/ch/insert.ts @@ -273,6 +273,21 @@ export interface CHInsert< crossTenant(): CHInsert } +/** + * Any insert, by what compiling and running one reads: its tag, its state, and + * the row and params its phantom carries. `compile` and `Database.run` take + * this rather than `CHInsert`, because TypeScript 5.x compares a + * concrete insert with `CHInsert` method by method, and the generic + * `onConflictDoUpdate` does not survive that (`ConflictTarget` is any + * `string`; `any` is not `never`). TypeScript 7 accepts either. + */ +export interface AnyInsert { + readonly _tag: "CHInsert" + /** @internal — runtime insert state */ + readonly _state: CHInsertState + readonly _phantom?: { readonly output: Output; readonly params: (entries: Params) => void } +} + const makeInsert = ( state: CHInsertState, ): CHInsert => ({ diff --git a/src/database/database.test-d.ts b/src/database/database.test-d.ts index 8a5fbc1..c77a281 100644 --- a/src/database/database.test-d.ts +++ b/src/database/database.test-d.ts @@ -61,3 +61,19 @@ void unwrapped // Settings are typed: an unknown isolation level does not compile. // @ts-expect-error not an isolation level void Db.transaction(Effect.void, { isolationLevel: "read uncommitted" }) + +// Any concrete insert is a `Runnable`, and `compile` takes it, upserts included: +// both take an insert by its tag, state and phantom, not by comparing its generic +// methods, which TypeScript 5 rejects for `CHInsert`. (tests/package-consumer-ts5.mts +// checks this against the built types with TypeScript 5 as well.) +const Counters = PG.table("counters", { columns: { key: PG.text, count: PG.int4 }, primaryKey: ["key"] }) +const upsert = CH.insertInto(Counters) + .values({ key: "k", count: CH.param.int("count") }) + .onConflictDoUpdate({ target: ["key"], set: { count: 1 } }) + .returning("count") +expectTypeOf(CH.insertInto(Counters).values({ key: "k", count: 1 })).toExtend() +expectTypeOf(upsert).toExtend() +expectTypeOf(Db.run(upsert, { count: 1 })).toEqualTypeOf, RunError, Db.Database>>() +// @ts-expect-error the param is still required +Db.run(upsert) +expectTypeOf(PG.compileUnsafe(upsert, { count: 1 })).toEqualTypeOf>() diff --git a/src/database/database.ts b/src/database/database.ts index a20c99b..8e2dd76 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -14,7 +14,7 @@ import { isSqlError } from "effect/sql/SqlError" import { compileCH, compileUnion, CompiledQueryDecodeError, type CompiledQuery } from "../ch/compile" import { noTransactions, type Dialect, type IsolationLevel, type TransactionSettings } from "../ch/dialect" import type { QueryBuilderError } from "../ch/errors" -import type { CHInsert } from "../ch/insert" +import type { AnyInsert } from "../ch/insert" import type { CHDelete, CHUpdate } from "../ch/update" import type { CHQuery, NeedsSelect } from "../ch/query" import type { CHUnionQuery, QueryParams } from "../ch/union" @@ -45,7 +45,7 @@ export type StatementInput = SqlTemplate | Statement export type Runnable = | CHQuery | CHUnionQuery - | CHInsert + | AnyInsert | CHUpdate | CHDelete | CompiledQuery diff --git a/src/kit/cli.ts b/src/kit/cli.ts index 353783e..cb856bd 100644 --- a/src/kit/cli.ts +++ b/src/kit/cli.ts @@ -25,7 +25,7 @@ const help = `effect-orm: schema migrations for ClickHouse and Postgres --config Default: effect-orm.config.ts --json Machine-readable output -Exit codes: 0 ok, 1 error, 2 missing hints (confirm data loss), 3 drift found.` +Exit codes: 0 ok, 1 error, 2 missing hints (confirm data loss or an enum label rename), 3 drift found.` const options = { config: { type: "string" }, @@ -59,7 +59,11 @@ const loadConfig = (path: string) => const promptConfirm = (hint: Hint): Effect.Effect => Effect.promise(async () => { const rl = createInterface({ input: process.stdin, output: process.stdout }) - const answer = await rl.question(`Drop ${hint.kind} ${hint.entity}? Its data will be lost. [y/N] `) + const answer = await rl.question( + hint.type === "confirm_rename" + ? `Rename enum label ${hint.entity} to ${JSON.stringify(hint.to)}? Rows holding it will hold the new label. [y/N] ` + : `Drop ${hint.kind} ${hint.entity}? ${hint.kind === "enum" ? "Postgres refuses while a column uses it." : "Its data will be lost."} [y/N] `, + ) rl.close() return /^y(es)?$/i.test(answer.trim()) }) diff --git a/src/kit/generate.ts b/src/kit/generate.ts index d9df747..3fed4e7 100644 --- a/src/kit/generate.ts +++ b/src/kit/generate.ts @@ -235,7 +235,7 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption if (result.unsupported.length > 0) { return yield* new KitError({ code: "unsupported", - message: "these changes need a table rebuild or a data rewrite, which generate does not write yet", + message: "these changes need a table or type rebuild, a data rewrite, or a migration of their own, which generate does not write", details: result.unsupported.map((u) => `${u.entity}: ${u.message}`), }) } @@ -248,7 +248,7 @@ export const generate = (config: KitConfig, cwd: string, options: GenerateOption if (result.missingHints.length > 0) { return yield* new KitError({ code: "missing_hints", - message: "confirm data loss with --hints, or run in a terminal to be asked", + message: "confirm data loss (or an enum label rename) with --hints, or run in a terminal to be asked", details: [JSON.stringify(result.missingHints)], }) } diff --git a/src/kit/graph.ts b/src/kit/graph.ts index 6c6dc9f..cd259ea 100644 --- a/src/kit/graph.ts +++ b/src/kit/graph.ts @@ -64,7 +64,7 @@ const changesBetween = ( /** The table an entity belongs to, so two branches editing one table conflict. */ const ownerOf = (key: string): string => { const [kind, rest = ""] = key.split(":") - return kind === "column" || kind === "index" || kind === "foreign_key" ? `table:${rest.split(".")[0]}` : key + return kind === "column" || kind === "index" || kind === "unique" || kind === "foreign_key" ? `table:${rest.split(".")[0]}` : key } export const analyze = (migrations: ReadonlyArray): Effect.Effect => diff --git a/src/kit/kit.test.ts b/src/kit/kit.test.ts index 6b3b190..b41cdff 100644 --- a/src/kit/kit.test.ts +++ b/src/kit/kit.test.ts @@ -1,7 +1,7 @@ import { mkdirSync, mkdtempSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join, resolve } from "node:path" -import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { runCli } from "../kit" const src = resolve(import.meta.dirname, "..") @@ -138,6 +138,51 @@ export const Dashboards = PG.table("dashboards", { expect(await cli("generate", "--config", "ch.config.ts")).toBe(1) }) + it("generates enum types and unique constraints, asks to rename a label, and refuses to remove one", async () => { + const module = (labels: string) => ` +import * as PG from "${src}/postgres" + +export const Status = PG.enumType("ticket_status", [${labels}]) +export const Tickets = PG.table("tickets", { + columns: { id: PG.text, slug: PG.column(PG.text, { unique: true }), status: PG.column(Status, { default: "open" }) }, + primaryKey: ["id"], +}) +` + // A fresh module and config path per step, so the import cache does not hand back the last one. + let config = "" + const step = (file: string, labels: string) => { + writeFileSync(join(dir, `${file}.ts`), module(labels)) + config = `${file}.config.ts` + pgConfig(`${file}.ts`, config) + } + const gen = (...args: Array) => cli("generate", "--config", config, ...args) + // Refusals go to stderr. + const errors = vi.spyOn(console, "error").mockImplementation(() => undefined) + const stderr = () => errors.mock.calls.map((call) => call.join(" ")).join("\n") + step("e1", `"open", "closed"`) + expect(await gen("--name", "init")).toBe(0) + const [first] = folders() + const file = JSON.parse(readFileSync(join(dir, "migrations", first!, "migration.json"), "utf8")) + expect(file.ops.map((op: { op: string }) => op.op)).toEqual(["create_enum", "create_table", "add_unique"]) + + step("e2", `"open", "pending", "closed"`) + expect(await gen("--name", "pending")).toBe(0) + expect(lines.join("\n")).toContain(`[metadata] ALTER TYPE "ticket_status" ADD VALUE IF NOT EXISTS 'pending' BEFORE 'closed'`) + + step("e3", `"open", "pending", "done"`) + expect(await gen("--name", "done")).toBe(2) + expect(stderr()).toContain('"type":"confirm_rename","kind":"enum_value","entity":"ticket_status.closed","to":"done"') + const hint = JSON.stringify([{ type: "confirm_rename", kind: "enum_value", entity: "ticket_status.closed", to: "done" }]) + expect(await gen("--name", "done", "--hints", hint)).toBe(0) + expect(lines.join("\n")).toContain(`ALTER TYPE "ticket_status" RENAME VALUE 'closed' TO 'done'`) + + step("e4", `"open", "done"`) + expect(await gen()).toBe(1) + expect(stderr()).toContain('ticket_status: removing "pending" needs the type rebuilt') + expect(await cli("check", "--config", config)).toBe(0) + errors.mockRestore() + }) + it("writes migration.sql with emit: sql, which the next generate reads as its parent", async () => { writeFileSync(join(dir, "pg1.ts"), pgModule()) writeFileSync(join(dir, "sql.config.ts"), `export default { dialect: "postgres", emit: "sql", schema: "./pg1.ts", out: "./migrations" }\n`) diff --git a/src/migrate/pg-enums-uniques.test.ts b/src/migrate/pg-enums-uniques.test.ts new file mode 100644 index 0000000..7f8c22d --- /dev/null +++ b/src/migrate/pg-enums-uniques.test.ts @@ -0,0 +1,389 @@ +// Enums and unique constraints against a live Postgres (PGlite): what +// `generate` writes applies, `verify` reads back clean, and a database +// drizzle-kit pushed is adopted without false drift. + +import { PgliteClient } from "@effect/sql-pglite" +import { assert, describe, expect, it } from "@effect/vitest" +import { Effect, Exit, Layer, Schema } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import * as Migrate from "../migrate" +import * as PG from "../postgres" +import * as S from "../schema" + +const Live = Migrate.layerSqlClient().pipe(Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" }))) + +const ChannelType = PG.enumType("channel_type", ["public", "private", "direct"]) +const OutboxStatus = PG.enumType("message_outbox_event_status", ["pending", "processing", "done"]) + +const schemaWith = (channelType: PG.PgEnum = ChannelType) => { + const Channels = PG.table("channels", { + columns: { + id: PG.text, + org_id: PG.text, + slug: PG.column(PG.text, { unique: true }), + type: PG.column(channelType, { default: "public" }), + allowed: PG.nullable(PG.array(channelType)), + }, + primaryKey: ["id"], + uniqueConstraints: [PG.unique("channels_org_slug_unique", ["org_id", "slug"], { nullsNotDistinct: true })], + }) + const Outbox = PG.table("message_outbox", { + columns: { + id: PG.text, + channel_org_id: PG.text, + channel_slug: PG.text, + status: PG.column(OutboxStatus, { default: "pending" }), + created_at: PG.column(PG.timestamptz, { defaultExpr: "now()" }), + }, + primaryKey: ["id"], + indexes: [PG.index("message_outbox_pending_idx", ["created_at"], { where: `"message_outbox"."status" = 'pending'` })], + foreignKeys: [ + // References the unique constraint, not the primary key. + PG.foreignKey({ columns: ["channel_org_id", "channel_slug"], references: Channels, foreignColumns: ["org_id", "slug"], name: "message_outbox_channel_fk" }), + ], + }) + return { Channels, Outbox } +} + +/** Each step generated from the one before, as `generate` writes them. */ +const history = (steps: ReadonlyArray<{ readonly name: string; readonly objects: ReadonlyArray; readonly hints?: ReadonlyArray }>) => + Effect.gen(function* () { + const record: Record = {} + let prev: ReadonlyArray = [] + let prevIds: ReadonlyArray = [S.ORIGIN_ID] + for (const step of steps) { + const next = yield* S.pgEntitiesOf(step.objects) + const { ops, missingHints, unsupported } = S.diffPgSchemas(prev, next, step.hints) + assert.deepStrictEqual([missingHints, unsupported], [[], []]) + const snapshot = yield* S.makeSnapshot(next, prevIds, "postgres") + record[step.name] = { + kind: "ops", + migration: JSON.stringify({ version: "1", dialect: "postgres", ops }), + snapshot: S.serializeSnapshot(snapshot), + } + prev = next + prevIds = [snapshot.id] + } + return yield* Migrate.fromRecord(record) + }) + +const labels = (type: string) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient + const rows = yield* sql.unsafe<{ label: string }>( + `SELECT e.enumlabel AS label FROM pg_enum e JOIN pg_type t ON t.oid = e.enumtypid WHERE t.typname = '${type}' ORDER BY e.enumsortorder`, + ) + return rows.map((r) => r.label) + }) + +describe("Postgres enums and unique constraints", () => { + it.effect("migrate creates them, verify reads them back clean, and queries use them", () => + Effect.gen(function* () { + const { Channels, Outbox } = schemaWith() + const migrations = yield* history([{ name: "20261013000000_init", objects: [ChannelType, OutboxStatus, Channels, Outbox] }]) + yield* Migrate.run({ migrations }) + expect(yield* Migrate.verify(migrations)).toEqual({ against: "20261013000000_init", drift: [] }) + + const sql = yield* SqlClient.SqlClient + const exec = (compiled: { readonly sql: string; readonly parameters: ReadonlyArray }) => + sql.unsafe>(compiled.sql, [...compiled.parameters]) + yield* exec(PG.compileUnsafe(PG.insertInto(Channels).values({ id: "c1", org_id: "o", slug: "general", allowed: ["public", "direct"] }), {})) + // The unique constraint is the conflict target, by its columns or by name. + yield* exec( + PG.compileUnsafe( + PG.insertInto(Channels) + .values({ id: "c2", org_id: "o", slug: "general", type: "private" }) + .onConflictDoUpdate({ target: ["slug"], set: { type: "private" } }), + {}, + ), + ) + yield* exec( + PG.compileUnsafe( + PG.insertInto(Channels).values({ id: "c3", org_id: "o", slug: "general" }).onConflictDoNothing({ target: { constraint: "channels_slug_unique" } }), + {}, + ), + ) + const query = PG.compileUnsafe(PG.from(Channels).select("id", "type", "allowed").where(($) => [$.type.in_("private", "direct")]), {}) + expect(yield* query.decodeRows(yield* exec(query))).toEqual([{ id: "c1", type: "private", allowed: ["public", "direct"] }]) + const bad = yield* Effect.exit(sql.unsafe(`INSERT INTO channels (id, org_id, slug, type) VALUES ('c4', 'o', 's', 'nope')`)) + expect(Exit.isFailure(bad)).toBe(true) + }).pipe(Effect.provide(Live)), + ) + + it.effect("adds and renames labels, swaps a unique index for a constraint, and drops a type, verifying clean after each", () => + Effect.gen(function* () { + const v1 = schemaWith() + const added = PG.enumType("channel_type", ["public", "announcement", "private", "direct", "thread"]) + const v2 = schemaWith(added) + const renamed = PG.enumType("channel_type", ["public", "announcement", "private", "dm", "thread"]) + const v3 = schemaWith(renamed) + const Tags = (unique: boolean) => + PG.table("tags", { + columns: { id: PG.text, name: PG.text }, + ...(unique ? { uniqueConstraints: [PG.unique("tags_name_unique", ["name"])] } : { indexes: [PG.uniqueIndex("tags_name_unique", ["name"])] }), + }) + const Mood = PG.enumType("mood", ["ok"]) + const Notes = (withMood: boolean) => PG.table("notes", { columns: { id: PG.text, mood: withMood ? Mood : PG.text } }) + + const steps = [ + { name: "20261013000000_init", objects: [v1.Channels, v1.Outbox, Tags(false), Notes(true)] }, + { name: "20261013000001_labels", objects: [v2.Channels, v2.Outbox, Tags(false), Notes(true)] }, + { + name: "20261013000002_rename", + objects: [v3.Channels, v3.Outbox, Tags(true), Notes(true)], + hints: [{ type: "confirm_rename", kind: "enum_value", entity: "channel_type.direct", to: "dm" } as const], + }, + { + name: "20261013000003_drop_mood", + objects: [v3.Channels, v3.Outbox, Tags(true), Notes(false)], + hints: [{ type: "confirm_data_loss", kind: "enum", entity: "mood" } as const], + }, + ] + const sql = yield* SqlClient.SqlClient + for (let i = 1; i <= steps.length; i++) { + const migrations = yield* history(steps.slice(0, i)) + yield* Migrate.run({ migrations }) + expect((yield* Migrate.verify(migrations)).drift).toEqual([]) + if (i === 1) yield* sql.unsafe(`INSERT INTO channels (id, org_id, slug, type) VALUES ('c1', 'o', 's', 'direct')`) + } + expect(yield* labels("channel_type")).toEqual(["public", "announcement", "private", "dm", "thread"]) + // The renamed label kept its rows. + expect(yield* sql.unsafe(`SELECT type::text AS type FROM channels`)).toEqual([{ type: "dm" }]) + expect(yield* labels("mood")).toEqual([]) + const tagsConstraint = yield* sql.unsafe(`SELECT conname FROM pg_constraint WHERE conname = 'tags_name_unique' AND contype = 'u'`) + expect(tagsConstraint).toHaveLength(1) + }).pipe(Effect.provide(Live)), + ) + + it.effect("a label the migration adds and uses fails the migration as a whole", () => + Effect.gen(function* () { + // What generate refuses to write, written by hand: Postgres rejects it and nothing is applied. + const migrations = yield* Migrate.fromRecord({ + "20261013000000_a": { kind: "sql", migration: `CREATE TYPE "k" AS ENUM ('a');\n--> statement-breakpoint\nCREATE TABLE "t" ("k" k NOT NULL)` }, + "20261013000001_b": { + kind: "sql", + migration: `ALTER TYPE "k" ADD VALUE 'b'\n--> statement-breakpoint\nALTER TABLE "t" ALTER COLUMN "k" SET DEFAULT 'b'`, + }, + }) + const exit = yield* Effect.exit(Migrate.run({ migrations, dialect: "postgres" })) + assert(Exit.isFailure(exit)) + expect(String(exit.cause)).toMatch(/unsafe use of new value/) + expect(yield* labels("k")).toEqual(["a"]) + }).pipe(Effect.provide(Live)), + ) + + it.effect("verify reports drift in labels, unique constraints and stray types", () => + Effect.gen(function* () { + const { Channels, Outbox } = schemaWith() + const migrations = yield* history([{ name: "20261013000000_init", objects: [Channels, Outbox] }]) + yield* Migrate.run({ migrations }) + const sql = yield* SqlClient.SqlClient + yield* sql.unsafe(`ALTER TYPE channel_type ADD VALUE 'thread'`) + yield* sql.unsafe(`ALTER TABLE channels DROP CONSTRAINT channels_slug_unique`) + // The same uniqueness as an index is still drift: it is not a constraint. + yield* sql.unsafe(`CREATE UNIQUE INDEX channels_slug_unique ON channels (slug)`) + yield* sql.unsafe(`ALTER TABLE channels DROP CONSTRAINT channels_org_slug_unique CASCADE`) + yield* sql.unsafe(`ALTER TABLE channels ADD CONSTRAINT channels_org_slug_unique UNIQUE (org_id, slug)`) + yield* sql.unsafe(`CREATE TYPE stray AS ENUM ('x')`) + const { drift } = yield* Migrate.verify(migrations) + expect(drift.map((d) => `${d.problem} ${d.entity}`).sort()).toEqual([ + "enum enum channel_type", + "missing channels.channels_slug_unique", + "missing message_outbox.message_outbox_channel_fk", + "unexpected channels.channels_slug_unique", + "unexpected enum stray", + "unique channels.channels_org_slug_unique", + ]) + expect(drift.find((d) => d.problem === "enum")).toMatchObject({ + expected: JSON.stringify(["public", "private", "direct"]), + actual: JSON.stringify(["public", "private", "direct", "thread"]), + }) + }).pipe(Effect.provide(Live)), + ) + + it.effect("an enum in another schema is created there and verified, and that schema's other enums are not drift", () => + Effect.gen(function* () { + const Role = PG.enumType("role", ["admin", "member"], { schema: "auth" }) + const Users = PG.table("users", { columns: { id: PG.text, role: PG.column(Role, { default: "member" }) } }) + const entities = yield* S.pgEntitiesOf([Users]) + const snapshot = yield* S.makeSnapshot(entities, [S.ORIGIN_ID], "postgres") + const { ops } = S.diffPgSchemas([], entities) + const migrations = yield* Migrate.fromRecord({ + // generate does not create schemas. + "20261013000000_auth": { kind: "sql", migration: `CREATE SCHEMA auth;\n--> statement-breakpoint\nCREATE TYPE auth.other AS ENUM ('x')` }, + "20261013000001_users": { + kind: "ops", + migration: JSON.stringify({ version: "1", dialect: "postgres", ops }), + snapshot: S.serializeSnapshot(snapshot), + }, + }) + yield* Migrate.run({ migrations }) + expect((yield* Migrate.verify(migrations)).drift).toEqual([]) + const sql = yield* SqlClient.SqlClient + yield* sql.unsafe(`ALTER TYPE auth.role ADD VALUE 'guest'`) + expect((yield* Migrate.verify(migrations)).drift.map((d) => `${d.problem} ${d.entity}`)).toEqual(["enum enum auth.role"]) + }).pipe(Effect.provide(Live)), + ) + + it.effect("a PG.custom enum type created by hand still verifies clean, qualified or not", () => + Effect.gen(function* () { + const Role = Schema.Literals(["owner", "member"]) + const Members = PG.table("members", { + columns: { + id: PG.text, + role: PG.column(PG.custom("public.organization_role", Role), { default: "member" }), + old_role: PG.nullable(PG.custom("organization_role", Role)), + }, + indexes: [PG.index("members_owner_idx", ["id"], { where: `role = 'owner'::public.organization_role` })], + }) + const entities = yield* S.pgEntitiesOf([Members]) + const snapshot = yield* S.makeSnapshot(entities, [S.ORIGIN_ID], "postgres") + const record: Record = { + "20261013000000_types": { kind: "sql", migration: `CREATE TYPE "public"."organization_role" AS ENUM ('owner', 'member')` }, + "20261013000001_members": { + kind: "sql", + migration: S.renderPgSchema(entities).join("\n--> statement-breakpoint\n"), + snapshot: S.serializeSnapshot(snapshot), + }, + } + const migrations = yield* Migrate.fromRecord(record) + yield* Migrate.run({ migrations, dialect: "postgres" }) + expect((yield* Migrate.verify(migrations)).drift).toEqual([]) + + // Then declared with enumType: the type is kept, the column moves to its unqualified name. + const OrganizationRole = PG.enumType("organization_role", ["owner", "member"]) + const declared = PG.table("members", { + columns: { id: PG.text, role: PG.column(OrganizationRole, { default: "member" }), old_role: PG.nullable(OrganizationRole) }, + indexes: [PG.index("members_owner_idx", ["id"], { where: `role = 'owner'::public.organization_role` })], + }) + const next = yield* S.pgEntitiesOf([declared]) + const { ops } = S.diffPgSchemas(entities, next) + expect(ops.map((op) => op.op)).toEqual(["create_enum", "alter_column"]) + const nextSnapshot = yield* S.makeSnapshot(next, [snapshot.id], "postgres") + const adopted = yield* Migrate.fromRecord({ + ...record, + "20261013000002_enum": { + kind: "ops", + migration: JSON.stringify({ version: "1", dialect: "postgres", ops }), + snapshot: S.serializeSnapshot(nextSnapshot), + }, + }) + expect((yield* Migrate.run({ migrations: adopted })).map((m) => m.name)).toEqual(["20261013000002_enum"]) + expect((yield* Migrate.verify(adopted)).drift).toEqual([]) + }).pipe(Effect.provide(Live)), + ) + + it.effect("a database drizzle-kit pushed, with enums and unique constraints, adopts through a baseline and verifies clean", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient + // What `drizzle-kit push` ran. + const pushed = [ + `CREATE TYPE "public"."channel_type" AS ENUM('public', 'private', 'direct')`, + `CREATE TYPE "public"."message_outbox_event_status" AS ENUM('pending', 'processing', 'done')`, + `CREATE TABLE "channels" ( + "id" text PRIMARY KEY NOT NULL, + "org_id" text NOT NULL, + "slug" text NOT NULL, + "type" "channel_type" DEFAULT 'public' NOT NULL, + "allowed" "channel_type"[], + CONSTRAINT "channels_org_slug_unique" UNIQUE NULLS NOT DISTINCT("org_id","slug") + )`, + `CREATE TABLE "message_outbox" ( + "id" text PRIMARY KEY NOT NULL, + "channel_org_id" text NOT NULL, + "channel_slug" text NOT NULL, + "status" "message_outbox_event_status" DEFAULT 'pending' NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL + )`, + `ALTER TABLE "channels" ADD CONSTRAINT "channels_slug_unique" UNIQUE("slug")`, + `ALTER TABLE "message_outbox" ADD CONSTRAINT "message_outbox_channel_fk" FOREIGN KEY ("channel_org_id","channel_slug") REFERENCES "public"."channels"("org_id","slug") ON DELETE no action ON UPDATE no action`, + `CREATE INDEX "message_outbox_pending_idx" ON "message_outbox" USING btree ("created_at") WHERE "message_outbox"."status" = 'pending'`, + ] + for (const statement of pushed) yield* sql.unsafe(statement) + + // drizzle-kit's snapshot of the same schema. + const column = (table: string, name: string, type: string, extra: Record = {}) => ({ + type, + typeSchema: null, + notNull: true, + dimensions: 0, + default: null, + generated: null, + identity: null, + name, + entityType: "columns", + schema: "public", + table, + ...extra, + }) + const drizzle = { + version: "8", + dialect: "postgres", + id: "d", + prevIds: [], + renames: [], + ddl: [ + { name: "channel_type", schema: "public", values: ["public", "private", "direct"], entityType: "enums" }, + { name: "message_outbox_event_status", schema: "public", values: ["pending", "processing", "done"], entityType: "enums" }, + { isRlsEnabled: false, name: "channels", entityType: "tables", schema: "public" }, + column("channels", "id", "text"), + column("channels", "org_id", "text"), + column("channels", "slug", "text"), + column("channels", "type", "channel_type", { typeSchema: "public", default: "'public'" }), + column("channels", "allowed", "channel_type", { typeSchema: "public", notNull: false, dimensions: 1 }), + { columns: ["id"], nameExplicit: false, name: "channels_pkey", entityType: "pks", schema: "public", table: "channels" }, + { nameExplicit: false, columns: ["slug"], nullsNotDistinct: false, name: "channels_slug_unique", entityType: "uniques", schema: "public", table: "channels" }, + { nameExplicit: true, columns: ["org_id", "slug"], nullsNotDistinct: true, name: "channels_org_slug_unique", entityType: "uniques", schema: "public", table: "channels" }, + { isRlsEnabled: false, name: "message_outbox", entityType: "tables", schema: "public" }, + column("message_outbox", "id", "text"), + column("message_outbox", "channel_org_id", "text"), + column("message_outbox", "channel_slug", "text"), + column("message_outbox", "status", "message_outbox_event_status", { typeSchema: "public", default: "'pending'" }), + column("message_outbox", "created_at", "timestamp with time zone", { default: "now()" }), + { columns: ["id"], nameExplicit: false, name: "message_outbox_pkey", entityType: "pks", schema: "public", table: "message_outbox" }, + { + nameExplicit: false, + columns: ["channel_org_id", "channel_slug"], + schemaTo: "public", + tableTo: "channels", + columnsTo: ["org_id", "slug"], + onUpdate: "NO ACTION", + onDelete: "NO ACTION", + name: "message_outbox_channel_fk", + entityType: "fks", + schema: "public", + table: "message_outbox", + }, + { + nameExplicit: true, + columns: [{ value: "created_at", isExpression: false, asc: true, nullsFirst: false, opclass: null }], + isUnique: false, + where: `"message_outbox"."status" = 'pending'`, + with: "", + method: "btree", + concurrently: false, + name: "message_outbox_pending_idx", + entityType: "indexes", + schema: "public", + table: "message_outbox", + }, + ], + } + + // `generate --baseline --from-drizzle`, then `effect-orm baseline`. + const imported = S.fromDrizzleSnapshot(drizzle) + expect(imported.unsupported).toEqual([]) + const base = yield* S.makeSnapshot(imported.entities, [S.ORIGIN_ID], "postgres") + const migrations = yield* Migrate.fromRecord({ + "20260101000000_drizzle_init": { kind: "sql", migration: pushed.join("\n--> statement-breakpoint\n"), snapshot: JSON.stringify(drizzle) }, + "20261013000000_effect_orm_baseline": { kind: "sql", migration: "-- Baseline: runs nothing.\n", snapshot: S.serializeSnapshot(base) }, + }) + yield* Migrate.baseline({ migrations, upTo: "20261013000000_effect_orm_baseline" }) + expect(yield* Migrate.verify(migrations)).toEqual({ against: "20261013000000_effect_orm_baseline", drift: [] }) + + // The same schema declared with enumType and unique: the next generate has nothing to write. + const { Channels, Outbox } = schemaWith() + expect(S.diffPgSchemas(imported.entities, yield* S.pgEntitiesOf([ChannelType, OutboxStatus, Channels, Outbox])).ops).toEqual([]) + }).pipe(Effect.provide(Live)), + ) +}) diff --git a/src/migrate/pg-verify.ts b/src/migrate/pg-verify.ts index 0abbf49..fad012e 100644 --- a/src/migrate/pg-verify.ts +++ b/src/migrate/pg-verify.ts @@ -8,15 +8,24 @@ // is always rolled back, and the two catalogs are read with the same queries: // the server deparses both sides, and equal definitions read back equal. // -// Checked: tables, columns (type, NOT NULL, default), primary keys, indexes -// (uniqueness, method, keys, predicate), foreign keys (columns, target, -// actions). Not checked: anything the entity model does not hold (triggers, -// grants, check constraints), and tables outside the current schema. +// The scratch schema comes first on the search path and the current schema +// second, so a type of the current schema (an enum the snapshot does not +// declare, say, behind `PG.custom`) resolves there as it does live, and both +// sides deparse it unqualified: `'open'::status`, never `'open'::public.status`. +// An enum the snapshot declares is used as it exists live, or created in the +// scratch schema when it does not; its labels are compared with the snapshot +// directly. +// +// Checked: enums (labels, in order), tables, columns (type, NOT NULL, +// default), primary keys, unique constraints (columns, NULLS NOT DISTINCT), +// indexes (uniqueness, method, keys, predicate), foreign keys (columns, +// target, actions). Not checked: anything the entity model does not hold +// (triggers, grants, check constraints), and tables outside the current schema. import { Effect, Schema } from "effect" import type { PgSnapshot } from "../schema/entities" -import { renderPgSchema } from "../schema/pg-ops" -import type { PgReferentialAction } from "../schema/pg-entities" +import { pgIdent, renderPgCreateEnum, renderPgSchema } from "../schema/pg-ops" +import { pgTypeName, type PgEnumEntity, type PgReferentialAction } from "../schema/pg-entities" import { MigrationDriver } from "./driver" import type { MigrateSourceError, MigrateSqlError } from "./errors" import { LEDGER_TABLES } from "./ledger" @@ -29,6 +38,7 @@ interface CatalogTable { readonly primaryKey: { readonly name: string; readonly columns: ReadonlyArray } | null readonly columns: Map readonly indexes: Map; readonly where: string | null }> + readonly uniques: Map; readonly nullsNotDistinct: boolean }> readonly foreignKeys: Map< string, { @@ -70,6 +80,7 @@ const readCatalog = (schema: string) => ) const constraints = yield* driver.query( `SELECT con.conname AS name, c.relname AS tbl, con.contype AS type, fc.relname AS foreign_table, + pg_get_constraintdef(con.oid) AS def, con.confdeltype AS on_delete, con.confupdtype AS on_update, (SELECT json_agg(att.attname::text ORDER BY k.ord) FROM unnest(con.conkey) WITH ORDINALITY k(attnum, ord) JOIN pg_attribute att ON att.attrelid = con.conrelid AND att.attnum = k.attnum)::text AS cols, @@ -79,7 +90,7 @@ const readCatalog = (schema: string) => JOIN pg_class c ON c.oid = con.conrelid JOIN pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_class fc ON fc.oid = con.confrelid - WHERE ${ns} AND con.contype IN ('p', 'f')`, + WHERE ${ns} AND con.contype IN ('p', 'f', 'u')`, ) // Key parts one by one, with their sort order: pg_get_indexdef(oid, k, true) // writes a key without DESC / NULLS, which live in indoption. @@ -104,7 +115,7 @@ const readCatalog = (schema: string) => const out = new Map() for (const row of tables) { - out.set(String(row.name), { primaryKey: null, columns: new Map(), indexes: new Map(), foreignKeys: new Map() }) + out.set(String(row.name), { primaryKey: null, columns: new Map(), indexes: new Map(), uniques: new Map(), foreignKeys: new Map() }) } for (const row of columns) { out.get(String(row.tbl))?.columns.set(String(row.name), { @@ -119,6 +130,12 @@ const readCatalog = (schema: string) => if (table === undefined) continue if (row.type === "p") { out.set(String(row.tbl), { ...table, primaryKey: { name: String(row.name), columns: jsonList(row.cols) } }) + } else if (row.type === "u") { + table.uniques.set(String(row.name), { + columns: jsonList(row.cols), + // Read from the definition: pg_index.indnullsnotdistinct only exists from Postgres 15. + nullsNotDistinct: /^UNIQUE NULLS NOT DISTINCT\b/i.test(String(row.def)), + }) } else { table.foreignKeys.set(String(row.name), { columns: jsonList(row.cols), @@ -140,13 +157,34 @@ const readCatalog = (schema: string) => return out }) +/** + * The labels of every enum in `schemas`, keyed by its name as a column type + * spells it from `current`: bare in the current schema, qualified elsewhere. + */ +const readEnums = (schemas: ReadonlyArray, current: string) => + Effect.gen(function* () { + const driver = yield* MigrationDriver + const rows = yield* driver.query( + `SELECT n.nspname AS schema, t.typname AS name, + (SELECT json_agg(e.enumlabel::text ORDER BY e.enumsortorder) FROM pg_enum e WHERE e.enumtypid = t.oid)::text AS labels + FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE t.typtype = 'e' AND n.nspname IN (${schemas.map(q).join(", ")})`, + ) + return new Map( + rows.map((row) => { + const local = String(row.schema) === current + return [pgTypeName(local ? null : String(row.schema), String(row.name)), { labels: jsonList(row.labels), local }] as const + }), + ) + }) + /** Rolls the scratch transaction back once its catalog is read. */ class Rollback extends Schema.TaggedError()("@maple-dev/effect-orm/VerifyRollback", { expected: Schema.Unknown, }) {} /** The snapshot as Postgres would store it: built in a scratch schema and rolled back. */ -const expectedCatalog = (snapshot: PgSnapshot, migration: string) => +const expectedCatalog = (snapshot: PgSnapshot, migration: string, current: string, liveEnums: ReadonlySet) => Effect.gen(function* () { const driver = yield* MigrationDriver const transaction = yield* transactionOf(driver, migration) @@ -154,9 +192,18 @@ const expectedCatalog = (snapshot: PgSnapshot, migration: string) => const result = yield* transaction( Effect.gen(function* () { yield* driver.execute(`CREATE SCHEMA "${scratch}"`) - // pg_catalog stays implicitly first, so built-ins resolve as they do in the real schema. - yield* driver.execute(`SET LOCAL search_path TO "${scratch}"`) - for (const statement of renderPgSchema(snapshot.entities)) yield* driver.execute(statement) + // pg_catalog stays implicitly first, so built-ins resolve as they do in the real schema; + // the current schema comes after the scratch one, so its types do too. + yield* driver.execute(`SET LOCAL search_path TO "${scratch}", ${pgIdent(current)}`) + // An enum that exists is used as it is, so a column or predicate naming it + // resolves to the same type on both sides; its labels are compared apart. + // One that does not is created: in the scratch schema, or in its own. + const enums = snapshot.entities.filter((e): e is PgEnumEntity => e.kind === "enum" && !liveEnums.has(pgTypeName(e.schema, e.name))) + for (const schema of new Set(enums.flatMap((e) => (e.schema === null ? [] : [e.schema])))) { + yield* driver.execute(`CREATE SCHEMA IF NOT EXISTS ${pgIdent(schema)}`) + } + for (const entity of enums) yield* driver.execute(renderPgCreateEnum(entity)) + for (const statement of renderPgSchema(snapshot.entities.filter((e) => e.kind !== "enum"))) yield* driver.execute(statement) const expected = yield* readCatalog(scratch) return yield* new Rollback({ expected }) }), @@ -182,7 +229,9 @@ export const verifyPg = ( const driver = yield* MigrationDriver const schema = String((yield* driver.query("SELECT current_schema() AS s"))[0]?.s ?? "public") const actual = yield* readCatalog(schema) - const expected = yield* expectedCatalog(snapshot, migration) + const declared = snapshot.entities.filter((e): e is PgEnumEntity => e.kind === "enum") + const liveEnums = yield* readEnums([schema, ...new Set(declared.flatMap((e) => (e.schema === null ? [] : [e.schema])))], schema) + const expected = yield* expectedCatalog(snapshot, migration, schema, new Set(liveEnums.keys())) const ignored = new Set([...Object.values(LEDGER_TABLES), ...(options.ignoreTables ?? [])]) const drift: Array = [] const push = (entity: string, problem: Drift["problem"], expectedValue?: unknown, actualValue?: unknown) => @@ -193,6 +242,18 @@ export const verifyPg = ( ...(actualValue !== undefined ? { actual: show(actualValue) } : undefined), }) + for (const entity of declared) { + const name = pgTypeName(entity.schema, entity.name) + const live = liveEnums.get(name) + if (live === undefined) push(`enum ${name}`, "missing") + else if (!same(entity.values, live.labels)) push(`enum ${name}`, "enum", entity.values, live.labels) + } + // An enum the snapshot does not declare is drift unless a column uses it (a `PG.custom` type). + const used = new Set([...expected.values()].flatMap((t) => [...t.columns.values()].map((c) => c.type.replace(/(\[\])+$/, "")))) + const names = new Set(declared.map((e) => pgTypeName(e.schema, e.name))) + // Another schema's enums are read only for the ones the snapshot declares there. + for (const [name, { local }] of liveEnums) if (local && !names.has(name) && !used.has(name)) push(`enum ${name}`, "unexpected") + for (const [name, want] of expected) { const have = actual.get(name) if (have === undefined) { @@ -218,6 +279,12 @@ export const verifyPg = ( else if (!same(w, h)) push(`${name}.${index}`, "index", w, h) } for (const index of have.indexes.keys()) if (!want.indexes.has(index)) push(`${name}.${index}`, "unexpected") + for (const [unique, w] of want.uniques) { + const h = have.uniques.get(unique) + if (h === undefined) push(`${name}.${unique}`, "missing") + else if (!same(w, h)) push(`${name}.${unique}`, "unique", w, h) + } + for (const unique of have.uniques.keys()) if (!want.uniques.has(unique)) push(`${name}.${unique}`, "unexpected") for (const [fk, w] of want.foreignKeys) { const h = have.foreignKeys.get(fk) if (h === undefined) push(`${name}.${fk}`, "missing") diff --git a/src/migrate/verify.ts b/src/migrate/verify.ts index 354cbb2..23369d6 100644 --- a/src/migrate/verify.ts +++ b/src/migrate/verify.ts @@ -34,6 +34,8 @@ export interface Drift { | "identity" | "default" | "index" + | "unique" + | "enum" | "foreign_key" | "view_target" | "view_body" diff --git a/src/pg/enum.test-d.ts b/src/pg/enum.test-d.ts new file mode 100644 index 0000000..645a896 --- /dev/null +++ b/src/pg/enum.test-d.ts @@ -0,0 +1,71 @@ +// An enum column reads, writes and compares as its labels, through +// `PG.column`, `nullable` and `array`, and a `unique` column stays required. + +import { expectTypeOf } from "expect-type" +import * as PG from "../postgres" +import * as S from "../schema" +import type { RowOf } from "../database" + +const ChannelType = PG.enumType("channel_type", ["public", "private", "direct"]) +type ChannelType = "public" | "private" | "direct" + +expectTypeOf(ChannelType.values).toEqualTypeOf>() +expectTypeOf(ChannelType.name).toEqualTypeOf<"channel_type">() +expectTypeOf(ChannelType).toExtend>() +expectTypeOf(ChannelType).toExtend() + +const Channels = PG.table("channels", { + columns: { + id: PG.text, + slug: PG.column(PG.text, { unique: true }), + type: PG.column(ChannelType, { default: "public" }), + previous: PG.nullable(ChannelType), + allowed: PG.array(ChannelType), + }, + primaryKey: ["id"], + uniqueConstraints: [PG.unique("channels_id_slug_unique", ["id", "slug"])], +}) + +// Rows hold the labels. +expectTypeOf>().toEqualTypeOf<{ + readonly id: string + readonly slug: string + readonly type: ChannelType + readonly previous: ChannelType | null + readonly allowed: ReadonlyArray +}>() +const selected = PG.from(Channels).select("type", "allowed") +expectTypeOf>().toEqualTypeOf<{ readonly type: ChannelType; readonly allowed: ReadonlyArray }>() + +// Inserts: a default makes the column optional, `unique` does not; only labels are accepted. +expectTypeOf>().toEqualTypeOf<{ + readonly id: PG.InsertValue + readonly slug: PG.InsertValue + readonly allowed: PG.InsertValue> + readonly type?: PG.InsertValue | undefined + readonly previous?: PG.InsertValue> | undefined +}>() +expectTypeOf<"public">().toExtend>() +expectTypeOf<"secret">().not.toExtend>() +PG.insertInto(Channels).values({ id: "c", slug: "s", previous: null, allowed: ["public"] }) +// @ts-expect-error not a label +PG.insertInto(Channels).values({ id: "c", slug: "s", type: "secret", previous: null, allowed: [] }) +// @ts-expect-error nor in an array +PG.insertInto(Channels).values({ id: "c", slug: "s", previous: null, allowed: ["secret"] }) +// @ts-expect-error `unique` does not make a column optional +PG.insertInto(Channels).values({ id: "c", previous: null, allowed: [] }) +// @ts-expect-error a default must be a label +PG.column(ChannelType, { default: "secret" }) + +// Comparisons take labels, a column of the enum, or a param. +PG.from(Channels).select("id").where(($) => [$.type.eq("public"), $.type.in_("private", "direct"), $.previous.eq($.type), $.type.eq(PG.param.of(ChannelType, "t"))]) +// @ts-expect-error not a label +PG.from(Channels).select("id").where(($) => [$.type.eq("secret")]) +// @ts-expect-error nor in a list +PG.from(Channels).select("id").where(($) => [$.type.in_("public", "secret")]) + +// The conflict target is the constraint's columns, or its name. +PG.insertInto(Channels).values({ id: "c", slug: "s", previous: null, allowed: [] }).onConflictDoNothing({ target: ["slug"] }) +PG.insertInto(Channels).values({ id: "c", slug: "s", previous: null, allowed: [] }).onConflictDoNothing({ target: { constraint: "channels_slug_unique" } }) +// @ts-expect-error not a column +PG.insertInto(Channels).values({ id: "c", slug: "s", previous: null, allowed: [] }).onConflictDoNothing({ target: ["nope"] }) diff --git a/src/pg/types.ts b/src/pg/types.ts index f42b8c7..2c066d7 100644 --- a/src/pg/types.ts +++ b/src/pg/types.ts @@ -177,11 +177,71 @@ export type PgNullable> = PgType< InferEncoded | null > +/** + * A one-dimensional array literal (`{a,"b c",NULL}`) as its elements, or + * `undefined` for anything else. Drivers parse the arrays of built-in types + * themselves, but not of an enum: node-postgres and PGlite send `{a,b}`. + */ +export const parsePgArrayLiteral = (text: string): Array | undefined => { + if (!text.startsWith("{") || !text.endsWith("}")) return undefined + const body = text.slice(1, -1) + const out: Array = [] + if (body.trim() === "") return out + let i = 0 + for (;;) { + if (body[i] === '"') { + let value = "" + for (i++; i < body.length && body[i] !== '"'; i++) { + if (body[i] === "\\") i++ + value += body[i] ?? "" + } + if (i >= body.length) return undefined + i++ + out.push(value) + } else { + const end = body.indexOf(",", i) + const raw = body.slice(i, end < 0 ? body.length : end).trim() + if (raw.includes("{") || raw.includes('"')) return undefined + out.push(raw.toUpperCase() === "NULL" ? null : raw) + i = end < 0 ? body.length : end + } + if (i >= body.length) return out + if (body[i] !== ",") return undefined + i++ + } +} + +const isArrayLiteral = Schema.makeFilter( + (value: string) => (parsePgArrayLiteral(value) === undefined ? `\`${value}\` is not a one-dimensional array literal` : undefined), + { title: "postgresArrayLiteral" }, +) + +/** + * An enum array as a driver sends it: an array, or its text form. Written as + * text, the first member, since a driver that does not know the type cannot + * serialize an array for it either. + */ +const enumArraySchema = (element: Schema.Codec): Schema.Codec => + Schema.Union([ + Schema.String.pipe( + Schema.check(isArrayLiteral), + Schema.decodeTo(Schema.Array(element), { + decode: SchemaGetter.transform((value: string) => parsePgArrayLiteral(value)!), + encode: SchemaGetter.transform((values: ReadonlyArray) => `{${values.map((v) => (v === null ? "NULL" : `"${v.replace(/["\\]/g, "\\$&")}"`)).join(",")}}`), + }), + ), + Schema.Array(element), + ]) + +/** Whether `e` is an enum, or a nullable one: its arrays arrive as text. */ +const isEnumElement = (e: PgType): boolean => + e._tag === "PgEnum" || (e._tag === "Nullable" && e.element?._tag === "PgEnum") + export const array = >(e: E): PgArray => ({ _tag: "Array", sql: `${e.sql}[]`, - schema: Schema.Array(e.schema), - literalSchema: Schema.Array(e.literalSchema) as Schema.Codec, + schema: isEnumElement(e) ? enumArraySchema(e.schema) : Schema.Array(e.schema), + literalSchema: (isEnumElement(e) ? enumArraySchema(e.literalSchema) : Schema.Array(e.literalSchema)) as Schema.Codec, element: e, }) as PgArray diff --git a/src/postgres.ts b/src/postgres.ts index 421cbbf..1610096 100644 --- a/src/postgres.ts +++ b/src/postgres.ts @@ -21,14 +21,22 @@ export { nullIf } from "./ch/functions" export { postgresDialect } // Tables and their DDL. `table` is the only way to declare a table: it carries -// the keys, indexes and foreign keys `effect-orm generate` diffs. +// the keys, indexes, unique constraints and foreign keys `effect-orm generate` +// diffs. `enumType` is a column type and a schema object both. export { table, column, index, uniqueIndex, + unique, foreignKey, defaultForeignKeyName, + enumType, + type EnumOptions, + type PgEnum, + type UniqueColumnOptions, + type UniqueOptions, + type UniqueSpec, type ColumnInput, type ColumnOptions, type ColumnSpec, diff --git a/src/schema.ts b/src/schema.ts index 44622fb..7fe9f4d 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -64,19 +64,24 @@ export { } from "./schema/snapshot" export { PgColumnEntity, + PgEnumEntity, PgForeignKeyEntity, PgIdentity, PgIndexEntity, PgReferentialAction, PgSchemaEntity, PgTableEntity, + PgUniqueEntity, canonicalPgType, + pgTypeName, } from "./schema/pg-entities" export { PgMigrationFile, PgMigrationOp, labelOfPg, pgIdent, + renderPgAddUnique, + renderPgCreateEnum, renderPgOp, renderPgSchema, type PgOpLabel, diff --git a/src/schema/diff.ts b/src/schema/diff.ts index 51287e8..2abc6bd 100644 --- a/src/schema/diff.ts +++ b/src/schema/diff.ts @@ -28,10 +28,19 @@ import type { MigrationOp } from "./ops" export const Hint = Schema.Union([ Schema.Struct({ type: Schema.Literal("confirm_data_loss"), - kind: Schema.Literals(["table", "column"]), - /** `table` or `table.column`. */ + /** `enum`: dropping a Postgres enum type. */ + kind: Schema.Literals(["table", "column", "enum"]), + /** `table`, `table.column`, or the enum's name. */ entity: Schema.String, }), + /** A Postgres enum label renamed in place (`ALTER TYPE ... RENAME VALUE`): rows holding it now hold `to`. */ + Schema.Struct({ + type: Schema.Literal("confirm_rename"), + kind: Schema.Literal("enum_value"), + /** `.`. */ + entity: Schema.String, + to: Schema.String, + }), ]) export type Hint = typeof Hint.Type @@ -48,7 +57,9 @@ export interface DiffResult { readonly unsupported: ReadonlyArray } -const hintKey = (hint: Hint): string => `${hint.type}:${hint.kind}:${hint.entity}` +/** Hints match on this; a rename's target is part of what was confirmed. */ +export const hintKey = (hint: Hint): string => + `${hint.type}:${hint.kind}:${hint.entity}${hint.type === "confirm_rename" ? `>${hint.to}` : ""}` const byKind = (entities: ReadonlyArray, kind: K) => new Map( diff --git a/src/schema/drizzle.ts b/src/schema/drizzle.ts index 7072726..acc493c 100644 --- a/src/schema/drizzle.ts +++ b/src/schema/drizzle.ts @@ -7,12 +7,24 @@ // The first `generate` after it then shows, as ordinary ops, every place the // definitions and the database disagree. // -// Only what the entity model holds converts: tables, columns, primary keys, -// indexes, foreign keys in the `public` schema. Anything else (enums, -// sequences, views, policies, check and unique constraints, identity and -// generated columns, other schemas) is reported, and nothing is written. +// Only what the entity model holds converts: enums, tables, columns, primary +// keys, indexes, unique constraints and foreign keys in the `public` schema, +// and enums in other schemas. Anything else (sequences, views, policies, +// check constraints, generated columns, identities with custom sequences, +// tables in other schemas) is reported, and nothing is written. +// +// `public` is read as the schema migrations run in, as drizzle-kit reads it: +// a `public` enum is unqualified, so a column of it has the type `name`, as +// `format_type` writes it. -import { canonicalPgType, type PgIdentity, type PgReferentialAction, type PgSchemaEntity, type PgTableEntity } from "./pg-entities" +import { + canonicalPgType, + pgTypeName, + type PgIdentity, + type PgReferentialAction, + type PgSchemaEntity, + type PgTableEntity, +} from "./pg-entities" import { sortEntities } from "./entities" export interface DrizzleImport { @@ -102,6 +114,10 @@ export const fromDrizzleSnapshot = (json: unknown): DrizzleImport => { if (str(raw.name) !== "public") unsupported.push(`schema ${str(raw.name)}`) continue } + if (type === "enums") { + entities.push({ kind: "enum", schema: schema === "public" ? null : schema, name: str(raw.name), values: strings(raw.values) }) + continue + } if (schema !== "public") { unsupported.push(`${where} (schema ${schema})`) continue @@ -114,7 +130,9 @@ export const fromDrizzleSnapshot = (json: unknown): DrizzleImport => { } case "columns": { const table = str(raw.table) - if (raw.typeSchema !== null && raw.typeSchema !== undefined) unsupported.push(`${where}: an enum or custom type`) + // drizzle-kit sets `typeSchema` for an enum column; `type` is then the enum's bare name. + const typeSchema = raw.typeSchema === null || raw.typeSchema === undefined ? undefined : str(raw.typeSchema) + const baseType = typeSchema === undefined ? str(raw.type) : pgTypeName(typeSchema === "public" ? null : typeSchema, str(raw.type)) if (raw.generated !== null && raw.generated !== undefined) unsupported.push(`${where}: a generated column`) const identity = identityOf(raw.identity, str(raw.table), str(raw.name), str(raw.type)) if (typeof identity === "string" && identity.startsWith("!")) unsupported.push(`${where}: ${identity.slice(1)}`) @@ -126,7 +144,7 @@ export const fromDrizzleSnapshot = (json: unknown): DrizzleImport => { table, name: str(raw.name), position, - type: canonicalPgType(`${str(raw.type)}${"[]".repeat(dimensions)}`), + type: typeSchema === undefined ? canonicalPgType(`${baseType}${"[]".repeat(dimensions)}`) : `${baseType}${"[]".repeat(dimensions)}`, notNull: raw.notNull === true, default: defaultOf(raw.default), identity: identity === "always" || identity === "by default" ? identity : null, @@ -152,6 +170,16 @@ export const fromDrizzleSnapshot = (json: unknown): DrizzleImport => { }) break } + case "uniques": { + entities.push({ + kind: "unique", + table: str(raw.table), + name: str(raw.name), + columns: strings(raw.columns), + nullsNotDistinct: raw.nullsNotDistinct === true, + }) + break + } case "fks": { if (raw.schemaTo !== undefined && raw.schemaTo !== "public") unsupported.push(`${where}: references schema ${str(raw.schemaTo)}`) entities.push({ diff --git a/src/schema/entities.ts b/src/schema/entities.ts index dbdd0e8..66788cd 100644 --- a/src/schema/entities.ts +++ b/src/schema/entities.ts @@ -114,17 +114,23 @@ export type AnySchemaEntity = SchemaEntity | PgSchemaEntity /** A stable key per entity, unique within a snapshot: `:`, or `:
.` for a table's parts. */ export const entityKey = (entity: AnySchemaEntity): string => - "table" in entity ? `${entity.kind}:${entity.table}.${entity.name}` : `${entity.kind}:${entity.name}` + "table" in entity + ? `${entity.kind}:${entity.table}.${entity.name}` + : entity.kind === "enum" && entity.schema !== null + ? `${entity.kind}:${entity.schema}.${entity.name}` + : `${entity.kind}:${entity.name}` const kindOrder: Record = { - table: 0, - column: 1, - index: 2, - foreign_key: 3, - materialized_view: 4, + enum: 0, + table: 1, + column: 2, + index: 3, + unique: 4, + foreign_key: 5, + materialized_view: 6, } -/** Tables, then columns in declaration order, then indexes, foreign keys, views. Deterministic. */ +/** Enums, tables, then columns in declaration order, then indexes, unique constraints, foreign keys, views. Deterministic. */ export const sortEntities = (entities: ReadonlyArray): ReadonlyArray => [...entities].sort((a, b) => { const byKind = kindOrder[a.kind] - kindOrder[b.kind] diff --git a/src/schema/pg-define.ts b/src/schema/pg-define.ts index a2bdb93..956d461 100644 --- a/src/schema/pg-define.ts +++ b/src/schema/pg-define.ts @@ -1,5 +1,5 @@ -// Postgres schema definitions: `table` from `/postgres` and its column, index and foreign -// key helpers. +// Postgres schema definitions: `table` from `/postgres` and its column, index, +// unique constraint and foreign key helpers, and `enumType`. // // The Postgres counterpart of the ClickHouse `table`: the value IS a `Table`, so every // query API accepts it, and its DDL rides beside it on `ddl` as Postgres @@ -7,6 +7,7 @@ // rendered once, here, with the Postgres dialect. A definition that cannot // become DDL records a problem, and `pgEntitiesOf` fails with it. +import { Schema } from "effect" import { compile as compileFragment } from "../sql/sql-fragment" import { withDialect } from "../ch/dialect" import type { Condition, Expr } from "../ch/expr" @@ -14,18 +15,22 @@ import { encodeColumnLiteral } from "../ch/literal" import { createColumnAccessor, type ColumnAccessor } from "../ch/query" import type { Table } from "../ch/table" import { columnSqlName, type CHType, type ColumnDefs, type InferTS } from "../ch/types" +import type { PgType } from "../pg/types" import { postgresDialect } from "../pg/dialect" import { externalTable, type DdlExpr, type DdlKey } from "./define" import { withRenderProblems, type DefinitionProblem, type ProblemSink } from "./problems" import { canonicalPgType, PG_MAX_IDENTIFIER, + pgTypeName, type PgColumnEntity, + type PgEnumEntity, type PgForeignKeyEntity, type PgIdentity, type PgIndexEntity, type PgReferentialAction, type PgTableEntity, + type PgUniqueEntity, } from "./pg-entities" const quoteIdent = (name: string): string => `"${name.replace(/"/g, '""')}"` @@ -63,6 +68,18 @@ export interface ColumnOptions> { * out; with `"always"`, Postgres also rejects an insert that gives it. */ readonly identity?: PgIdentity + /** + * A one-column `UNIQUE` constraint. `true` names it + * `
__unique`, drizzle-orm's name; a string names it. + */ + readonly unique?: true | string | UniqueColumnOptions +} + +export interface UniqueColumnOptions { + /** Default `
__unique`. */ + readonly name?: string + /** `NULLS NOT DISTINCT` (Postgres 15+). */ + readonly nullsNotDistinct?: boolean } /** `Given` records which options were set, so the table knows which columns an insert may leave out. */ @@ -94,7 +111,7 @@ export type ColumnsOf> = { /** Columns declared with `default`, `defaultExpr` or `identity`: an insert may leave them out. */ export type DefaultedColumnsOf> = { [K in keyof I]: I[K] extends ColumnSpec - ? [Exclude] extends [never] + ? [Exclude] extends [never] ? never : K : never @@ -188,6 +205,106 @@ export const foreignKey = { + readonly _tag: "PgUniqueSpec" + readonly name: string + readonly columns: ReadonlyArray + readonly options: UniqueOptions +} + +/** + * A `UNIQUE` constraint: `ALTER TABLE ... ADD CONSTRAINT name UNIQUE (columns)`, + * what drizzle-kit writes for `unique()`. Postgres backs it with a unique index + * of the same name, but it is a constraint: drop it with `DROP CONSTRAINT`, and + * name it in `onConflictDoNothing({ target: { constraint } })`. + */ +export const unique = ( + name: string, + columns: ReadonlyArray, + options: UniqueOptions = {}, +): UniqueSpec => ({ _tag: "PgUniqueSpec", name, columns, options }) + +// Enums + +export interface EnumOptions { + /** The schema the type lives in. Omit it for the schema the migrations run in. */ + readonly schema?: string +} + +/** + * A Postgres enum type. It is a column type (rows decode to `Values`, and + * inserts and comparisons take only its labels) and a schema object + * (`generate` creates and alters the type, before any table that uses it). + */ +export interface PgEnum extends PgType<"PgEnum", Values, string> { + /** The type's name, without its schema. */ + readonly name: Name + /** The labels, in sort order. */ + readonly values: ReadonlyArray + readonly ddl: { readonly dialect: "postgres"; readonly enum: PgEnumEntity } + /** What is wrong with the definition; `pgEntitiesOf` fails when any object has one. */ + readonly problems: ReadonlyArray +} + +/** + * `CREATE TYPE name AS ENUM (values...)`: `PG.enumType("channel_type", ["public", "private"])`. + * Use it as a column type, bare or in `PG.column`, `PG.nullable` and `PG.array`, + * and export it from a schema module, or let the tables that use it bring it. + */ +export const enumType = ]>( + name: Name, + values: Values, + options: EnumOptions = {}, +): PgEnum => { + const schema = options.schema ?? null + const entity: PgEnumEntity = { kind: "enum", schema, name, values: [...values] } + const literals = Schema.Literals(values) as unknown as Schema.Codec + return { + _tag: "PgEnum", + sql: pgTypeName(schema, name), + schema: literals, + literalSchema: literals as Schema.Codec, + name, + values: [...values], + ddl: { dialect: "postgres", enum: entity }, + problems: enumProblems(entity), + } +} + +/** What is wrong with an enum: its names, and labels that are empty, repeated or too long. */ +export const enumProblems = (entity: PgEnumEntity): ReadonlyArray => { + const problems: ProblemSink = [] + const object = entity.schema === null ? entity.name : `${entity.schema}.${entity.name}` + checkIdentifier(problems, object, entity.name) + if (entity.schema !== null) checkIdentifier(problems, object, entity.schema) + if (entity.values.length === 0) problems.push({ object, message: "an enum needs at least one value" }) + const seen = new Set() + for (const value of entity.values) { + if (seen.has(value)) problems.push({ object, message: `${JSON.stringify(value)} is listed twice` }) + seen.add(value) + if (value.length === 0) problems.push({ object, message: "a label cannot be empty" }) + if (new TextEncoder().encode(value).length > PG_MAX_IDENTIFIER) { + problems.push({ object, message: `${JSON.stringify(value)} is longer than ${PG_MAX_IDENTIFIER} bytes, the most a label holds` }) + } + } + return problems +} + +const isPgEnum = (type: CHType): type is PgEnum => type._tag === "PgEnum" && "ddl" in type + +/** The enum a column type is, or holds through `nullable` / `array`. */ +export const enumOfType = (type: CHType): PgEnum | undefined => { + for (let t: CHType | undefined = type; t !== undefined; t = t.element) { + if (isPgEnum(t)) return t + } + return undefined +} + // Tables export interface TableDefinition> { @@ -197,6 +314,8 @@ export interface TableDefinition> { | ReadonlyArray | { readonly columns: ReadonlyArray; readonly name?: string } readonly indexes?: ReadonlyArray>> + /** `UNIQUE` constraints over several columns; one column can say `unique` in `PG.column`. */ + readonly uniqueConstraints?: ReadonlyArray> readonly foreignKeys?: ReadonlyArray> /** The column carrying row-level tenancy; see docs/tenant-scoping.md. */ readonly tenantColumn?: keyof Columns & string @@ -215,7 +334,10 @@ export interface TableDdl { readonly table: PgTableEntity readonly columns: ReadonlyArray readonly indexes: ReadonlyArray + readonly uniques: ReadonlyArray readonly foreignKeys: ReadonlyArray + /** The enum types its columns use, so a schema has them without exporting them. */ + readonly enums: ReadonlyArray } export interface PgSchemaTable @@ -397,6 +519,42 @@ function buildTable = [] + for (const [column, input] of inputs) { + const option = isColumnSpec(input) ? input.options.unique : undefined + if (option === undefined) continue + const config: UniqueColumnOptions = option === true ? {} : typeof option === "string" ? { name: option } : option + const uniqueName = config.name ?? `${name}_${sqlName(column)}_unique` + checkIdentifier(problems, `${name}.${column} unique`, uniqueName) + uniqueEntities.push({ + kind: "unique", + table: name, + name: uniqueName, + columns: [sqlName(column)], + nullsNotDistinct: config.nullsNotDistinct === true, + }) + } + for (const spec of definition.uniqueConstraints ?? []) { + checkIdentifier(problems, `${name} unique`, spec.name) + for (const column of spec.columns) { + if (!has(column)) problems.push({ object: `${name} unique ${spec.name}`, message: `${column} is not a column` }) + } + if (spec.columns.length === 0) problems.push({ object: `${name} unique ${spec.name}`, message: "a unique constraint needs columns" }) + uniqueEntities.push({ + kind: "unique", + table: name, + name: spec.name, + columns: spec.columns.map(sqlName), + nullsNotDistinct: spec.options.nullsNotDistinct === true, + }) + } + + const enumEntities = new Map() + for (const [, input] of inputs) { + const found = enumOfType(isColumnSpec(input) ? input.type : input) + if (found !== undefined) enumEntities.set(found.sql, found.ddl.enum) + } + const foreignKeyEntities = (definition.foreignKeys ?? []).map((spec): PgForeignKeyEntity => { const fkColumns = spec.columns.map(sqlName) // A table referencing itself names its own keys; another table's were mapped by `foreignKey`. @@ -443,7 +601,9 @@ function buildTable `${hint.type}:${hint.kind}:${hint.entity}` - const same = (a: unknown, b: unknown): boolean => canonicalJson(a) === canonicalJson(b) const byKind = (entities: ReadonlyArray, kind: K) => @@ -26,19 +40,28 @@ const byKind = (entities: ReadonlyArray [entityKey(e), e] as const), ) -// Constraints and indexes go first, so a dropped column or table is not still -// referenced; new tables exist before the indexes and keys that need them. +// Enum types and labels come first, so every column can use them; constraints +// and indexes go next, so a dropped column or table is not still referenced; +// new tables exist before the indexes and keys that need them, a unique key +// before a foreign key that references it, and a type is dropped once nothing +// uses it. const opOrder: Record = { - drop_foreign_key: 0, - drop_index: 1, - drop_column: 2, - drop_table: 3, - create_table: 4, - add_column: 5, - alter_column: 6, - set_primary_key: 7, - create_index: 8, - add_foreign_key: 9, + create_enum: 0, + rename_enum_value: 1, + add_enum_value: 2, + drop_foreign_key: 3, + drop_unique: 4, + drop_index: 5, + drop_column: 6, + drop_table: 7, + create_table: 8, + add_column: 9, + alter_column: 10, + set_primary_key: 11, + add_unique: 12, + create_index: 13, + add_foreign_key: 14, + drop_enum: 15, } export const diffPgSchemas = ( @@ -49,12 +72,15 @@ export const diffPgSchemas = ( const given = new Set(hints.map(hintKey)) const ops: Array = [] const missingHints: Array = [] + const unsupported: Array = [] const confirm = (hint: Hint, op: PgMigrationOp): void => { if (given.has(hintKey(hint))) ops.push(op) else missingHints.push(hint) } + const addedLabels = diffEnums(byKind(prev, "enum"), byKind(next, "enum"), ops, confirm, unsupported) + const prevTables = byKind(prev, "table") const nextTables = byKind(next, "table") const prevColumns = byKind(prev, "column") @@ -86,6 +112,10 @@ export const diffPgSchemas = ( drop: (i: PgIndexEntity) => ({ op: "drop_index", table: i.table, name: i.name }), create: (i: PgIndexEntity) => ({ op: "create_index", index: i }), }) + diffNamed(byKind(prev, "unique"), byKind(next, "unique"), prevTables, ops, { + drop: (u: PgUniqueEntity) => ({ op: "drop_unique", table: u.table, name: u.name }), + create: (u: PgUniqueEntity) => ({ op: "add_unique", unique: u }), + }) diffNamed(byKind(prev, "foreign_key"), byKind(next, "foreign_key"), prevTables, ops, { drop: (fk: PgForeignKeyEntity) => ({ op: "drop_foreign_key", table: fk.table, name: fk.name }), create: (fk: PgForeignKeyEntity) => ({ op: "add_foreign_key", foreignKey: fk }), @@ -97,8 +127,110 @@ export const diffPgSchemas = ( .sort(([a, i], [b, j]) => opOrder[a.op] - opOrder[b.op] || i - j) .map(([op]) => op), missingHints, - unsupported: [], + unsupported: [...unsupported, ...usesOfNewLabels(addedLabels, ops, [...byKind(next, "column").values()])], + } +} + +/** Labels added per enum, keyed by the enum's name as a column type spells it. */ +type AddedLabels = Map> + +const enumLabel = (e: PgEnumEntity): string => (e.schema === null ? e.name : `${e.schema}.${e.name}`) + +const diffEnums = ( + before: Map, + after: Map, + ops: Array, + confirm: (hint: Hint, op: PgMigrationOp) => void, + unsupported: Array, +): AddedLabels => { + const added: AddedLabels = new Map() + for (const [key, entity] of after) { + const old = before.get(key) + if (old === undefined) { + ops.push({ op: "create_enum", enum: entity }) + continue + } + if (same(old.values, entity.values)) continue + const ref = { schema: entity.schema, name: entity.name } + const oldSet = new Set(old.values) + const newSet = new Set(entity.values) + const kept = entity.values.filter((v) => oldSet.has(v)) + if (same(kept, old.values)) { + // Only additions: each new label goes before the next label that already exists. + const labels: Array = [] + entity.values.forEach((value, i) => { + if (oldSet.has(value)) return + const before = entity.values.slice(i + 1).find((v) => oldSet.has(v)) ?? null + ops.push({ op: "add_enum_value", ...ref, value, before }) + labels.push(value) + }) + added.set(pgTypeName(entity.schema, entity.name), labels) + continue + } + const swapped = + old.values.length === entity.values.length && + old.values.every((v, i) => v === entity.values[i] || (!newSet.has(v) && !oldSet.has(entity.values[i]!))) + if (swapped) { + old.values.forEach((from, i) => { + const to = entity.values[i]! + if (from === to) return + confirm( + { type: "confirm_rename", kind: "enum_value", entity: `${enumLabel(entity)}.${from}`, to }, + { op: "rename_enum_value", ...ref, from, to }, + ) + }) + continue + } + const removed = old.values.filter((v) => !newSet.has(v)) + unsupported.push({ + entity: enumLabel(entity), + message: `${removed.length > 0 ? `removing ${removed.map((v) => JSON.stringify(v)).join(", ")}` : "reordering labels"} needs the type rebuilt: create a new type, move every column to it, drop the old one. Write it with generate --custom`, + }) + } + for (const [key, entity] of before) { + if (after.has(key)) continue + confirm( + { type: "confirm_data_loss", kind: "enum", entity: enumLabel(entity) }, + { op: "drop_enum", schema: entity.schema, name: entity.name }, + ) + } + return added +} + +/** + * A label `ALTER TYPE ... ADD VALUE` adds cannot be used until its + * transaction commits, and a migration is one transaction. A default or a + * partial-index predicate naming it would fail when the migration runs, so it + * is reported now: add the label in one migration, use it in the next. + */ +const usesOfNewLabels = ( + added: AddedLabels, + ops: ReadonlyArray, + columns: ReadonlyArray, +): ReadonlyArray => { + if (added.size === 0) return [] + const out: Array = [] + const labelsOf = (type: string) => added.get(type.replace(/(\[\])+$/, "")) ?? [] + const mentions = (sql: string | null, labels: ReadonlyArray) => + sql === null ? [] : labels.filter((label) => sql.includes(`'${label.replace(/'/g, "''")}'`)) + const report = (entity: string, labels: ReadonlyArray) => { + for (const label of labels) { + out.push({ + entity, + message: `uses ${JSON.stringify(label)}, which this migration adds; Postgres cannot use a new enum label in the transaction that adds it. Generate the label alone first, then this change`, + }) + } + } + const changedColumns = ops.flatMap((op) => + op.op === "create_table" ? op.columns : op.op === "add_column" ? [op.column] : op.op === "alter_column" ? [op.to] : [], + ) + for (const column of changedColumns) report(`${column.table}.${column.name}`, mentions(column.default, labelsOf(column.type))) + for (const op of ops) { + if (op.op !== "create_index") continue + const labels = columns.filter((c) => c.table === op.index.table).flatMap((c) => labelsOf(c.type)) + report(`${op.index.table}.${op.index.name}`, [...new Set(mentions(op.index.where, labels))]) } + return out } const diffColumns = ( @@ -135,7 +267,7 @@ const diffPrimaryKey = (before: PgTableEntity, after: PgTableEntity, ops: Array< } } -const diffNamed = ( +const diffNamed = ( before: Map, after: Map, prevTables: Map, @@ -153,9 +285,9 @@ const diffNamed = ( } for (const [key, entity] of before) { if (after.has(key)) continue - // Only an index of a dropped table can be left to the DROP TABLE; a foreign - // key is dropped either way, because one pointing *at* the table blocks it. - if (entity.kind === "index" && dropped.has(entity.table) && prevTables.has(`table:${entity.table}`)) continue + // Only an index or unique constraint of a dropped table can be left to the DROP + // TABLE; a foreign key is dropped either way, because one pointing *at* the table blocks it. + if (entity.kind !== "foreign_key" && dropped.has(entity.table) && prevTables.has(`table:${entity.table}`)) continue ops.push(make.drop(entity)) } } diff --git a/src/schema/pg-entities.ts b/src/schema/pg-entities.ts index b1de457..1d743ae 100644 --- a/src/schema/pg-entities.ts +++ b/src/schema/pg-entities.ts @@ -67,9 +67,52 @@ export const PgForeignKeyEntity = Schema.Struct({ }) export type PgForeignKeyEntity = typeof PgForeignKeyEntity.Type -export const PgSchemaEntity = Schema.Union([PgTableEntity, PgColumnEntity, PgIndexEntity, PgForeignKeyEntity]) +/** + * An enum type (`CREATE TYPE ... AS ENUM`). `schema` is `null` for the schema + * migrations run in, which is how a column of the type spells it too. + */ +export const PgEnumEntity = Schema.Struct({ + kind: Schema.Literal("enum"), + schema: Schema.NullOr(Schema.String), + name: Schema.String, + /** The labels, in sort order. */ + values: Schema.Array(Schema.String), +}) +export type PgEnumEntity = typeof PgEnumEntity.Type + +/** A `UNIQUE` constraint, as `ALTER TABLE ... ADD CONSTRAINT ... UNIQUE` creates it (not a unique index). */ +export const PgUniqueEntity = Schema.Struct({ + kind: Schema.Literal("unique"), + table: Schema.String, + name: Schema.String, + columns: Schema.Array(Schema.String), + /** `NULLS NOT DISTINCT` (Postgres 15+): two rows with nulls in the key conflict. */ + nullsNotDistinct: Schema.Boolean, +}) +export type PgUniqueEntity = typeof PgUniqueEntity.Type + +export const PgSchemaEntity = Schema.Union([ + PgEnumEntity, + PgTableEntity, + PgColumnEntity, + PgIndexEntity, + PgUniqueEntity, + PgForeignKeyEntity, +]) export type PgSchemaEntity = typeof PgSchemaEntity.Type +const PLAIN_TYPE_NAME = /^[a-z_][a-z0-9_$]*$/ + +/** + * How `format_type` writes an enum's name in a column type: bare when it is a + * plain lowercase identifier, double-quoted otherwise, and qualified by its + * schema when it has one. + */ +export const pgTypeName = (schema: string | null, name: string): string => { + const part = (id: string) => (PLAIN_TYPE_NAME.test(id) ? id : `"${id.replace(/"/g, '""')}"`) + return schema === null ? part(name) : `${part(schema)}.${part(name)}` +} + /** Postgres truncates longer identifiers silently, which would make every later comparison miss. */ export const PG_MAX_IDENTIFIER = 63 diff --git a/src/schema/pg-enums-uniques.test.ts b/src/schema/pg-enums-uniques.test.ts new file mode 100644 index 0000000..bc95b97 --- /dev/null +++ b/src/schema/pg-enums-uniques.test.ts @@ -0,0 +1,291 @@ +import { Effect } from "effect" +import { describe, expect, it } from "vitest" +import * as PG from "../postgres" +import { parsePgArrayLiteral } from "../pg/types" +import * as S from "../schema" + +const entitiesOf = (...objects: ReadonlyArray) => Effect.runSync(S.pgEntitiesOf(objects)) +const failureOf = (...objects: ReadonlyArray) => Effect.runSync(Effect.flip(S.pgEntitiesOf(objects))).message + +const ChannelType = PG.enumType("channel_type", ["public", "private", "direct"]) + +const channels = (type: S.SchemaObject & PG.PgEnum = ChannelType) => + PG.table("channels", { + columns: { + id: PG.text, + org_id: PG.text, + slug: PG.column(PG.text, { unique: true }), + type: PG.column(type, { default: "public" }), + allowed: PG.nullable(PG.array(type)), + }, + primaryKey: ["id"], + uniqueConstraints: [PG.unique("channels_org_slug_unique", ["org_id", "slug"], { nullsNotDistinct: true })], + indexes: [PG.index("channels_public_idx", ["org_id"], { where: ($) => $.type.eq("public") })], + }) + +describe("PG.enumType", () => { + it("is a column type and a schema object", () => { + expect(S.isSchemaObject(ChannelType)).toBe(true) + expect(S.dialectOfObject(ChannelType)).toBe("postgres") + expect(ChannelType.values).toEqual(["public", "private", "direct"]) + expect(ChannelType.sql).toBe("channel_type") + expect(PG.enumType("Kind", ["a"]).sql).toBe('"Kind"') + expect(PG.enumType("role", ["a"], { schema: "auth" }).sql).toBe("auth.role") + }) + + it("renders the type before the tables that use it, whether or not it is exported", () => { + const statements = S.renderPgSchema(entitiesOf(channels())) + expect(statements).toEqual([ + `DO $effect_orm$ BEGIN CREATE TYPE "channel_type" AS ENUM ('public', 'private', 'direct'); EXCEPTION WHEN duplicate_object THEN NULL; END $effect_orm$`, + [ + 'CREATE TABLE IF NOT EXISTS "channels" (', + '\t"id" text NOT NULL,', + '\t"org_id" text NOT NULL,', + '\t"slug" text NOT NULL,', + "\t\"type\" channel_type NOT NULL DEFAULT 'public',", + '\t"allowed" channel_type[],', + '\tCONSTRAINT "channels_pkey" PRIMARY KEY ("id")', + ")", + ].join("\n"), + 'ALTER TABLE "channels" ADD CONSTRAINT "channels_org_slug_unique" UNIQUE NULLS NOT DISTINCT ("org_id", "slug")', + 'ALTER TABLE "channels" ADD CONSTRAINT "channels_slug_unique" UNIQUE ("slug")', + `CREATE INDEX IF NOT EXISTS "channels_public_idx" ON "channels" USING btree ("org_id") WHERE "type" = 'public'`, + ]) + // Exported as well, it is still one type. + expect(entitiesOf(ChannelType, channels()).filter((e) => e.kind === "enum")).toHaveLength(1) + }) + + it("quotes labels and qualifies a type of another schema", () => { + const Role = PG.enumType("role", ["it's", "admin"], { schema: "auth" }) + const users = PG.table("users", { columns: { id: PG.text, role: Role } }) + const statements = S.renderPgSchema(entitiesOf(users)) + expect(statements[0]).toContain(`CREATE TYPE "auth"."role" AS ENUM ('it''s', 'admin');`) + expect(statements[1]).toContain('"role" auth.role NOT NULL') + }) + + it("rejects a definition Postgres would refuse", () => { + expect(PG.enumType("x", ["a", "a"]).problems.map((p) => p.message)).toEqual(['"a" is listed twice']) + expect(PG.enumType("x", ["a".repeat(64)]).problems[0]?.message).toMatch(/longer than 63 bytes/) + expect(PG.enumType("bad name", ["a"]).problems[0]?.message).toMatch(/not a plain identifier/) + expect(failureOf(PG.enumType("x", ["a", "a"]))).toMatch(/listed twice/) + expect(failureOf(PG.enumType("channel_type", ["public"]), channels())).toMatch(/declared twice, with different values/) + expect(failureOf(PG.enumType("channels", ["a"]), channels())).toMatch(/names both a table and an enum/) + }) + + it("decodes an enum array from its text form, which drivers send for it", () => { + expect(parsePgArrayLiteral("{public,private}")).toEqual(["public", "private"]) + expect(parsePgArrayLiteral(`{"a b","it's","x\\"y",NULL}`)).toEqual(["a b", "it's", 'x"y', null]) + expect(parsePgArrayLiteral("{}")).toEqual([]) + expect(parsePgArrayLiteral("{{a}}")).toBeUndefined() + expect(parsePgArrayLiteral("public")).toBeUndefined() + }) +}) + +describe("PG.unique", () => { + it("names a column's constraint as drizzle-orm does, or as given", () => { + const t = PG.table("users", { + columns: { + email: PG.column(PG.text, { unique: true }), + handle: PG.column(PG.text, { unique: "users_handle_key" }), + phone: PG.column(PG.nullable(PG.text), { unique: { nullsNotDistinct: true } }), + orgId: PG.column(PG.text, { name: "org_id" }), + }, + uniqueConstraints: [PG.unique("users_org_email_unique", ["orgId", "email"])], + }) + expect(t.ddl.uniques).toEqual([ + { kind: "unique", table: "users", name: "users_email_unique", columns: ["email"], nullsNotDistinct: false }, + { kind: "unique", table: "users", name: "users_handle_key", columns: ["handle"], nullsNotDistinct: false }, + { kind: "unique", table: "users", name: "users_phone_unique", columns: ["phone"], nullsNotDistinct: true }, + { kind: "unique", table: "users", name: "users_org_email_unique", columns: ["org_id", "email"], nullsNotDistinct: false }, + ]) + // `unique` does not make a column optional on insert. + expect(t.defaults).toBeUndefined() + }) + + it("checks names and columns", () => { + const t = PG.table("t", { columns: { a: PG.text }, uniqueConstraints: [PG.unique("t_b_unique", ["b"]) as PG.UniqueSpec] }) + // @ts-expect-error not a column + PG.table("t", { columns: { a: PG.text }, uniqueConstraints: [PG.unique("t_b_unique", ["b"])] }) + expect(t.problems.map((p) => p.message)).toEqual(["b is not a column"]) + const clash = PG.table("t", { columns: { a: PG.column(PG.text, { unique: "t_a_unique" }) }, indexes: [PG.uniqueIndex("t_a_unique", ["a"])] }) + expect(failureOf(clash)).toMatch(/one namespace/) + }) +}) + +describe("diffPgSchemas: enums", () => { + const before = entitiesOf(channels()) + const withValues = (values: readonly [string, ...Array]) => entitiesOf(channels(PG.enumType("channel_type", values))) + + it("creates a new type before the table that uses it", () => { + const { ops } = S.diffPgSchemas([], before) + expect(ops.map((op) => op.op)).toEqual(["create_enum", "create_table", "add_unique", "add_unique", "create_index"]) + }) + + it("adds labels in place, each before the label it precedes", () => { + const { ops, unsupported, missingHints } = S.diffPgSchemas(before, withValues(["public", "announcement", "private", "direct", "thread", "bot"])) + expect(unsupported).toEqual([]) + expect(missingHints).toEqual([]) + expect(ops.flatMap((op) => S.renderPgOp(op))).toEqual([ + `ALTER TYPE "channel_type" ADD VALUE IF NOT EXISTS 'announcement' BEFORE 'private'`, + `ALTER TYPE "channel_type" ADD VALUE IF NOT EXISTS 'thread'`, + `ALTER TYPE "channel_type" ADD VALUE IF NOT EXISTS 'bot'`, + ]) + expect(ops.map(S.labelOfPg)).toEqual(["metadata", "metadata", "metadata"]) + }) + + it("renames a label swapped in place, once confirmed", () => { + const next = withValues(["public", "private", "dm"]) + const first = S.diffPgSchemas(before, next) + expect(first.ops).toEqual([]) + expect(first.missingHints).toEqual([{ type: "confirm_rename", kind: "enum_value", entity: "channel_type.direct", to: "dm" }]) + const { ops } = S.diffPgSchemas(before, next, first.missingHints) + expect(ops.flatMap((op) => S.renderPgOp(op))).toEqual([`ALTER TYPE "channel_type" RENAME VALUE 'direct' TO 'dm'`]) + // A hint for another target does not confirm this one. + expect(S.diffPgSchemas(before, next, [{ ...first.missingHints[0]!, to: "direct_message" } as S.Hint]).ops).toEqual([]) + }) + + it("reports removing or reordering labels as unsupported", () => { + expect(S.diffPgSchemas(before, withValues(["public", "private"])).unsupported).toEqual([ + { entity: "channel_type", message: expect.stringMatching(/^removing "direct" needs the type rebuilt/) }, + ]) + expect(S.diffPgSchemas(before, withValues(["private", "public", "direct"])).unsupported).toEqual([ + { entity: "channel_type", message: expect.stringMatching(/^reordering labels needs the type rebuilt/) }, + ]) + // A removal next to an addition is not read as a rename. + expect(S.diffPgSchemas(before, withValues(["public", "private", "dm", "group"])).unsupported).toHaveLength(1) + }) + + it("reports a new label used in the migration that adds it", () => { + const Type = PG.enumType("channel_type", ["public", "private", "direct", "thread"]) + const next = entitiesOf( + PG.table("channels", { + columns: { + id: PG.text, + org_id: PG.text, + slug: PG.column(PG.text, { unique: true }), + type: PG.column(Type, { default: "thread" }), + allowed: PG.nullable(PG.array(Type)), + }, + primaryKey: ["id"], + uniqueConstraints: [PG.unique("channels_org_slug_unique", ["org_id", "slug"], { nullsNotDistinct: true })], + indexes: [PG.index("channels_public_idx", ["org_id"], { where: ($) => $.type.eq("thread") })], + }), + ) + expect(S.diffPgSchemas(before, next).unsupported).toEqual([ + { entity: "channels.type", message: expect.stringMatching(/^uses "thread", which this migration adds/) }, + { entity: "channels.channels_public_idx", message: expect.stringMatching(/^uses "thread", which this migration adds/) }, + ]) + }) + + it("asks before dropping a type, and drops it after the columns that used it", () => { + const next = entitiesOf( + PG.table("channels", { + columns: { id: PG.text, org_id: PG.text, slug: PG.column(PG.text, { unique: true }), type: PG.text, allowed: PG.nullable(PG.array(PG.text)) }, + primaryKey: ["id"], + uniqueConstraints: [PG.unique("channels_org_slug_unique", ["org_id", "slug"], { nullsNotDistinct: true })], + }), + ) + const first = S.diffPgSchemas(before, next) + expect(first.missingHints).toEqual([{ type: "confirm_data_loss", kind: "enum", entity: "channel_type" }]) + const { ops } = S.diffPgSchemas(before, next, first.missingHints) + expect(ops.map((op) => op.op)).toEqual(["drop_index", "alter_column", "alter_column", "drop_enum"]) + expect(S.renderPgOp(ops.at(-1)!)).toEqual(['DROP TYPE IF EXISTS "channel_type"']) + expect(S.labelOfPg(ops.at(-1)!)).toBe("destructive") + }) +}) + +describe("diffPgSchemas: unique constraints", () => { + const table = (uniques: ReadonlyArray>, indexes: ReadonlyArray> = []) => + PG.table("projects", { columns: { id: PG.text, org_id: PG.text, slug: PG.text }, primaryKey: ["id"], uniqueConstraints: uniques, indexes }) + + it("drops and re-adds a changed constraint as a constraint", () => { + const v1 = entitiesOf(table([PG.unique("projects_slug_unique", ["slug"])])) + const v2 = entitiesOf(table([PG.unique("projects_slug_unique", ["org_id", "slug"])])) + expect(S.diffPgSchemas(v1, v2).ops.flatMap((op) => S.renderPgOp(op))).toEqual([ + 'ALTER TABLE "projects" DROP CONSTRAINT IF EXISTS "projects_slug_unique"', + 'ALTER TABLE "projects" ADD CONSTRAINT "projects_slug_unique" UNIQUE ("org_id", "slug")', + ]) + expect(S.diffPgSchemas(v2, v2).ops).toEqual([]) + }) + + it("turns a unique index into a constraint of the same name: the index goes first", () => { + const v1 = entitiesOf(table([], [PG.uniqueIndex("projects_slug_unique", ["slug"])])) + const v2 = entitiesOf(table([PG.unique("projects_slug_unique", ["slug"])])) + expect(S.diffPgSchemas(v1, v2).ops.map((op) => op.op)).toEqual(["drop_index", "add_unique"]) + expect(S.diffPgSchemas(v2, v1).ops.map((op) => op.op)).toEqual(["drop_unique", "create_index"]) + }) + + it("adds a constraint before a foreign key that references it, and drops it after", () => { + const Projects = table([PG.unique("projects_org_slug_unique", ["org_id", "slug"])]) + const Links = PG.table("links", { + columns: { id: PG.text, org_id: PG.text, slug: PG.text }, + foreignKeys: [PG.foreignKey({ columns: ["org_id", "slug"], references: Projects, foreignColumns: ["org_id", "slug"] })], + }) + const both = entitiesOf(Projects, Links) + expect(S.renderPgSchema(both).map((sql) => sql.split(/[\n(]/)[0]!.split(" ").slice(0, 6).join(" ").trim())).toEqual([ + 'CREATE TABLE IF NOT EXISTS "links"', + 'CREATE TABLE IF NOT EXISTS "projects"', + 'ALTER TABLE "projects" ADD CONSTRAINT "projects_org_slug_unique"', + 'ALTER TABLE "links" ADD CONSTRAINT "links_org_id_slug_projects_org_id_slug_fk"', + ]) + const hints = [{ type: "confirm_data_loss", kind: "table", entity: "links" } as const] + expect(S.diffPgSchemas(both, entitiesOf(table([])), hints).ops.map((op) => op.op)).toEqual(["drop_foreign_key", "drop_unique", "drop_table"]) + // A dropped table takes its own constraints with it. + const dropAll = [...hints, { type: "confirm_data_loss", kind: "table", entity: "projects" } as const] + expect(S.diffPgSchemas(both, [], dropAll).ops.map((op) => op.op)).toEqual(["drop_foreign_key", "drop_table", "drop_table"]) + }) +}) + +describe("fromDrizzleSnapshot: enums and unique constraints", () => { + const snapshot = { + version: "8", + dialect: "postgres", + id: "x", + prevIds: [], + renames: [], + ddl: [ + { name: "channel_type", schema: "public", values: ["public", "private", "direct"], entityType: "enums" }, + { name: "role", schema: "auth", values: ["admin"], entityType: "enums" }, + { isRlsEnabled: false, name: "channels", entityType: "tables", schema: "public" }, + { type: "text", typeSchema: null, notNull: true, dimensions: 0, default: null, generated: null, identity: null, name: "id", entityType: "columns", schema: "public", table: "channels" }, + { type: "text", typeSchema: null, notNull: true, dimensions: 0, default: null, generated: null, identity: null, name: "org_id", entityType: "columns", schema: "public", table: "channels" }, + { type: "text", typeSchema: null, notNull: true, dimensions: 0, default: null, generated: null, identity: null, name: "slug", entityType: "columns", schema: "public", table: "channels" }, + { type: "channel_type", typeSchema: "public", notNull: true, dimensions: 0, default: "'public'", generated: null, identity: null, name: "type", entityType: "columns", schema: "public", table: "channels" }, + { type: "channel_type", typeSchema: "public", notNull: false, dimensions: 1, default: null, generated: null, identity: null, name: "allowed", entityType: "columns", schema: "public", table: "channels" }, + { columns: ["id"], nameExplicit: false, name: "channels_pkey", entityType: "pks", schema: "public", table: "channels" }, + { nameExplicit: false, columns: ["slug"], nullsNotDistinct: false, name: "channels_slug_unique", entityType: "uniques", schema: "public", table: "channels" }, + { nameExplicit: true, columns: ["org_id", "slug"], nullsNotDistinct: true, name: "channels_org_slug_unique", entityType: "uniques", schema: "public", table: "channels" }, + { + nameExplicit: true, + columns: [{ value: "org_id", isExpression: false, asc: true, nullsFirst: false, opclass: null }], + isUnique: false, + where: `"channels"."type" = 'public'`, + with: "", + method: "btree", + concurrently: false, + name: "channels_public_idx", + entityType: "indexes", + schema: "public", + table: "channels", + }, + ], + } + + it("converts enums, enum columns and unique constraints", () => { + const { entities, unsupported } = S.fromDrizzleSnapshot(snapshot) + expect(unsupported).toEqual([]) + expect(entities.filter((e) => e.kind === "enum")).toEqual([ + { kind: "enum", schema: "auth", name: "role", values: ["admin"] }, + { kind: "enum", schema: null, name: "channel_type", values: ["public", "private", "direct"] }, + ]) + expect(entities).toContainEqual({ kind: "column", table: "channels", name: "type", position: 3, type: "channel_type", notNull: true, default: "'public'", identity: null }) + expect(entities).toContainEqual({ kind: "column", table: "channels", name: "allowed", position: 4, type: "channel_type[]", notNull: false, default: null, identity: null }) + expect(entities).toContainEqual({ kind: "unique", table: "channels", name: "channels_org_slug_unique", columns: ["org_id", "slug"], nullsNotDistinct: true }) + }) + + it("matches the same schema declared with PG.enumType and PG.unique, but for how a predicate is written", () => { + const imported = S.fromDrizzleSnapshot({ ...snapshot, ddl: snapshot.ddl.filter((e) => e.name !== "role") }).entities + const { ops } = S.diffPgSchemas(imported, entitiesOf(channels())) + expect(ops.map((op) => op.op)).toEqual(["drop_index", "create_index"]) + }) +}) diff --git a/src/schema/pg-ops.ts b/src/schema/pg-ops.ts index 75071c0..f22bd7a 100644 --- a/src/schema/pg-ops.ts +++ b/src/schema/pg-ops.ts @@ -6,11 +6,27 @@ // replayed against a hand-fixed database does not trip over its own objects. import { Schema } from "effect" -import { PgColumnEntity, PgForeignKeyEntity, PgIndexEntity, PgTableEntity, type PgSchemaEntity } from "./pg-entities" +import { + PgColumnEntity, + PgEnumEntity, + PgForeignKeyEntity, + PgIndexEntity, + PgTableEntity, + PgUniqueEntity, + type PgSchemaEntity, +} from "./pg-entities" const PrimaryKey = Schema.NullOr(Schema.Struct({ name: Schema.String, columns: Schema.Array(Schema.String) })) +/** An enum type, by name: `schema` is `null` for the schema migrations run in. */ +const EnumRef = { schema: Schema.NullOr(Schema.String), name: Schema.String } + export const PgMigrationOp = Schema.Union([ + Schema.Struct({ op: Schema.Literal("create_enum"), enum: PgEnumEntity }), + Schema.Struct({ op: Schema.Literal("drop_enum"), ...EnumRef }), + /** `ALTER TYPE ... ADD VALUE`, placed `BEFORE` an existing label, or appended when `before` is `null`. */ + Schema.Struct({ op: Schema.Literal("add_enum_value"), ...EnumRef, value: Schema.String, before: Schema.NullOr(Schema.String) }), + Schema.Struct({ op: Schema.Literal("rename_enum_value"), ...EnumRef, from: Schema.String, to: Schema.String }), Schema.Struct({ op: Schema.Literal("create_table"), table: PgTableEntity, columns: Schema.Array(PgColumnEntity) }), Schema.Struct({ op: Schema.Literal("drop_table"), name: Schema.String }), Schema.Struct({ op: Schema.Literal("add_column"), column: PgColumnEntity }), @@ -20,6 +36,8 @@ export const PgMigrationOp = Schema.Union([ Schema.Struct({ op: Schema.Literal("set_primary_key"), table: Schema.String, from: PrimaryKey, to: PrimaryKey }), Schema.Struct({ op: Schema.Literal("create_index"), index: PgIndexEntity }), Schema.Struct({ op: Schema.Literal("drop_index"), table: Schema.String, name: Schema.String }), + Schema.Struct({ op: Schema.Literal("add_unique"), unique: PgUniqueEntity }), + Schema.Struct({ op: Schema.Literal("drop_unique"), table: Schema.String, name: Schema.String }), Schema.Struct({ op: Schema.Literal("add_foreign_key"), foreignKey: PgForeignKeyEntity }), Schema.Struct({ op: Schema.Literal("drop_foreign_key"), table: Schema.String, name: Schema.String }), ]) @@ -38,6 +56,25 @@ export const pgIdent = (name: string): string => `"${name.replace(/"/g, '""')}"` const list = (names: ReadonlyArray): string => names.map(pgIdent).join(", ") +/** A string literal, quotes doubled. */ +const pgString = (value: string): string => `'${value.replace(/'/g, "''")}'` + +/** An enum's name as DDL writes it: always quoted, qualified when it has a schema. */ +const enumIdent = (ref: { readonly schema: string | null; readonly name: string }): string => + ref.schema === null ? pgIdent(ref.name) : `${pgIdent(ref.schema)}.${pgIdent(ref.name)}` + +/** + * `CREATE TYPE ... AS ENUM`, skipped when the type exists: Postgres has no + * `IF NOT EXISTS` for it, and a type created by hand (behind `PG.custom`) + * before it was declared with `enumType` must not fail the migration. A type + * that exists with other labels is left as it is, and `verify` reports it. + */ +export const renderPgCreateEnum = (entity: PgEnumEntity): string => + `DO $effect_orm$ BEGIN CREATE TYPE ${enumIdent(entity)} AS ENUM (${entity.values.map(pgString).join(", ")}); EXCEPTION WHEN duplicate_object THEN NULL; END $effect_orm$` + +export const renderPgAddUnique = (unique: PgUniqueEntity): string => + `ALTER TABLE ${pgIdent(unique.table)} ADD CONSTRAINT ${pgIdent(unique.name)} UNIQUE${unique.nullsNotDistinct ? " NULLS NOT DISTINCT" : ""} (${list(unique.columns)})` + export const renderPgColumnDefinition = (column: PgColumnEntity): string => { const parts = [pgIdent(column.name), column.type] if (column.notNull) parts.push("NOT NULL") @@ -68,15 +105,19 @@ export const renderPgAddForeignKey = (fk: PgForeignKeyEntity): string => { const alter = (table: string, action: string): string => `ALTER TABLE ${pgIdent(table)} ${action}` /** - * Every CREATE statement for a schema: tables, then indexes, then foreign - * keys, so every table a key references exists before the key. + * Every CREATE statement for a schema: enums, tables, unique constraints, + * indexes, then foreign keys, so every type a column uses exists before its + * table, and every table and unique key a foreign key references exists + * before the key. */ export const renderPgSchema = (entities: ReadonlyArray): ReadonlyArray => { const pick = (kind: K) => entities.filter((e): e is Extract => e.kind === kind) const columns = pick("column") return [ + ...pick("enum").map(renderPgCreateEnum), ...pick("table").map((table) => renderPgCreateTable(table, columns.filter((c) => c.table === table.name))), + ...pick("unique").map(renderPgAddUnique), ...pick("index").map(renderPgCreateIndex), ...pick("foreign_key").map(renderPgAddForeignKey), ] @@ -89,6 +130,7 @@ export const labelOfPg = (op: PgMigrationOp): PgOpLabel => { switch (op.op) { case "drop_table": case "drop_column": + case "drop_enum": return "destructive" case "alter_column": return op.from.type === op.to.type ? "metadata" : "rewrite" @@ -100,6 +142,14 @@ export const labelOfPg = (op: PgMigrationOp): PgOpLabel => { /** The statements one op runs, in order. */ export const renderPgOp = (op: PgMigrationOp): ReadonlyArray => { switch (op.op) { + case "create_enum": + return [renderPgCreateEnum(op.enum)] + case "drop_enum": + return [`DROP TYPE IF EXISTS ${enumIdent(op)}`] + case "add_enum_value": + return [`ALTER TYPE ${enumIdent(op)} ADD VALUE IF NOT EXISTS ${pgString(op.value)}${op.before === null ? "" : ` BEFORE ${pgString(op.before)}`}`] + case "rename_enum_value": + return [`ALTER TYPE ${enumIdent(op)} RENAME VALUE ${pgString(op.from)} TO ${pgString(op.to)}`] case "create_table": return [renderPgCreateTable(op.table, op.columns)] case "drop_table": @@ -142,6 +192,10 @@ export const renderPgOp = (op: PgMigrationOp): ReadonlyArray => { return [renderPgCreateIndex(op.index)] case "drop_index": return [`DROP INDEX IF EXISTS ${pgIdent(op.name)}`] + case "add_unique": + return [renderPgAddUnique(op.unique)] + case "drop_unique": + return [alter(op.table, `DROP CONSTRAINT IF EXISTS ${pgIdent(op.name)}`)] case "add_foreign_key": return [renderPgAddForeignKey(op.foreignKey)] case "drop_foreign_key": diff --git a/src/schema/pg-schema.test.ts b/src/schema/pg-schema.test.ts index 6e00bba..d1eb7e3 100644 --- a/src/schema/pg-schema.test.ts +++ b/src/schema/pg-schema.test.ts @@ -292,8 +292,8 @@ describe("fromDrizzleSnapshot", () => { it("reports what it cannot model instead of dropping it", () => { const { unsupported } = S.fromDrizzleSnapshot({ ...drizzle, - ddl: [...drizzle.ddl, { name: "mood", values: ["a"], entityType: "enums", schema: "public" }], + ddl: [...drizzle.ddl, { name: "ticket_seq", entityType: "sequences", schema: "public" }, { name: "c", value: "x > 0", entityType: "checks", schema: "public", table: "dashboard_shares" }], }) - expect(unsupported).toEqual(["enums mood"]) + expect(unsupported).toEqual(["sequences ticket_seq", "checks dashboard_shares.c"]) }) }) diff --git a/src/schema/snapshot.ts b/src/schema/snapshot.ts index 136201b..181d249 100644 --- a/src/schema/snapshot.ts +++ b/src/schema/snapshot.ts @@ -16,22 +16,22 @@ import { type SchemaEntity, type Snapshot, } from "./entities" -import type { PgSchemaTable } from "./pg-define" -import type { PgSchemaEntity } from "./pg-entities" +import { enumProblems, type PgEnum, type PgSchemaTable } from "./pg-define" +import type { PgEnumEntity, PgSchemaEntity } from "./pg-entities" /** Anything a schema module may export that `generate` collects. */ -export type SchemaObject = SchemaTable | MaterializedView | PgSchemaTable +export type SchemaObject = SchemaTable | MaterializedView | PgSchemaTable | PgEnum export const isSchemaObject = (value: unknown): value is SchemaObject => typeof value === "object" && value !== null && "ddl" in value && "_tag" in value && - (value._tag === "Table" || value._tag === "MaterializedView") + (value._tag === "Table" || value._tag === "MaterializedView" || value._tag === "PgEnum") /** The dialect a definition was written for. */ export const dialectOfObject = (object: SchemaObject): SchemaDialect => - object._tag === "Table" && "dialect" in object.ddl ? object.ddl.dialect : "clickhouse" + object._tag === "PgEnum" ? "postgres" : object._tag === "Table" && "dialect" in object.ddl ? object.ddl.dialect : "clickhouse" /** Fail with every problem, or succeed with the sorted entities. */ const validated = ( @@ -53,7 +53,7 @@ export const entitiesOf = (objects: ReadonlyArray): Effect.Effect< if (object._tag === "Table") { const ddl = object.ddl as SchemaTable["ddl"] entities.push(ddl.table, ...ddl.columns, ...ddl.indexes) - } else entities.push(object.ddl) + } else entities.push((object as MaterializedView).ddl) } const seen = new Set() const names = new Map() @@ -88,23 +88,47 @@ export const entitiesOf = (objects: ReadonlyArray): Effect.Effect< } /** - * The entities of a set of Postgres tables, validated as one schema: names are - * unique, index and primary-key names do not collide (Postgres keeps them in - * one namespace per schema), and every foreign key references a table and - * columns of this schema. + * The entities of a set of Postgres tables and enums, validated as one schema: + * names are unique, index, unique-constraint and primary-key names do not + * collide (Postgres keeps them in one namespace per schema), an enum is + * declared one way, and every foreign key references a table and columns of + * this schema. Enums a table's columns use come with it, exported or not. */ export const pgEntitiesOf = (objects: ReadonlyArray): Effect.Effect, SchemaDefinitionError> => { const problems: ProblemSink = [] const tables: Array> = [] + const enumSources: Array = [] for (const object of objects) { - problems.push(...object.problems) if (dialectOfObject(object) !== "postgres") { + problems.push(...object.problems) problems.push({ object: object.name, message: "is a ClickHouse definition; use entitiesOf" }) continue } - tables.push(object as PgSchemaTable) + if (object._tag === "PgEnum") { + enumSources.push(object.ddl.enum) + continue + } + problems.push(...object.problems) + const table = object as PgSchemaTable + tables.push(table) + enumSources.push(...table.ddl.enums) } const entities: Array = [] + const enums = new Map() + for (const entity of enumSources) { + const key = entityKey(entity) + const other = enums.get(key) + if (other === undefined) { + enums.set(key, entity) + problems.push(...enumProblems(entity)) + } else if (other.values.join("\u0000") !== entity.values.join("\u0000")) { + problems.push({ + object: entity.schema === null ? entity.name : `${entity.schema}.${entity.name}`, + message: `is declared twice, with different values: ${JSON.stringify(other.values)} and ${JSON.stringify(entity.values)}`, + }) + } + } + entities.push(...enums.values()) const relations = new Map() const claim = (name: string, what: string) => { const other = relations.get(name) @@ -113,11 +137,15 @@ export const pgEntitiesOf = (objects: ReadonlyArray): Effect.Effec } relations.set(name, what) } + // A table is a type too (its row type), so an enum may not share its name. + const types = new Set([...enums.values()].filter((e) => e.schema === null).map((e) => e.name)) for (const { ddl } of tables) { claim(ddl.table.name, `table ${ddl.table.name}`) + if (types.has(ddl.table.name)) problems.push({ object: ddl.table.name, message: "names both a table and an enum; Postgres keeps them in one namespace" }) if (ddl.table.primaryKey !== null) claim(ddl.table.primaryKey.name, `the primary key of ${ddl.table.name}`) for (const index of ddl.indexes) claim(index.name, `an index on ${ddl.table.name}`) - entities.push(ddl.table, ...ddl.columns, ...ddl.indexes, ...ddl.foreignKeys) + for (const unique of ddl.uniques) claim(unique.name, `a unique constraint on ${ddl.table.name}`) + entities.push(ddl.table, ...ddl.columns, ...ddl.indexes, ...ddl.uniques, ...ddl.foreignKeys) } const columnsOf = new Map(tables.map(({ ddl }) => [ddl.table.name, new Set(ddl.columns.map((c) => c.name))])) const seenConstraints = new Set() diff --git a/tests/__snapshots__/core-sql.test.ts.snap b/tests/__snapshots__/core-sql.test.ts.snap index bd6e9a2..fba9e28 100644 --- a/tests/__snapshots__/core-sql.test.ts.snap +++ b/tests/__snapshots__/core-sql.test.ts.snap @@ -546,6 +546,31 @@ OFFSET 1", } `; +exports[`core SQL (clickhouse) > union-cross-tenant 1`] = ` +{ + "parameters": [], + "sql": "WITH orders AS ( + +), +customers AS ( + +) +SELECT + count() AS orders + FROM orders +UNION ALL +WITH orders AS ( + +), +customers AS ( + +) +SELECT + count() AS orders + FROM orders", +} +`; + exports[`core SQL (clickhouse) > union-format 1`] = ` { "parameters": [], @@ -1213,3 +1238,32 @@ LIMIT 2 OFFSET 1", } `; + +exports[`core SQL (postgres) > union-cross-tenant 1`] = ` +{ + "parameters": [], + "sql": "( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + count(*) AS "orders" + FROM "orders" +) +UNION ALL +( +WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + count(*) AS "orders" + FROM "orders" +)", +} +`; diff --git a/tests/core-cases.ts b/tests/core-cases.ts index 3388781..595f296 100644 --- a/tests/core-cases.ts +++ b/tests/core-cases.ts @@ -55,7 +55,7 @@ export interface CoreCase { readonly expectedBy?: Partial> /** ClickHouse output format, for the `format` cases. */ readonly format?: "JSON" | "JSONEachRow" - readonly metadata?: { readonly route: string; readonly tenantScope: CH.TenantScope } + readonly metadata?: { readonly route?: string; readonly tenantScope: CH.TenantScope } /** Dialects that must refuse to compile this case, and the error they give. */ readonly rejects?: Partial> } @@ -581,6 +581,19 @@ export const coreCases: readonly CoreCase[] = [ ), expected: [{ orders: 5 }], }, + { + id: "union-cross-tenant", + covers: u("crossTenant"), + metadata: { tenantScope: "cross-tenant" }, + build: (ctx) => + ctx.compileUnion( + CH.unionAll( + ctx.from(ctx.orders).select(() => ({ orders: ctx.fn.count() })), + ctx.from(ctx.orders).select(() => ({ orders: ctx.fn.count() })), + ).crossTenant(), + ), + expected: [{ orders: 5 }, { orders: 5 }], + }, { id: "format", covers: q("format"), diff --git a/tests/dialect-cases.postgres.ts b/tests/dialect-cases.postgres.ts index 7a7e850..b502fac 100644 --- a/tests/dialect-cases.postgres.ts +++ b/tests/dialect-cases.postgres.ts @@ -5,6 +5,7 @@ import { DateTime, Effect, Schema } from "effect" import { expect } from "vitest" import * as CH from "@maple-dev/effect-orm/clickhouse" import * as PG from "@maple-dev/effect-orm/postgres" +import * as S from "@maple-dev/effect-orm/schema" import { postgresContext as ctx } from "./core-cases" export interface PostgresCase { @@ -72,7 +73,29 @@ const typedRow = `SELECT NULL::int4 AS "Missing"` const typedRows = () => CH.from(typed).withCTE("typed", typedRow) +// An enum, created before the fixtures run. Its array arrives as text (`{ok,"it's"}`), +// since the driver does not know the array type, and decodes through parsePgArrayLiteral. +const Mood = PG.enumType("fixture_mood", ["ok", "meh", "it's"]) +const moods = PG.table("moods", { external: true, columns: { Mood, Moods: PG.array(Mood), Maybe: PG.nullable(Mood) } }) +const moodRow = `SELECT 'meh'::fixture_mood AS "Mood", ARRAY['ok', 'it''s']::fixture_mood[] AS "Moods", NULL::fixture_mood AS "Maybe"` + +/** Statements the fixtures need run once, before them. */ +export const postgresSetup: readonly string[] = [S.renderPgCreateEnum(Mood.ddl.enum)] + export const postgresCases: readonly PostgresCase[] = [ + { + id: "enums", + covers: pg("enumType", "parsePgArrayLiteral"), + build: () => + PG.compileUnsafe( + CH.from(moods) + .withCTE("moods", moodRow) + .select("Mood", "Moods", "Maybe") + .where(($) => [$.Mood.in_("meh", "it's"), $.Moods.eq(["ok", "it's"])]), + {}, + ), + expected: [{ Mood: "meh", Moods: ["ok", "it's"], Maybe: null }], + }, { id: "aggregates", covers: pg("count", "countDistinct", "countIf", "sum", "sumIf", "avg", "min", "max", "percentileCont", "compileUnsafe"), diff --git a/tests/dialect-cases.ts b/tests/dialect-cases.ts index 82e6c48..16da70f 100644 --- a/tests/dialect-cases.ts +++ b/tests/dialect-cases.ts @@ -560,7 +560,7 @@ export const dialectCases: readonly DialectCase[] = [ ), { id: "union-pagination", - covers: ["union:orderBy", "union:limit", "union:offset", "union:format"], + covers: ["union:orderBy", "union:limit", "union:offset", "union:format", "union:crossTenant"], build: () => CH.compileUnionUnsafe( CH.unionAll( @@ -570,7 +570,8 @@ export const dialectCases: readonly DialectCase[] = [ .orderBy(["n", "desc"]) .limit(1) .offset(1) - .format("JSONEachRow"), + .format("JSONEachRow") + .crossTenant(), {}, ), expected: [{ n: 1 }], diff --git a/tests/dialect-coverage.test.ts b/tests/dialect-coverage.test.ts index e6f2828..4bb5901 100644 --- a/tests/dialect-coverage.test.ts +++ b/tests/dialect-coverage.test.ts @@ -172,7 +172,7 @@ export const postgresInventory = Object.keys(PG) .sort() const ddl = - "DDL definition, not a query: rendered and diffed in src/schema/pg-schema.test.ts and applied to PGlite in src/migrate/pg-migrate.test.ts." + "DDL definition, not a query: rendered and diffed in src/schema/pg-schema.test.ts and pg-enums-uniques.test.ts, and applied to PGlite in src/migrate/pg-migrate.test.ts and pg-enums-uniques.test.ts." const postgresExemptions = { "pg:PgNumber": "Wire codec behind every numeric type; the types fixture decodes it from number, bigint and string.", @@ -182,6 +182,7 @@ const postgresExemptions = { "pg:column": ddl, "pg:index": ddl, "pg:uniqueIndex": ddl, + "pg:unique": ddl, "pg:foreignKey": ddl, "pg:defaultForeignKeyName": ddl, } diff --git a/tests/dialect.postgres.test.ts b/tests/dialect.postgres.test.ts index 77501f5..ee4d310 100644 --- a/tests/dialect.postgres.test.ts +++ b/tests/dialect.postgres.test.ts @@ -1,12 +1,15 @@ // Every Postgres function and type fixture, executed on Postgres 17 (PGlite). import { PGlite } from "@electric-sql/pglite" -import { afterAll, describe, expect, it } from "@effect/vitest" +import { afterAll, beforeAll, describe, expect, it } from "@effect/vitest" import { Effect } from "effect" -import { postgresCases } from "./dialect-cases.postgres" +import { postgresCases, postgresSetup } from "./dialect-cases.postgres" import { runOn } from "./postgres-support" // PGlite 0.5 takes the session time zone from the host; the fixtures assume UTC. const db = new PGlite({ postgresqlconf: "timezone = 'UTC'" }) +beforeAll(async () => { + for (const statement of postgresSetup) await db.exec(statement) +}) afterAll(() => db.close()) describe("postgres dialect fixtures", () => { diff --git a/tests/package-consumer-ts5.mts b/tests/package-consumer-ts5.mts new file mode 100644 index 0000000..4c69522 --- /dev/null +++ b/tests/package-consumer-ts5.mts @@ -0,0 +1,18 @@ +// Copied into an isolated directory by check-package.ts and typechecked on its +// own, with TypeScript 5 as well as 7. Kept alone and first: TypeScript 5 +// measures a generic type's variance once, from whichever comparison asks +// first, so a file that touched `CHInsert` elsewhere could hide a regression. +import assert from "node:assert/strict" +import * as Db from "@maple-dev/effect-orm/database" +import * as PG from "@maple-dev/effect-orm/postgres" + +const Counters = PG.table("counters", { columns: { key: PG.text, count: PG.int4 }, primaryKey: ["key"] }) + +// A concrete insert, upsert included, is a Runnable and compiles. +const insert: Db.Runnable = PG.insertInto(Counters).values({ key: "k", count: 1 }) +const upsert = PG.insertInto(Counters) + .values({ key: "k", count: PG.param.int("count") }) + .onConflictDoUpdate({ target: ["key"], set: { count: 1 } }) +const runnable: Db.Runnable = upsert +assert.ok(insert !== undefined && runnable === upsert) +assert.match(PG.compileUnsafe(upsert, { count: 1 }).sql, /ON CONFLICT \("key"\) DO UPDATE/)