Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,17 @@ here.**
every port at this version. A model that does not use it is unchanged. See
[docs/features/reporting.md](docs/features/reporting.md#calendar-aligned-windows-startof).

### Changed

- **A list or report sort puts nulls last ascending and first descending, on every engine, in
all five ports.** Nulls sort as larger than any value, which is Postgres's own default, so
**Postgres output does not change; SQLite and MySQL sorted output does** for any column with
nulls: those engines used to put a null first ascending. It applies to every `?sort=` the
generated routes and the runtime `ObjectManager` drivers emit (TypeScript, C#, Java, Kotlin,
Python), using native `NULLS LAST` / `NULLS FIRST` where the engine has it and a leading
`col IS NULL` key where it does not (MySQL). Pinned by the api-contract `report/` scenario
`sort-null-ordering`; see [api-contract.md](docs/features/api-contract.md#null-ordering).

### Fixed

- **A relative date with months or years now lands on the same day on SQLite and D1 as on
Expand Down
10 changes: 10 additions & 0 deletions docs/features/api-contract.md
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,16 @@ each handler they register instead; an `HTTPException` is rethrown to your
(`fixtures/api-contract-conformance/scenarios/`), whose three arms cover the
declared field, an explicit order beating it, and an undeclared field falling
back to `asc`.
- <a id="null-ordering"></a>**Null ordering — nulls sort as larger than any value, on every
engine.** `sort=<field>:asc` puts null rows last and `sort=<field>:desc` puts them first, in
every port (TypeScript, C#, Java, Kotlin, Python), on every engine each supports
(Postgres, SQLite and D1, MySQL), for entity lists, projections and served reports alike. That
is Postgres's own default; a bare `ORDER BY col` put nulls first ascending on SQLite and
MySQL, so the same request used to answer a different order per engine. Each port now pins
it: native `NULLS LAST` / `NULLS FIRST` where the engine has the clause, a leading
`col IS NULL` key (or the ORM's equivalent) where it does not. Guarded cross-port by the
`sort-null-ordering` scenario (`fixtures/api-contract-conformance/report/scenarios/`), an
ascending and a descending sort over a column with a null row.
- **Known limit — enums sort by stored value, not declared order.** A
string-backed `field.enum` sorts alphabetically (`high` < `low` < `medium`
< `urgent`), not in `@values` order; an int-backed (`@intValueMap`) enum
Expand Down
4 changes: 2 additions & 2 deletions docs/features/reporting.md
Original file line number Diff line number Diff line change
Expand Up @@ -460,8 +460,8 @@ rating with no ratings is not `0`; null is the honest answer there.
**Filter and sort.** The default is the value. `?filter[revenue][eq]=0` matches the empty
rows, `isNull=true` matches nothing and `isNull=false` every row. A sort on a defaulted
measure is the same on every engine, since the empty rows sort as their default. A measure
without one sorts its null rows where the engine puts them: Postgres last ascending, SQLite
and MySQL first.
without one has null rows, and they sort as larger than any value: last ascending, first
descending, the same on every engine (see [Null ordering](api-contract.md#null-ordering)).

### Boolean measures: `any` and `all`

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ export function parseFilterParams(opts: ParseFilterOpts): ParseFilterResult {
}

if (typeof opts.query.sort === "string") {
result.orderBy = parseSort(opts.query.sort, opts.table, opts.sortAllowlist);
result.orderBy = parseSort(opts.query.sort, opts.table, opts.sortAllowlist, opts.dialect);
}

if (typeof opts.query.search === "string" && opts.query.search !== "") {
Expand Down Expand Up @@ -421,7 +421,7 @@ function ensureArray(v: unknown, key: string): unknown[] {
throw new FilterParseError("filter.invalid_value", `Expected array for "${key}".`, { field: key });
}

function parseSort(spec: string, table: AnyTable, sortAllowlist: SortAllowlist): SQLWrapper[] {
function parseSort(spec: string, table: AnyTable, sortAllowlist: SortAllowlist, dialect: SqlDialect | undefined): SQLWrapper[] {
const colonIdx = spec.indexOf(":");
const field = colonIdx === -1 ? spec : spec.slice(0, colonIdx);
if (!sortAllowlist[field]) {
Expand Down Expand Up @@ -458,5 +458,21 @@ function parseSort(spec: string, table: AnyTable, sortAllowlist: SortAllowlist):
);
}
const col = table[field];
return [order === "asc" ? asc(col as any) : desc(col as any)];
return nullsAsLargest(col as any, order, dialect);
}

/**
* Order a column with nulls as LARGER than any value: ASC puts them last, DESC first, on
* every engine. That is Postgres's own default; SQLite and MySQL put nulls first on ASC, so
* a bare `ORDER BY col` answered the same list in a different order per engine. SQLite
* (3.30+, D1 included) and Postgres take `NULLS FIRST/LAST`; MySQL has no such clause, so it
* gets a leading `col IS NULL` key (0 for a value, 1 for null).
*/
function nullsAsLargest(col: SQLWrapper, order: "asc" | "desc", dialect: SqlDialect | undefined): SQLWrapper[] {
if (dialect === "mysql") {
return order === "asc"
? [asc(sql`(${col} IS NULL)`), asc(col)]
: [desc(sql`(${col} IS NULL)`), desc(col)];
}
return [order === "asc" ? sql`${col} ASC NULLS LAST` : sql`${col} DESC NULLS FIRST`];
}
2 changes: 1 addition & 1 deletion examples/showcase/.metaobjects/.gen-state/.hashes.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"generated/csharp/SubscriberBlurbPayload.g.cs": "6b1ce5b5a4018c647586d86125718c4764ae5d6eeb94e9545e8006a7ec2b2b6f",
"generated/csharp/SubscriberFilterAllowlist.g.cs": "7428d47f6ccb64bb315266ee0564fa3803736623a34ab491435c042848c5d472",
"generated/csharp/SubscriberNames.g.cs": "91f6eba78657b31fceef7021c6ca6bd799524dcb5120acbc8fcbe18786787174",
"generated/csharp/SubscriberRoutes.g.cs": "e520084727642d242609cbf82b3598d7c996224c3d0aef250bd26c7540ab4423",
"generated/csharp/SubscriberRoutes.g.cs": "acb153e2aac648a487d78d6d5321c0f250451f47b691f988e3b99ff410bcfa06",
"generated/python/Subscriber.py": "9dd4c0acea47f99d1daf79930f9de2e400f30780ebce6c63f023be528a72b762",
"generated/python/SubscriberBlurbPayload.py": "a5ef8a08e63efbe0f73e39a21a368cff5150c68cc8beeea6e8aab0c755f8bbdd",
"generated/python/__init__.py": "daa04cec586768866d0996c779f62073e6773fe63750ff668327991fac2fd0f5",
Expand Down
22 changes: 19 additions & 3 deletions examples/showcase/codegen/runtime/drizzle-fastify/filter-parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ export function parseFilterParams(opts: ParseFilterOpts): ParseFilterResult {
}

if (typeof opts.query.sort === "string") {
result.orderBy = parseSort(opts.query.sort, opts.table, opts.sortAllowlist);
result.orderBy = parseSort(opts.query.sort, opts.table, opts.sortAllowlist, opts.dialect);
}

if (typeof opts.query.search === "string" && opts.query.search !== "") {
Expand Down Expand Up @@ -421,7 +421,7 @@ function ensureArray(v: unknown, key: string): unknown[] {
throw new FilterParseError("filter.invalid_value", `Expected array for "${key}".`, { field: key });
}

function parseSort(spec: string, table: AnyTable, sortAllowlist: SortAllowlist): SQLWrapper[] {
function parseSort(spec: string, table: AnyTable, sortAllowlist: SortAllowlist, dialect: SqlDialect | undefined): SQLWrapper[] {
const colonIdx = spec.indexOf(":");
const field = colonIdx === -1 ? spec : spec.slice(0, colonIdx);
if (!sortAllowlist[field]) {
Expand Down Expand Up @@ -458,5 +458,21 @@ function parseSort(spec: string, table: AnyTable, sortAllowlist: SortAllowlist):
);
}
const col = table[field];
return [order === "asc" ? asc(col as any) : desc(col as any)];
return nullsAsLargest(col as any, order, dialect);
}

/**
* Order a column with nulls as LARGER than any value: ASC puts them last, DESC first, on
* every engine. That is Postgres's own default; SQLite and MySQL put nulls first on ASC, so
* a bare `ORDER BY col` answered the same list in a different order per engine. SQLite
* (3.30+, D1 included) and Postgres take `NULLS FIRST/LAST`; MySQL has no such clause, so it
* gets a leading `col IS NULL` key (0 for a value, 1 for null).
*/
function nullsAsLargest(col: SQLWrapper, order: "asc" | "desc", dialect: SqlDialect | undefined): SQLWrapper[] {
if (dialect === "mysql") {
return order === "asc"
? [asc(sql`(${col} IS NULL)`), asc(col)]
: [desc(sql`(${col} IS NULL)`), desc(col)];
}
return [order === "asc" ? sql`${col} ASC NULLS LAST` : sql`${col} DESC NULLS FIRST`];
}
8 changes: 4 additions & 4 deletions examples/showcase/generated/csharp/SubscriberRoutes.g.cs
Original file line number Diff line number Diff line change
Expand Up @@ -197,10 +197,10 @@ private static IOrderedQueryable<Subscriber> ApplySortSubscriber(IQueryable<Subs
{
return field switch
{
"Id" => desc ? q.OrderByDescending(x => EF.Property<object>(x!, "Id")) : q.OrderBy(x => EF.Property<object>(x!, "Id")),
"Email" => desc ? q.OrderByDescending(x => EF.Property<object>(x!, "Email")) : q.OrderBy(x => EF.Property<object>(x!, "Email")),
"Name" => desc ? q.OrderByDescending(x => EF.Property<object>(x!, "Name")) : q.OrderBy(x => EF.Property<object>(x!, "Name")),
"CreatedAt" => desc ? q.OrderByDescending(x => EF.Property<object>(x!, "CreatedAt")) : q.OrderBy(x => EF.Property<object>(x!, "CreatedAt")),
"Id" => desc ? q.OrderByDescending(x => EF.Property<object>(x!, "Id") == null).ThenByDescending(x => EF.Property<object>(x!, "Id")) : q.OrderBy(x => EF.Property<object>(x!, "Id") == null).ThenBy(x => EF.Property<object>(x!, "Id")),
"Email" => desc ? q.OrderByDescending(x => EF.Property<object>(x!, "Email") == null).ThenByDescending(x => EF.Property<object>(x!, "Email")) : q.OrderBy(x => EF.Property<object>(x!, "Email") == null).ThenBy(x => EF.Property<object>(x!, "Email")),
"Name" => desc ? q.OrderByDescending(x => EF.Property<object>(x!, "Name") == null).ThenByDescending(x => EF.Property<object>(x!, "Name")) : q.OrderBy(x => EF.Property<object>(x!, "Name") == null).ThenBy(x => EF.Property<object>(x!, "Name")),
"CreatedAt" => desc ? q.OrderByDescending(x => EF.Property<object>(x!, "CreatedAt") == null).ThenByDescending(x => EF.Property<object>(x!, "CreatedAt")) : q.OrderBy(x => EF.Property<object>(x!, "CreatedAt") == null).ThenBy(x => EF.Property<object>(x!, "CreatedAt")),
_ => (IOrderedQueryable<Subscriber>)q.OrderBy(x => 0),
};
}
Expand Down
9 changes: 6 additions & 3 deletions fixtures/api-contract-conformance/report/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,12 @@ object uses (`InvoicesByMonth` is `/api/invoices_by_months`).
(`revenueCents`) is `null` with the key present, and the sum with `@default: 0`
(`revenueOrZero`) is `0`. `?filter[sales][gt]=0` removes the empty row at request time.
- **A defaulted measure filters and sorts as its default.** `?filter[revenueOrZero][eq]=0`
matches the empty row, and `?sort=revenueOrZero:asc` puts it first on every engine. The
corpus never sorts a measure without a `@default` across a null row: where a null sorts is
the engine's choice (Postgres last ascending, SQLite and MySQL first).
matches the empty row, and `?sort=revenueOrZero:asc` puts it first on every engine.
- **Nulls sort as larger than any value** (`sort-null-ordering`): ascending puts them last,
descending first, on every engine. `?sort=revenueCents:asc` ends on `Cinder`'s null row and
`:desc` starts on it. That is Postgres's own default; SQLite and MySQL would put a null first
ascending, so each port pins the order (`NULLS LAST` / `NULLS FIRST`, or a leading
`col IS NULL` key where the engine has no such clause).
- **No grid or form is generated for a report in any port.** The read route and its row type
are the surface in every port; TypeScript, the one port with a client tier, also generates
the list hook. The corpus gates the route, not the hook.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
name: report-sort-null-ordering
description: >
Nulls sort as larger than any value, on every engine: ascending puts them last, descending
puts them first. Cinder has no sale, so its revenueCents (a sum with no @default) is null.
That is Postgres's own default; SQLite and MySQL put a null first ascending, so each port
lowers the sort to `NULLS LAST` / `NULLS FIRST`, or to a leading `col IS NULL` key where
the engine has no such clause, and the same request answers the same order everywhere.
requests:
- id: asc-nulls-last
method: GET
path: /api/product_revenues?sort=revenueCents:asc
expect:
status: 200
body:
equals:
- { productId: 2, productName: "Beacon", sales: 1, revenueCents: 4500, revenueOrZero: 4500 }
- { productId: 1, productName: "Atlas", sales: 2, revenueCents: 30000, revenueOrZero: 30000 }
- { productId: 3, productName: "Cinder", sales: 0, revenueCents: null, revenueOrZero: 0 }
- id: desc-nulls-first
method: GET
path: /api/product_revenues?sort=revenueCents:desc
expect:
status: 200
body:
equals:
- { productId: 3, productName: "Cinder", sales: 0, revenueCents: null, revenueOrZero: 0 }
- { productId: 1, productName: "Atlas", sales: 2, revenueCents: 30000, revenueOrZero: 30000 }
- { productId: 2, productName: "Beacon", sales: 1, revenueCents: 4500, revenueOrZero: 4500 }
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ description: >
A sort on a defaulted boolean measure is deterministic on every engine: false sorts
before true, and the product with no sale sorts as its default, false, never as a null
whose place the engine chooses. Atlas is the only true row, so it is first descending.
The corpus never sorts `hasRefund` (no @default) across Cinder's null row.
A sort across `hasRefund` (no @default), where Cinder's row is null, is pinned by
sort-null-ordering.
requests:
- id: desc
method: GET
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
name: report-sort-on-defaulted-measure
description: >
A sort on a defaulted measure is deterministic on every engine: the product with no sale
sorts as its default, 0, never as a null whose place the engine chooses. The corpus never
sorts revenueCents (no @default) across that row, because Postgres puts a null last
ascending and SQLite and MySQL put it first.
sorts as its default, 0. A sort across a measure with no @default, where that row is null,
is pinned by sort-null-ordering.
requests:
- id: asc
method: GET
Expand Down
5 changes: 2 additions & 3 deletions fixtures/persistence-conformance/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -408,9 +408,8 @@ Wire shapes follow the view's column types and [`normalization.md`](./normalizat
and `"0"`), a `date`-grain bucket is a `DATE` string, and an instant `hour` bucket is a
`TIMESTAMPTZ` string in UTC (`"2026-05-04T03:00:00Z"`). A measure with `@default` keeps its
column type, so a defaulted integral `sum` is still a `BIGINT` string and a defaulted ratio a
canonical decimal string. No scenario sorts a measure that can be null across a row where it
is null: engines disagree on where nulls sort (Postgres last ascending, SQLite and MySQL
first), while a defaulted measure sorts as its default everywhere.
canonical decimal string. A defaulted measure sorts as its default everywhere. Where a null
sorts is fixed in every port: see [Null ordering](../../docs/features/api-contract.md#null-ordering).

### Filter operators

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ public void An_enum_dimension_of_a_report_is_sortable()
routes.IndexOf("SortAllowlist =", StringComparison.Ordinal)..routes.IndexOf("SortDefaultDesc =", StringComparison.Ordinal)];
Assert.Contains(" \"Status\",\n", allowlist.ReplaceLineEndings("\n"));
Assert.Contains(
" \"Status\" => desc ? q.OrderByDescending(x => EF.Property<object>(x!, \"Status\")) : q.OrderBy(x => EF.Property<object>(x!, \"Status\")),",
" \"Status\" => desc ? q.OrderByDescending(x => EF.Property<object>(x!, \"Status\") == null).ThenByDescending(x => EF.Property<object>(x!, \"Status\")) : q.OrderBy(x => EF.Property<object>(x!, \"Status\") == null).ThenBy(x => EF.Property<object>(x!, \"Status\")),",
routes);
// Every derived field of the cube is in the sort allowlist, in Table B order.
var sortable = allowlist.Split('\n').Select(l => l.Trim()).Where(l => l.StartsWith('"'))
Expand Down
9 changes: 9 additions & 0 deletions server/csharp/MetaObjects.Codegen.Tests/WriteThroughTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,15 @@ public void Routes_read_via_view_and_write_via_table()
Assert.Contains("ApplySortOrder(IQueryable<OrderView> q", src);
}

[Fact]
public void Entity_list_sort_puts_nulls_largest()
{
// Nulls sort as larger than any value on every provider: ASC last, DESC first.
var src = Src(new RoutesGenerator().Generate(Ctx(Load())), "OrderRoutes.g.cs");
Assert.Contains(".OrderByDescending(x => EF.Property<object>(x!, \"Id\") == null).ThenByDescending(", src);
Assert.Contains(".OrderBy(x => EF.Property<object>(x!, \"Id\") == null).ThenBy(", src);
}

// ---- Reverse finders are reads → route to the view DbSet --------------------

[Fact]
Expand Down
17 changes: 13 additions & 4 deletions server/csharp/MetaObjects.Codegen/Generators/RoutesGenerator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -433,9 +433,9 @@ protected virtual EmittedFile GenerateStandardRoutes(MetaObject entity, GenConte
sb.AppendLine(" {");
foreach (var name in sortFields)
{
var exact = $"desc ? q.OrderByDescending(x => EF.Property<object>(x!, \"{name}\")) : q.OrderBy(x => EF.Property<object>(x!, \"{name}\"))";
var exact = NullsLargestSort("q", $"EF.Property<object>(x!, \"{name}\")");
sb.AppendLine(decimalSortFields.Contains(name)
? $" \"{name}\" => sqlite ? (desc ? q.OrderByDescending(x => (double?)EF.Property<decimal?>(x!, \"{name}\")) : q.OrderBy(x => (double?)EF.Property<decimal?>(x!, \"{name}\"))) : ({exact}),"
? $" \"{name}\" => sqlite ? ({NullsLargestSort("q", $"(double?)EF.Property<decimal?>(x!, \"{name}\")")}) : ({exact}),"
: $" \"{name}\" => {exact},");
}
sb.AppendLine(" _ => (IOrderedQueryable<" + readCls + ">)q.OrderBy(x => 0),");
Expand Down Expand Up @@ -635,7 +635,7 @@ protected virtual EmittedFile GenerateTphRoutes(MetaObject baseEntity, TphPlan t
sb.AppendLine(" return field switch");
sb.AppendLine(" {");
foreach (var name in sortFields)
sb.AppendLine($" \"{name}\" => desc ? q.OrderByDescending(x => EF.Property<object>(x!, \"{name}\")) : q.OrderBy(x => EF.Property<object>(x!, \"{name}\")),");
sb.AppendLine($" \"{name}\" => {NullsLargestSort("q", $"EF.Property<object>(x!, \"{name}\")")},");
sb.AppendLine(" _ => (IOrderedQueryable<" + baseCls + ">)q.OrderBy(x => 0),");
sb.AppendLine(" };");
sb.AppendLine(" }");
Expand Down Expand Up @@ -1208,7 +1208,7 @@ private static void AppendTphSubtypeRoutes(
sb.AppendLine(" var desc = parts.Length > 1");
sb.AppendLine(" ? string.Equals(parts[1], \"desc\", System.StringComparison.OrdinalIgnoreCase)");
sb.AppendLine($" : {subCls}SortDefaultDesc.TryGetValue(resolved, out var dd) && dd;");
sb.AppendLine(" q = desc ? q.OrderByDescending(x => EF.Property<object>(x!, resolved)) : q.OrderBy(x => EF.Property<object>(x!, resolved));");
sb.AppendLine(" q = " + NullsLargestSort("q", "EF.Property<object>(x!, resolved)") + ";");
sb.AppendLine(" }");
sb.AppendLine(" if (qs.TryGetValue(\"offset\", out var offRaw) && int.TryParse(offRaw, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var off) && off > 0) q = q.Skip(off);");
sb.AppendLine(" if (qs.TryGetValue(\"limit\", out var limRaw) && int.TryParse(limRaw, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var lim) && lim > 0) q = q.Take(lim);");
Expand Down Expand Up @@ -1449,4 +1449,13 @@ private static void AppendSortDefaultOrder(StringBuilder sb, MetaObject owner, s
sb.AppendLine();
}


/// <summary>
/// The emitted sort expression for one column, with nulls LARGER than any value: ASC puts
/// them last, DESC first, on every provider (Postgres's own default; SQLite and MySQL
/// would otherwise put them first on ASC). A leading `col == null` key says it in a way
/// every EF Core provider translates, where `NULLS LAST` is not portable.
/// </summary>
private static string NullsLargestSort(string query, string column) =>
$"desc ? {query}.OrderByDescending(x => {column} == null).ThenByDescending(x => {column}) : {query}.OrderBy(x => {column} == null).ThenBy(x => {column})";
}
Loading
Loading