diff --git a/TSLANG_ISSUES.md b/TSLANG_ISSUES.md index f20f40246..6c0f0807b 100644 --- a/TSLANG_ISSUES.md +++ b/TSLANG_ISSUES.md @@ -2150,3 +2150,36 @@ Still open, looked at: - `a[a.length] = x` on an array does not grow it (gc writes past the end; rc and own crash). Tests: `00object_dictionary.ts`, `00own_param_exclusive_moves.ts` (compile, JIT, rc, own). + +153 (iterables and generators, the compiler side of the default library's iteration types - TypeScriptCompilerDefaultLib +#113, whose `Iterable`, `Generator` and the rest are objects with `next`, 2026-11-21). Fixed: +- An array or a string given where an iterator interface is wanted (`sum([1.5, 2])` of `sum(v: Iterable)`, + `cat("abc")` of `cat(v: Iterable)`): "type 'number[]' can't be cast to interface 'Iterable': it has + none of the interface's members". A generator over its elements is given - `__array_values` / + `__string_values`, made from source once a module, generic so that a pass whose module is thrown away does not + leave a call to nothing - its element the interface's (`next`'s result's `value`): an `s32[]` given to an + `Iterable` is converted first (castIterableToIterator). Under -mm=own the generator borrows the array. +- A generator typed `Generator` (`Iterator`, `Iterable`, `IterableIterator`) yields T: its next() is + declared `{ value: T; done: boolean }`, so `yield 1` is a number. It was typed by what it yields, the literal's + s32: "can't assign '() => {value:s32, done:boolean}' to '() => {value:number, done:boolean}'". +- A value returned by a generator typed `Generator` with a TReturn other than T (and not void, + undefined or any) is an error at the `return`: next() gives `{ value: T }`, and `return "end"` in a + `Generator` was cast to a number without a word (it read 0). + +Still open, looked at: +- next(value) / TNext: next() would take `value?: TNext` and the resumed `yield` evaluate to it - tried, and it works + for a generator cast to an interface declaring `next(value?)` - but a generator's next() is then cast to + interfaces declaring `next: () => ...` (a program's own, the library's Iterable), and an object's method of + another arity has no adapter in the interface's vtable (castObjectToInterface; a class has them, 150). Needs that + adapter, then `Generator`'s next in the library. return(v) / throw(e) likewise. +- -mm=own: a generator typed as an interface other than the library-instantiated Iterator - a program's own, or + Iterable - is "a value takes a second reference; -mm=own cannot prove a move or a borrow here yet" at its + `for...of` (before these changes too), and so is an array given as an Iterable. +- A generator as its own iterator (`gen[Symbol.iterator]()`: "field "iterator" can't be found"): a method + `[Symbol.iterator]() { return this; }` on the generator object was tried - a generator made inside another's + machinery (the builtin map/filter of an iterator) then resolved that `this` through the outer one's alias + ("can't resolve name: .this"), and under -mm=own a method returning `this` is refused. Likewise + `[Symbol.asyncIterator]` of an async generator. + +Tests: `00iterables.ts`, `00generator_typed_yield.ts` (compile, JIT, rc), +diagnostic-location `generator_return_of_other_type.ts`. diff --git a/tslang/lib/TypeScript/MLIRGenCast.cpp b/tslang/lib/TypeScript/MLIRGenCast.cpp index a967fe347..1a61990bb 100644 --- a/tslang/lib/TypeScript/MLIRGenCast.cpp +++ b/tslang/lib/TypeScript/MLIRGenCast.cpp @@ -1789,6 +1789,17 @@ namespace mlirgen // lowering could not make, and the callee read garbage. An interface without members // takes any value, as in TypeScript. auto baseValueType = mth.stripLiteralType(valueType); + + // an array or a string given where an iterator is wanted (`Iterable`, `Generator`: an object + // with `next`, as the default library declares them) is iterated by a generator over its elements + if (isa(baseValueType)) + { + if (auto iterated = castIterableToIterator(location, value, baseValueType, interfaceType, genContext)) + { + return *iterated; + } + } + if (baseValueType.isIntOrIndexOrFloat() || isa(baseValueType)) @@ -1806,6 +1817,123 @@ namespace mlirgen return std::nullopt; } + // `sum([1.5, 2])` of `sum(v: Iterable)`, `cat("abc")` of `cat(v: Iterable)`: the interface is + // an iterator - it has `next`, whose result's `value` is its element - and an array or a string has none of + // its members. A generator over the elements, made here from source once a module (`__array_values`, + // `__string_values`), is: its elements are cast to the interface's element type as they are given (`s32` + // to `number`). Under -mm=own the generator borrows the array (@borrows), as an array's values() does. + std::optional MLIRGenImpl::castIterableToIterator(mlir::Location location, mlir::Value value, + mlir::Type baseValueType, mlir_ts::InterfaceType interfaceType, const GenContext &genContext) + { + auto interfaceInfo = getInterfaceInfoByFullName(interfaceType.getName().getValue()); + if (!interfaceInfo) + { + return std::nullopt; + } + + // the element: the `value` of what `next` returns + mlir::Type nextType; + if (auto *field = interfaceInfo->findField(builder.getStringAttr("next"))) + { + nextType = field->type; + } + else if (auto *method = interfaceInfo->findMethod("next")) + { + nextType = method->funcType; + } + + auto nextFuncType = dyn_cast_or_null(nextType); + if (!nextFuncType || nextFuncType.getNumResults() != 1) + { + return std::nullopt; + } + + mlir::Type elementType; + llvm::SmallVector resultFields; + if (mlir::succeeded(mth.getFields(nextFuncType.getResult(0), resultFields))) + { + for (auto &field : resultFields) + { + if (auto name = dyn_cast_or_null(field.id); name && name.getValue() == "value") + { + elementType = field.type; + } + } + } + + if (!elementType) + { + return std::nullopt; + } + + auto isString = isa(baseValueType); + // info, we add "_" extra as scanner append "_" in front of "__"; + auto funcName = isString ? "___string_values" : "___array_values"; + auto funcRef = resolveIdentifier(location, funcName, genContext); + if (!funcRef) + { + std::string src = isString ? "@borrows(\"s\")\n" + "function* __string_values(s: string) {\n" + "for (const c of s) yield c;\n" + "}\n" + : "@borrows(\"a\")\n" + "function* __array_values(a: T[]) {\n" + "for (const v of a) yield v;\n" + "}\n"; + MLIRLocationGuard vgLoc(overwriteLoc); + overwriteLoc = location; + if (mlir::failed(parsePartialStatements(stows(src)))) + { + return ValueOrLogicalResult(mlir::failure()); + } + + funcRef = resolveIdentifier(location, funcName, genContext); + if (!funcRef) + { + return ValueOrLogicalResult(mlir::failure()); + } + } + + SmallVector operands; + NodeArray typeArguments; + GenContext iterGenContext(genContext); + if (isString) + { + auto stringValue = cast(location, getStringType(), value, genContext); + if (stringValue.failed_or_no_value()) + { + return ValueOrLogicalResult(mlir::failure()); + } + + operands.push_back(V(stringValue)); + } + else + { + // the array of the interface's elements: `s32[]` given to `Iterable` is converted first + auto arrayValue = cast(location, getArrayType(elementType), value, genContext); + if (arrayValue.failed_or_no_value()) + { + return ValueOrLogicalResult(mlir::failure()); + } + + operands.push_back(V(arrayValue)); + } + + // generic, the string one too (its T unused): a function made only in a pass whose module is thrown away + // (an evaluation, a discovery pass) left a call to nothing; a specialization is made again where it is used + NodeFactory nf(NodeFactoryFlags::None); + iterGenContext.typeAliasMap.insert({".ITER_T", isString ? mlir::Type(getStringType()) : elementType}); + typeArguments.push_back(nf.createTypeReferenceNode(nf.createIdentifier(S(".ITER_T")).as())); + + auto generator = mlirGenCallExpression(location, funcRef, typeArguments, operands, iterGenContext); + if (generator.failed_or_no_value()) + { + return ValueOrLogicalResult(mlir::failure()); + } + + return cast(location, interfaceType, V(generator), genContext); + } + ValueOrLogicalResult MLIRGenImpl::mlirGenClassInterfaceVTable(mlir::Location location, mlir::Value value, mlir_ts::ClassType classType, mlir_ts::InterfaceType interfaceType, const GenContext &genContext) { auto classInfo = getClassInfoByFullName(classType.getName().getValue()); diff --git a/tslang/lib/TypeScript/MLIRGenFunctions.cpp b/tslang/lib/TypeScript/MLIRGenFunctions.cpp index 8e85d5bcd..8d849f358 100644 --- a/tslang/lib/TypeScript/MLIRGenFunctions.cpp +++ b/tslang/lib/TypeScript/MLIRGenFunctions.cpp @@ -1209,9 +1209,31 @@ namespace mlirgen // the statements of next() that use the generator object's own `this`, not the body's auto ownStatementsCount = nextStatements.size(); + // Generator with a TReturn of its own (not T, void, undefined or any): a returned value has no + // place in next()'s `{ value: T }` (see nextResultType below) + auto returnTypeOfItsOwn = false; + if (auto declaredType = functionLikeDeclarationBaseAST->type; declaredType && declaredType == SyntaxKind::TypeReference) + { + auto typeReference = declaredType.as(); + if (typeReference->typeName == SyntaxKind::Identifier && + MLIRHelper::getName(typeReference->typeName.as()) == "Generator" && + typeReference->typeArguments.size() >= 2) + { + auto yieldType = typeReference->typeArguments[0]; + auto returnType = typeReference->typeArguments[1]; + SyntaxKind returnKind = returnType; + returnTypeOfItsOwn = returnKind != SyntaxKind::VoidKeyword && returnKind != SyntaxKind::UndefinedKeyword && + returnKind != SyntaxKind::AnyKeyword && print(returnType) != print(yieldType); + } + } + // a `return` of the body finishes the generator; one of a nested function is that function's own FilterVisitorSkipFuncsAST returnsVisitor(SyntaxKind::ReturnStatement, [&](auto returnStatement) { returnStatement->internalFlags |= InternalFlags::GeneratorReturn; + if (returnTypeOfItsOwn && returnStatement->expression) + { + returnStatement->internalFlags |= InternalFlags::GeneratorReturnOfOtherType; + } }); if (functionLikeDeclarationBaseAST->body) @@ -1242,10 +1264,35 @@ namespace mlirgen // create next body auto nextBody = nf.createBlock(nextStatements, /*multiLine*/ false); + // `function* g(): Generator` (Iterator, Iterable, IterableIterator): next() returns + // `{ value: T; done: boolean }`, so what is yielded is given T as its type (`yield 1` a number, not the + // s32 the literal is), as TypeScript types it; without it the yields' own type was the element's, and + // `yield 1` could not be a Generator ("can't assign '() => {value:s32, ...}'") + TypeNode nextResultType = undefined; + if (auto declaredType = functionLikeDeclarationBaseAST->type; + declaredType && declaredType == SyntaxKind::TypeReference && !hasModifier(functionLikeDeclarationBaseAST, SyntaxKind::AsyncKeyword)) + { + auto typeReference = declaredType.as(); + if (typeReference->typeName == SyntaxKind::Identifier && typeReference->typeArguments.size() >= 1) + { + auto typeName = MLIRHelper::getName(typeReference->typeName.as()); + if (typeName == "Generator" || typeName == "Iterator" || typeName == "Iterable" || + typeName == "IterableIterator") + { + NodeArray members; + members.push_back(nf.createPropertySignature(undefined, nf.createIdentifier(S("value")), undefined, + typeReference->typeArguments.front())); + members.push_back(nf.createPropertySignature(undefined, nf.createIdentifier(S("done")), undefined, + nf.createToken(SyntaxKind::BooleanKeyword).as())); + nextResultType = nf.createTypeLiteralNode(members); + } + } + } + // create method next in object auto nextMethodDecl = nf.createMethodDeclaration(undefined, undefined, nf.createIdentifier(S(ITERATOR_NEXT)), undefined, - undefined, undefined, undefined, nextBody); + undefined, undefined, nextResultType, nextBody); nextMethodDecl->internalFlags |= InternalFlags::VarsInObjectContext | InternalFlags::GeneratorNext; // copy location info, to fix issue with names of anonymous functions diff --git a/tslang/lib/TypeScript/MLIRGenImpl.h b/tslang/lib/TypeScript/MLIRGenImpl.h index ead83f7c4..e4227342e 100644 --- a/tslang/lib/TypeScript/MLIRGenImpl.h +++ b/tslang/lib/TypeScript/MLIRGenImpl.h @@ -18415,6 +18415,8 @@ class MLIRGenImpl SmallVector &operands, const GenContext &genContext); std::optional mlirGenDictionaryDelete(mlir::Location location, Expression operand, const GenContext &genContext); + std::optional castIterableToIterator(mlir::Location location, mlir::Value value, + mlir::Type baseValueType, mlir_ts::InterfaceType interfaceType, const GenContext &genContext); ValueOrLogicalResult castTupleToDictionary(mlir::Location location, mlir::Value value, mlir_ts::TupleType srcTupleType, mlir_ts::ClassType classType, const GenContext &genContext); std::optional mlirGenObjectLiteralAsDictionary(mlir::Location location, diff --git a/tslang/lib/TypeScript/MLIRGenStatements.cpp b/tslang/lib/TypeScript/MLIRGenStatements.cpp index 437203cfb..670976a18 100644 --- a/tslang/lib/TypeScript/MLIRGenStatements.cpp +++ b/tslang/lib/TypeScript/MLIRGenStatements.cpp @@ -859,6 +859,14 @@ namespace mlirgen // `return e` in a generator: {value: e, done: true}, and every next() after it is done (step -1) if (!!(returnStatementAST->internalFlags & InternalFlags::GeneratorReturn)) { + if (!!(returnStatementAST->internalFlags & InternalFlags::GeneratorReturnOfOtherType)) + { + emitError(location) << "a value returned by a generator typed Generator with a TReturn other " + "than T is not supported: next() gives `{ value: T }` - return it as a T, or " + "yield it"; + return mlir::failure(); + } + NodeFactory nf(NodeFactoryFlags::None); // as yield sets its resume point: `this.step` of the generator object, or `step` of a captured one diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index 93aaf7c5a..c462862ca 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -466,6 +466,8 @@ tslang_add_test(NAME test-compile-00-interface-numeric-return COMMAND test-runne tslang_add_test(NAME test-compile-00-interface-method-adapters COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00interface_method_adapters.ts") tslang_add_test(NAME test-compile-00-own-param-exclusive-moves COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00own_param_exclusive_moves.ts") tslang_add_test(NAME test-compile-00-object-dictionary COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00object_dictionary.ts") +tslang_add_test(NAME test-compile-00-iterables COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00iterables.ts") +tslang_add_test(NAME test-compile-00-generator-typed-yield COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00generator_typed_yield.ts") tslang_add_test(NAME test-compile-00-constructor-type-value COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00constructor_type_value.ts") tslang_add_test(NAME test-compile-00-string-concat-chain COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00string_concat_chain.ts") tslang_add_test(NAME test-compile-00-arrays COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00array.ts") @@ -1294,6 +1296,10 @@ tslang_add_test(NAME test-jit-own-00-own-param-exclusive-moves COMMAND test-runn tslang_add_test(NAME test-jit-00-object-dictionary COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00object_dictionary.ts") tslang_add_test(NAME test-jit-rc-00-object-dictionary COMMAND test-runner -jit -mm=rc "${PROJECT_SOURCE_DIR}/test/tester/tests/00object_dictionary.ts") tslang_add_test(NAME test-jit-own-00-object-dictionary COMMAND test-runner -jit -mm=own "${PROJECT_SOURCE_DIR}/test/tester/tests/00object_dictionary.ts") +tslang_add_test(NAME test-jit-00-iterables COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00iterables.ts") +tslang_add_test(NAME test-jit-rc-00-iterables COMMAND test-runner -jit -mm=rc "${PROJECT_SOURCE_DIR}/test/tester/tests/00iterables.ts") +tslang_add_test(NAME test-jit-00-generator-typed-yield COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00generator_typed_yield.ts") +tslang_add_test(NAME test-jit-rc-00-generator-typed-yield COMMAND test-runner -jit -mm=rc "${PROJECT_SOURCE_DIR}/test/tester/tests/00generator_typed_yield.ts") tslang_add_test(NAME test-jit-00-constructor-type-value COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00constructor_type_value.ts") tslang_add_test(NAME test-jit-rc-00-constructor-type-value COMMAND test-runner -jit -mm=rc "${PROJECT_SOURCE_DIR}/test/tester/tests/00constructor_type_value.ts") tslang_add_test(NAME test-jit-own-00-constructor-type-value COMMAND test-runner -jit -mm=own "${PROJECT_SOURCE_DIR}/test/tester/tests/00constructor_type_value.ts") @@ -4143,6 +4149,15 @@ set_tests_properties(test-compile-diagnostic-location-generic-construct-signatur PROPERTIES PASS_REGULAR_EXPRESSION "generic_construct_signature_type_literal\\.ts:3:5: error: a generic construct signature in a type literal is not supported yet" FAIL_REGULAR_EXPRESSION "Stack dump|Assertion failed") +# A value returned by a generator typed Generator of another TReturn is an error at the return. +add_test(NAME test-compile-diagnostic-location-generator-return-of-other-type + COMMAND $ --emit=obj --no-default-lib -mm=none + "${PROJECT_SOURCE_DIR}/test/tester/diagnostic-location/generator_return_of_other_type.ts" + -o "${CMAKE_CURRENT_BINARY_DIR}/diagnostic-location-generator-return-of-other-type.obj") +set_tests_properties(test-compile-diagnostic-location-generator-return-of-other-type + PROPERTIES PASS_REGULAR_EXPRESSION "generator_return_of_other_type\\.ts:9:5: error: a value returned by a generator typed Generator with a TReturn other than T is not supported" + FAIL_REGULAR_EXPRESSION "Stack dump|Assertion failed") + # A generic method in a type literal is an error at the method. add_test(NAME test-compile-diagnostic-location-generic-method-type-literal COMMAND $ --emit=obj --no-default-lib -mm=none diff --git a/tslang/test/tester/diagnostic-location/generator_return_of_other_type.ts b/tslang/test/tester/diagnostic-location/generator_return_of_other_type.ts new file mode 100644 index 000000000..38e69820b --- /dev/null +++ b/tslang/test/tester/diagnostic-location/generator_return_of_other_type.ts @@ -0,0 +1,10 @@ +// A value returned by a generator typed Generator with a TReturn other than T is an error at the return: +// next() gives `{ value: T }`, and "end" was cast to a number without a word (it read 0). +interface Generator { + next: () => { value: T, done: boolean }; +} + +function* h(): Generator { + yield 1; + return "end"; +} diff --git a/tslang/test/tester/tests/00generator_typed_yield.ts b/tslang/test/tester/tests/00generator_typed_yield.ts new file mode 100644 index 000000000..255e02a55 --- /dev/null +++ b/tslang/test/tester/tests/00generator_typed_yield.ts @@ -0,0 +1,68 @@ +// A generator typed Generator (or Iterator, Iterable, IterableIterator) yields T: `yield 1` in a +// Generator is a number. Its next() was typed by what it yields, the literal's s32, and the generator +// could not be a Generator: "can't assign '() => {value:s32, done:boolean}' to '() => {value:number, ...}'". +// (-mm=own: a generator typed as an interface other than the library's is still refused.) +// as the default library declares them (the tests compile without it) +type IterateResult = { value: T, done: boolean }; +interface Iterable { + next: () => IterateResult; +} +interface IterableIterator { + next: () => IterateResult; +} +interface Generator { + next: () => IterateResult; +} + +function* g(): Generator { + yield 1; + yield 2.5; +} + +function* k(): IterableIterator { + let i = 0; + while (i < 3) { + yield i++; + } +} + +function* names(): Iterable { + yield "a"; + yield "b"; +} + +function* stop(): Generator { + yield 1; + return; +} + +function main() { + let s = 0.0; + for (const x of g()) { + s += x; + } + + assert(s == 3.5); + + let t = 0; + for (const x of k()) { + t += x; + } + + assert(t == 3); + + let n = ""; + for (const x of names()) { + n += x; + } + + assert(n == "ab"); + + let c = 0; + for (const x of stop()) { + c += x; + } + + assert(c == 1); + print("done."); +} diff --git a/tslang/test/tester/tests/00iterables.ts b/tslang/test/tester/tests/00iterables.ts new file mode 100644 index 000000000..52ff884ea --- /dev/null +++ b/tslang/test/tester/tests/00iterables.ts @@ -0,0 +1,45 @@ +// An array or a string given where an iterator interface is wanted (`Iterable`: an object with `next`, as the +// default library declares it) was "type 'number[]' can't be cast to interface 'Iterable': it has none of +// the interface's members". A generator over its elements is given; an `s32[]` given to an `Iterable` is +// converted first. (-mm=own: a generator typed as an interface other than the library's is still refused.) +// as the default library declares them (the tests compile without it) +type IterateResult = { value: T, done: boolean }; +interface Iterable { + next: () => IterateResult; +} +interface IterableIterator { + next: () => IterateResult; +} +interface Generator { + next: () => IterateResult; +} + +function sum(v: Iterable) { + let s = 0.0; + for (const x of v) { + s += x; + } + + return s; +} + +function cat(v: Iterable) { + let s = ""; + for (const x of v) { + s += x; + } + + return s; +} + +function main() { + assert(sum([1.5, 2]) == 3.5); + const a: number[] = [1, 2, 3]; + assert(sum(a) == 6); + const b = [4, 5]; + assert(sum(b) == 9); + assert(cat("abc") == "abc"); + const names: string[] = ["x", "y"]; + assert(cat(names) == "xy"); + print("done."); +} diff --git a/tslang/ts-new-parser/enums.h b/tslang/ts-new-parser/enums.h index 1184ef0bc..3c04f05ae 100644 --- a/tslang/ts-new-parser/enums.h +++ b/tslang/ts-new-parser/enums.h @@ -581,7 +581,10 @@ enum class InternalFlags : number InModuleBlock = 1 << 18, // a `let`/`const` of a function's body read by a function before it: one of the function's variables, declared // before the body as a `var` is (declareFunctionVars) - HoistedLexical = 1 << 19 + HoistedLexical = 1 << 19, + // `return v` of a generator typed Generator whose TReturn is not T: next() gives `{ value: T }`, and + // v has no place in it (it was cast to T without a word: "end" became 0) + GeneratorReturnOfOtherType = 1 << 20 }; ENUM_OPS(InternalFlags)