gh-153569: simplify tokenizer source and formatted-string state

Lib/test/support/os_helper.py

@@ -433,7 +433,7 @@ def _rmtree_inner(path):
                           % (fullname, exc),
                           file=sys.__stderr__)
                     mode = 0
-                if stat.S_ISDIR(mode):
+                if stat.S_ISDIR(mode) and not os.path.isjunction(fullname):
                     _waitfor(_rmtree_inner, fullname, waitall=True)
                     _force_run(fullname, os.rmdir, fullname)
                 else:

Lib/test/test_capi/test_tokenizer.py

@@ -9,9 +9,6 @@ class TokenizerTests(unittest.TestCase):
     def test_source(self):
         _testinternalcapi.test_tokenizer_source()
 
-    def test_cursor(self):
-        _testinternalcapi.test_tokenizer_cursor()
-
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_fstring.py

@@ -1657,6 +1657,7 @@ def __repr__(self):
         self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x')
         self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********')
         self.assertEqual(f"{C():{20=}}", 'FORMAT-20=20')
+        self.assertEqual(f"{C():{C():{4=}}}", 'FORMAT-FORMAT-4=4')
 
         self.assertRaises(SyntaxError, eval, "f'{C=]'")
 
@@ -1679,6 +1680,20 @@ def __repr__(self):
 
         self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'')
         self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'')
+        self.assertEqual(f'{"""a" # inside"""=}',
+                         '"""a" # inside"""=\'a" # inside\'')
+        self.assertEqual(f"{'''a' # inside'''=}",
+                         "'''a' # inside'''=\"a' # inside\"")
+        self.assertEqual(f'{"""a""""#" # outside
+=}', '"""a""""#" \n=\'a#\'')
+
+        x, y = 1, 2
+        self.assertEqual(f'{x != y # outside
+=}', 'x != y \n=True')
+
+        d = {'a#b': 42}
+        self.assertEqual(f'''{f"{d["a#b"]}"=}''',
+                         'f"{d["a#b"]}"=\'42\'')
 
         self.assertEqual(f'{ # some comment goes here
   """hello"""=}',  ' \n  """hello"""=\'hello\'')

Lib/test/test_repl.py

@@ -184,9 +184,8 @@ def read_until(marker, start=0):
 
     @cpython_only
     def test_lexer_buffer_realloc_with_null_start(self):
-        # gh-144759: NULL pointer arithmetic in the lexer when start and
-        # multi_line_start are NULL (uninitialized in tok_mode_stack[0])
-        # and the lexer buffer is reallocated while parsing long input.
+        # gh-144759: NULL pointer arithmetic when the lexer buffer grows
+        # while parsing long input.
         long_value = "a" * 2000
         user_input = dedent(f"""\
         x = f'{{{long_value!r}}}'
@@ -198,6 +197,22 @@ def test_lexer_buffer_realloc_with_null_start(self):
         self.assertEqual(p.returncode, 0)
         self.assertIn(long_value, output)
 
+    @cpython_only
+    def test_multiline_fstring_source_reallocation(self):
+        long_line = " " * 9000 + "+ 2"
+        user_input = (
+            'value = f"""{(\n'
+            '1\n'
+            f'{long_line}\n'
+            ')}"""\n'
+            'print(value)\n'
+        )
+        p = spawn_repl()
+        p.stdin.write(user_input)
+        output = kill_python(p)
+        self.assertEqual(p.returncode, 0)
+        self.assertIn(">>> 3\n>>> ", output)
+
     def test_close_stdin(self):
         user_input = dedent('''
             import os

Lib/test/test_syntax.py

@@ -3393,6 +3393,10 @@ def test_invalid_line_continuation_error_position(self):
         self._check_error('\nfgdfgf\n1,\\#\n2\n',
                           "unexpected character after line continuation character",
                           lineno=3, offset=4)
+        for prefix in ("f", "t"):
+            self._check_error(f'{prefix}"""{{\n\\ x}}"""',
+                              "unexpected character after line continuation character",
+                              lineno=2, offset=2)
 
     def test_invalid_line_continuation_left_recursive(self):
         # Check bpo-42218: SyntaxErrors following left-recursive rules

Lib/test/test_tokenize.py

@@ -2427,6 +2427,31 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self):
             (token.ENDMARKER, "", (1, 0), (1, 0), ""),
         )
 
+    def test_fstring_offsets_survive_buffer_reallocation(self):
+        padding = " " * 9000
+        expression_line = ")=:>{2}}\n"
+        physical_lines = [
+            'f"""\n',
+            "{(\n",
+            padding + "1\n",
+            expression_line,
+            '"""\n',
+        ]
+        source = "".join(physical_lines)
+        chunks = iter([
+            "".join(physical_lines[:2]),
+            "".join(physical_lines[2:4]),
+            physical_lines[4],
+            "",
+        ])
+
+        expected = self._get_tokens(source, extra_tokens=True)
+        tokens = list(tokenize._generate_tokens_from_c_tokenizer(
+            chunks.__next__,
+            extra_tokens=True,
+        ))
+        self.assertEqual(tokens, expected)
+
     def test_extra_tokens_relaxes_lexer_errors(self):
         cases = [
             (
@@ -2549,6 +2574,40 @@ def test_degraded_fstring_format_spec(self):
             ("f-string: single '}' is not allowed", (1, 11)),
         )
 
+    def test_carriage_return_after_debug_comment(self):
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                tokens = self._get_tokens(f"{prefix}'''{{x=# comment\r}}'''")
+                self.assertEqual(tokens[4].string, "# comment\r}")
+
+    def test_incomplete_formatted_string_comment_after_carriage_return(self):
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                for extra_tokens in (False, True):
+                    with self.assertRaises(tokenize.TokenError) as caught:
+                        self._get_tokens(
+                            f"{prefix}'{{#\r!", extra_tokens=extra_tokens
+                        )
+                    self.assertEqual(
+                        caught.exception.args,
+                        ("unexpected EOF in multi-line statement", (1, 7)),
+                    )
+
+    def test_formatted_string_nesting_limit(self):
+        def nested_string(depth, prefix):
+            source = "'x'"
+            for _ in range(depth):
+                source = f'{prefix}"{{{source}}}"'
+            return source
+
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                self._get_tokens(nested_string(149, prefix))
+                with self.assertRaisesRegex(
+                        tokenize.TokenError,
+                        "too many nested f-strings or t-strings"):
+                    self._get_tokens(nested_string(150, prefix))
+
     def test_escaped_fstring_brace_has_a_position_gap(self):
         tokens = self._get_tokens('f"a{{"', extra_tokens=True)
         self.assertEqual(

Lib/test/test_tstring.py

@@ -140,6 +140,15 @@ def test_debug_specifier(self):
         )
         self.assertEqual(fstring(t), "Value: value = 42")
 
+        class C:
+            def __format__(self, spec):
+                return f"FORMAT-{spec}"
+
+        x = y = C()
+        t = t"{x:{y:{value=}}}"
+        self.assertEqual(t.interpolations[0].format_spec,
+                         "FORMAT-value=42")
+
     def test_raw_tstrings(self):
         path = r"C:\Users"
         t = rt"{path}\Documents"
@@ -217,6 +226,10 @@ def test_syntax_errors(self):
             ("t'{x=!}'", "t-string: missing conversion character"),
             ("t'{x!z}'", "t-string: invalid conversion character 'z': "
                          "expected 's', 'r', or 'a'"),
+            ("f\"{t'{x!z}'}\"", "t-string: invalid conversion character 'z': "
+                                "expected 's', 'r', or 'a'"),
+            ("t'{f\"{x!z}\"}'", "f-string: invalid conversion character 'z': "
+                                "expected 's', 'r', or 'a'"),
             ("t'{lambda:1}'", "t-string: lambda expressions are not allowed "
                               "without parentheses"),
             ("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"),
@@ -287,5 +300,22 @@ def test_triple_quoted(self):
         )
         self.assertEqual(fstring(t), "\n        Hello,\n        Python\n        ")
 
+        t = t'{"""a" # inside"""}'
+        self.assertEqual(t.interpolations[0].expression,
+                         '"""a" # inside"""')
+
+        t = t'{"""a""""#" # outside
+}'
+        self.assertEqual(t.interpolations[0].expression, '"""a""""#"')
+
+        x, y = 1, 2
+        t = t'{x != y # outside
+}'
+        self.assertEqual(t.interpolations[0].expression, 'x != y')
+
+        d = {'a#b': 42}
+        t = t'''{f"{d["a#b"]}"}'''
+        self.assertEqual(t.interpolations[0].expression, 'f"{d["a#b"]}"')
+
 if __name__ == '__main__':
     unittest.main()

Makefile.pre.in

@@ -394,12 +394,10 @@ PEGEN_OBJS=		\
 		Parser/peg_api.o
 
 TOKENIZER_OBJS=		\
-		Parser/lexer/buffer.o \
 		Parser/lexer/lexer.o \
 		Parser/lexer/number.o \
 		Parser/lexer/state.o \
 		Parser/lexer/string.o \
-		Parser/tokenizer/cursor.o \
 		Parser/tokenizer/decoder.o \
 		Parser/tokenizer/reader.o \
 		Parser/tokenizer/source.o \
@@ -411,11 +409,9 @@ PEGEN_HEADERS= \
 		$(srcdir)/Parser/string_parser.h
 
 TOKENIZER_HEADERS= \
-		Parser/lexer/buffer.h \
 		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \
-		Parser/tokenizer/cursor.h \
 		Parser/tokenizer/reader.h \
 		Parser/tokenizer/reader_internal.h \
 		Parser/tokenizer/source.h \
@@ -3462,7 +3458,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
 MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
 MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
 MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
 MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
 MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h