chore: merge upstream 18-latest (PG 18.6, stack-overflow protection, upb) into 18-constructive

.gitattributes

@@ -1,2 +1,7 @@
-*.sql binary
-*.psql binary
+# Never convert line endings for test SQL files (e.g. to CRLF on Windows checkouts),
+# since the tests compare exact bytes. These are still text, so diffs show up in reviews.
+*.sql -text
+*.psql -text
+
+# Contains WIN1252-encoded (non-UTF-8) data
+test/sql/postgres_regress/collate.windows.win1252.sql binary

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         compiler: [clang, gcc]
-        protobuf_lib: [protobuf-c, protobuf-cpp]
+        protobuf_lib: [upb, protobuf-cpp]
         valgrind: [valgrind,no-valgrind]
         exclude:
           # this combination hits linking errors: see https://github.com/pganalyze/libpg_query/pull/289

.gitignore

@@ -9,6 +9,10 @@ pg_query.lib
 examples/*
 !examples/*.c
 
+benchmark/*
+!benchmark/*.c
+!benchmark/*.md
+
 test/*
 !test/*.c
 !test/*.sql
@@ -21,3 +25,5 @@ tmp/*
 !tmp/.gitkeep
 
 .vs/*
+
+vendor/upb/.upb-update-tmp/

CHANGELOG.md

@@ -2,6 +2,88 @@
 
 All versions are tagged by the major Postgres version, plus a minor/patch version to indicate changes to the libpg_query supporting code. The minor version does not reflect the Postgres minor version, but is instead used to indicate major or API breaking changes in libpg_query itself.
 
+## 18.1.0     2026-09-30
+
+* Security fix: Heap out-of-bounds write and read in pg_query_normalize ([GHSA-6ggm-xmc9-8ffg](https://github.com/pganalyze/libpg_query/security/advisories/GHSA-6ggm-xmc9-8ffg))
+  - When normalizing certain utility statements (e.g. `DO ... LANGUAGE`, statements with
+    string options, or `CREATE/ALTER SUBSCRIPTION ... CONNECTION`), pg_query_normalize
+    searched the query text for the location of string constants, which could yield
+    wrong locations for crafted input. This could cause out-of-bounds writes and reads
+    on the heap, leaking process memory in the normalized output or crashing the process.
+  - Constant locations are now recorded by the parser instead, and the normalizer checks
+    at runtime that constant locations never overlap
+  - This adds new location fields to the parse tree output (`DefElem.arg_location`,
+    `NotifyStmt.payload_location`, `CreateSubscriptionStmt.conninfo_location` and
+    `AlterSubscriptionStmt.conninfo_location`). Like other location fields, these are
+    ignored for fingerprinting.
+  - Applications that normalize untrusted query text should upgrade
+  - Reported by Paul Gerste (Cure53)
+* Deparser: Add strict checking for unexpected pointer values
+  - This ensures that a bad input parse tree doesn't cause the deparser to crash, and
+    instead returns an error
+  - Use cases that do not work with user input can define `PG_QUERY_DEPARSE_NO_STRICT_CHECKS`
+    to turn off the most detailed checks, for slightly better performance
+  - Reported by Paul Gerste (Cure53)
+* Add stack overflow crash protection, error out instead [#348](https://github.com/pganalyze/libpg_query/pull/348)
+  - Overly deep queries now return the standard Postgres "stack depth limit exceeded"
+    error instead of crashing the process
+  - The allowed stack depth defaults to 100 kB, auto-sized up to 2 MB, and is
+    recalculated on each call, since callers may use threads with varying stack sizes
+  - Stack depth is also checked when recursing directly into specific node types (e.g.
+    long `UNION` chains), and in `exprLocation` during raw parsing
+* Switch Protobuf implementation from protobuf-c to upb
+  - upb is developed as part of the main Protobuf project, and is substantially faster,
+    in part due to its built-in arena allocation
+  - upb also allows limiting parse depth for complex Protobuf input, avoiding crashes
+* Update to Postgres 18.6 release
+* Add `pg_query_scan_tokens` to get scan results without involving Protobuf
+  - This allows pure C callers to walk a simple list of `PgQueryScanToken` structs
+* Ignore comments when parsing queries, only treat them as significant for scanning [#378](https://github.com/pganalyze/libpg_query/pull/378)
+  - This fixes parse errors when comments are placed between related tokens
+    (e.g. `NOT /* comment */ IN`), or between string literals that get concatenated
+* Parser: Avoid quadratic memory use for rules that involve dotted names [#374](https://github.com/pganalyze/libpg_query/pull/374)
+* Return errors for PL/pgSQL statements without bodies [#363](https://github.com/pganalyze/libpg_query/pull/363)
+  - This avoids an assertion failure or crash when `CREATE FUNCTION` or `DO` omits
+    its function body
+* Fingerprinting: Add fingerprint options to `pg_query_fingerprint_opts` [#361](https://github.com/pganalyze/libpg_query/pull/361)
+  - This is a breaking change for callers of `pg_query_fingerprint_opts`,
+    which now takes a fingerprint options bitmask as a third argument
+  - By default, relation references are fingerprinted following Postgres 18+
+    query ID behavior: in SELECT/DML statements the alias name replaces the
+    relation name when present, and schema names are ignored
+  - `PG_QUERY_FINGERPRINT_RANGEVAR_IGNORE_ALIASES` always fingerprints
+    relation names and ignores aliases
+  - `PG_QUERY_FINGERPRINT_RANGEVAR_INCLUDE_SCHEMA` also fingerprints schema
+    names in SELECT/DML statements
+  - Combining both flags (`PG_QUERY_FINGERPRINT_RANGEVAR_PG17_COMPAT`)
+    matches how Postgres 17 and earlier calculate query IDs, and how
+    libpg_query 17 and earlier calculated fingerprints
+  - `PG_QUERY_FINGERPRINT_FULL_RELNAME` fingerprints the full relation name,
+    instead of the default behavior of ignoring 2+ consecutive digits (which
+    groups queries on date/number-suffixed tables together)
+* Fingerprinting:
+  - Ignore `NOTIFY` payloads, similar to channel names [#353](https://github.com/pganalyze/libpg_query/pull/353)
+  - Ignore role names (e.g. in `CREATE ROLE`, `DROP ROLE`, `GRANT` and `ALTER ... RENAME`) [#357](https://github.com/pganalyze/libpg_query/pull/357)
+  - Include `BEGIN`/`START TRANSACTION` options (e.g. read-only, isolation level) [#358](https://github.com/pganalyze/libpg_query/pull/358)
+  - Apply the depth cutoff when recursing into specific node types
+    - This changes fingerprints for set operation chains deeper than 100 levels,
+      which are now cut off consistently like other deeply nested nodes
+* Deparser:
+  - Rework when parentheses are added based on operator precedence [#371](https://github.com/pganalyze/libpg_query/pull/371)
+    - This fixes cases where the deparsed SQL changed meaning or was invalid, and
+      avoids adding unnecessary parentheses in others
+  - Use `strlcpy` instead of `strncpy` in deparser and summary truncate [#345](https://github.com/pganalyze/libpg_query/pull/345)
+* pg_query_normalize:
+  - Add support for `NOTIFY` statements [#340](https://github.com/pganalyze/libpg_query/pull/340)
+  - Avoid undefined behaviour for overly large parameter references [#346](https://github.com/pganalyze/libpg_query/pull/346)
+  - Fix handling of `U&` special constants in DefElem nodes [#347](https://github.com/pganalyze/libpg_query/pull/347)
+  - Don't swallow stack depth errors and return a partially normalized query
+* pg_query_summary:
+  - Fix a relation going missing when a CTE shares its name [#367](https://github.com/pganalyze/libpg_query/pull/367)
+  - Fix memory leak when the tree walk throws an error
+* Use built-in `strlcpy`/`strlcat` on older glibc versions [#339](https://github.com/pganalyze/libpg_query/pull/339)
+* Add new OSS-Fuzz fuzzer targets for Protobuf processing and PL/pgSQL parsing [#341](https://github.com/pganalyze/libpg_query/pull/341) [#343](https://github.com/pganalyze/libpg_query/pull/343)
+
 ## 18.0.0     2026-05-20
 
 * Upgrade to Postgres 18
@@ -22,6 +104,40 @@ All versions are tagged by the major Postgres version, plus a minor/patch versio
     pointer values, that would be retained in such a copy).
 * Fingerprinting: Rework alias/schema name handling to match Postgres 18
 
+## 17-6.2.4   2026-09-30
+
+* Security fix: Heap out-of-bounds write and read in pg_query_normalize ([GHSA-6ggm-xmc9-8ffg](https://github.com/pganalyze/libpg_query/security/advisories/GHSA-6ggm-xmc9-8ffg))
+  - When normalizing certain utility statements (e.g. `DO ... LANGUAGE`, statements with
+    string options, or `CREATE/ALTER SUBSCRIPTION ... CONNECTION`), pg_query_normalize
+    searched the query text for the location of string constants, which could yield
+    wrong locations for crafted input. This could cause out-of-bounds writes and reads
+    on the heap, leaking process memory in the normalized output or crashing the process.
+  - Constant locations are now recorded by the parser instead, and the normalizer checks
+    at runtime that constant locations never overlap
+  - This adds new location fields to the parse tree output (`DefElem.arg_location`,
+    `NotifyStmt.payload_location`, `CreateSubscriptionStmt.conninfo_location` and
+    `AlterSubscriptionStmt.conninfo_location`). Like other location fields, these are
+    ignored for fingerprinting.
+  - Applications that normalize untrusted query text should upgrade
+  - Reported by Paul Gerste (Cure53)
+* Deparser: Add strict checking for unexpected pointer values
+  - This ensures that a bad input parse tree doesn't cause the deparser to crash, and
+    instead returns an error
+  - Use cases that do not work with user input can define `PG_QUERY_DEPARSE_NO_STRICT_CHECKS`
+    to turn off the most detailed checks, for slightly better performance
+  - Reported by Paul Gerste (Cure53)
+* pg_query_normalize:
+  - Add support for `NOTIFY` statements [#340](https://github.com/pganalyze/libpg_query/pull/340)
+  - Avoid undefined behaviour for overly large parameter references [#346](https://github.com/pganalyze/libpg_query/pull/346)
+
+## 17-6.2.3   2026-08-24
+
+* Deparser:
+  - Preserve parentheses around subscripted array constructors
+    - This prevents `(ARRAY[...])[...]` from being deparsed as invalid SQL
+  - Fix handling of constraint key named `value` in `ALTER TABLE` [#329](https://github.com/pganalyze/libpg_query/pull/329)
+* Preserve `ARFLAGS` when overriding `AR` [#328](https://github.com/pganalyze/libpg_query/pull/328)
+
 ## 17-6.2.2   2026-01-26
 
 * pg_query_normalize: Fix handling of special strings in DefElem [#325](https://github.com/pganalyze/libpg_query/pull/325)

Makefile

@@ -7,12 +7,13 @@ PGDIR = $(root_dir)/tmp/postgres
 PGDIRBZ2 = $(root_dir)/tmp/postgres.tar.bz2
 PGDIRZIP = $(root_dir)/tmp/postgres.zip
 
-PG_VERSION = 18.4
+PG_VERSION = 18.6
 PG_VERSION_MAJOR = $(call word-dot,$(PG_VERSION),1)
-PG_VERSION_NUM = 180004
+PG_VERSION_NUM = 180006
 PROTOC_VERSION = 25.1
+UPB_PROTOC_VERSION := $(patsubst v%,%,$(shell head -n1 vendor/upb/VERSION 2>/dev/null))
 
-VERSION = 18.0.0
+VERSION = 18.1.0
 VERSION_MAJOR = $(call word-dot,$(VERSION),1)
 VERSION_MINOR = $(call word-dot,$(VERSION),2)
 VERSION_PATCH = $(call word-dot,$(VERSION),3)
@@ -31,10 +32,14 @@ else
 	SOFLAG = -soname
 endif
 
-SRC_FILES := $(wildcard src/*.c src/postgres/*.c) vendor/protobuf-c/protobuf-c.c vendor/xxhash/xxhash.c protobuf/pg_query.pb-c.c
+UPB_DIR := vendor/upb
+UPB_INCLUDES := -I./$(UPB_DIR) -I./$(UPB_DIR)/third_party/utf8_range
+UPB_SRC_FILES := $(UPB_DIR)/upb.c $(UPB_DIR)/third_party/utf8_range/utf8_range.c protobuf/pg_query.upb_minitable.c
+
+SRC_FILES := $(wildcard src/*.c src/postgres/*.c) vendor/xxhash/xxhash.c $(UPB_SRC_FILES)
 OBJ_FILES := $(SRC_FILES:.c=.o)
 
-override CFLAGS += -g -I. -I./vendor -I./src/include -I./src/postgres/include -Wall -Wno-unused-function -Wno-unused-value -Wno-unused-variable -fno-strict-aliasing -fwrapv -fPIC
+override CFLAGS += -g -I. -I./vendor $(UPB_INCLUDES) -I./src/include -I./src/postgres/include -Wall -Wno-unused-function -Wno-unused-value -Wno-unused-variable -fno-strict-aliasing -fwrapv -fPIC
 
 ifeq ($(OS),Windows_NT)
 override CFLAGS += -I./src/postgres/include/port/win32
@@ -43,7 +48,7 @@ endif
 
 override PG_CONFIGURE_FLAGS += -q --without-readline --without-zlib --without-icu
 
-override TEST_CFLAGS += -g -I. -I./vendor -Wall
+override TEST_CFLAGS += -g -I. -I./vendor $(UPB_INCLUDES) -Wall
 override TEST_LDFLAGS += -pthread
 
 CFLAGS_OPT_LEVEL = -O3
@@ -109,11 +114,11 @@ build: $(ARLIB)
 build_shared: $(SOLIB)
 
 clean:
-	-@ $(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) $(EXAMPLES) $(TESTS)
-	-@ $(RM) -rf {test,examples}/*.dSYM
+	-@ $(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) $(EXAMPLES) $(TESTS) $(BENCHMARKS)
+	-@ $(RM) -rf {test,examples,benchmark}/*.dSYM
 	-@ $(RM) -r $(PGDIR) $(PGDIRBZ2) $(PGDIRZIP)
 
-.PHONY: all clean build build_shared extract_source examples test install
+.PHONY: all clean build build_shared extract_source examples test benchmark install
 
 $(PGDIR):
 	curl -o $(PGDIRBZ2) https://ftp.postgresql.org/pub/source/v$(PG_VERSION)/postgresql-$(PG_VERSION).tar.bz2
@@ -130,6 +135,9 @@ $(PGDIR):
 	cd $(PGDIR); patch -p1 < $(root_dir)/patches/10_avoid_namespace_hashtab_impl_gen.patch
 	cd $(PGDIR); patch -p1 < $(root_dir)/patches/11_ifndef_namedatalen.patch
 	cd $(PGDIR); patch -p1 < $(root_dir)/patches/12_remove_acl_check.patch
+	cd $(PGDIR); patch -p1 < $(root_dir)/patches/13_exprlocation_check_stack_depth.patch
+	cd $(PGDIR); patch -p1 < $(root_dir)/patches/14_avoid_quadratic_memory_dotted_names.patch
+	cd $(PGDIR); patch -p1 < $(root_dir)/patches/15_node_string_constant_locations.patch
 	cd $(PGDIR); ./configure $(PG_CONFIGURE_FLAGS)
 	cd $(PGDIR); make -C src/pl/plpgsql/src pl_gram.h plerrcodes.h pl_reserved_kwlist_d.h pl_unreserved_kwlist_d.h
 	cd $(PGDIR); make -C src/port pg_config_paths.h
@@ -194,14 +202,24 @@ $(ARLIB): $(OBJ_FILES) Makefile
 $(SOLIB): $(OBJ_FILES) Makefile
 	@$(CC) $(CFLAGS) -shared -Wl,$(SOFLAG),$(SONAME) $(LDFLAGS) -o $@ $(OBJ_FILES) $(LIBS)
 
-protobuf/pg_query.pb-c.c protobuf/pg_query.pb-c.h: protobuf/pg_query.proto
-ifneq ($(shell which protoc-gen-c), )
-	protoc --c_out=. protobuf/pg_query.proto
+# upb-generated message code + minitables (regenerated only when the upb protoc
+# plugins are installed; otherwise the committed files are used as-is). Must be
+# generated with a protoc/protoc-gen-upb matching vendor/upb/VERSION.
+#
+# protoc-gen-upb also emits an empty pg_query.upb.c (all accessors are inline
+# in the header), which we don't keep.
+protobuf/pg_query.upb.h protobuf/pg_query.upb_minitable.h protobuf/pg_query.upb_minitable.c: protobuf/pg_query.proto
+ifneq ($(shell which protoc-gen-upb), )
+ifneq ($(shell protoc --version 2>/dev/null | cut -f2 -d" "), $(UPB_PROTOC_VERSION))
+	$(error ERROR - upb codegen needs protoc $(UPB_PROTOC_VERSION) to match vendor/upb/VERSION (found "$(shell protoc --version 2>/dev/null | cut -f2 -d' ')"); run 'make -C vendor/upb update TAG=...' to change the pinned version)
+endif
+	protoc --upb_out=. --upb_minitable_out=. protobuf/pg_query.proto
+	rm -f protobuf/pg_query.upb.c
 else
-	@echo 'Warning: protoc-gen-c not found, skipping protocol buffer regeneration'
+	@echo 'Warning: protoc-gen-upb not found, skipping upb regeneration'
 endif
 
-src/pg_query_protobuf.c src/pg_query_scan.c: protobuf/pg_query.pb-c.h
+src/pg_query_scan.c: protobuf/pg_query.upb.h
 
 # Only used when USE_PROTOBUF_CPP is used (experimental for testing only)
 src/pg_query_outfuncs_protobuf_cpp.cc: protobuf/pg_query.pb.cc
@@ -211,6 +229,18 @@ ifneq ($(shell protoc --version 2>/dev/null | cut -f2 -d" "), $(PROTOC_VERSION))
 endif
 	protoc --cpp_out=. protobuf/pg_query.proto
 
+BENCHMARKS = benchmark/bench_protobuf benchmark/microbench_protobuf
+BENCHMARK_CFLAGS = $(TEST_CFLAGS) -O2 -I./src -I./src/include -I./src/postgres/include
+benchmark: $(BENCHMARKS)
+	benchmark/bench_protobuf
+	benchmark/microbench_protobuf
+
+benchmark/bench_protobuf: benchmark/bench_protobuf.c $(ARLIB)
+	$(CC) $(BENCHMARK_CFLAGS) -o $@ benchmark/bench_protobuf.c $(ARLIB) $(TEST_LDFLAGS)
+
+benchmark/microbench_protobuf: benchmark/microbench_protobuf.c $(ARLIB)
+	$(CC) $(BENCHMARK_CFLAGS) -o $@ benchmark/microbench_protobuf.c $(ARLIB) $(TEST_LDFLAGS)
+
 EXAMPLES = examples/simple examples/scan examples/normalize examples/simple_error examples/normalize_error examples/simple_plpgsql
 examples: $(EXAMPLES)
 	examples/simple
@@ -238,12 +268,16 @@ examples/normalize_error: examples/normalize_error.c $(ARLIB)
 examples/simple_plpgsql: examples/simple_plpgsql.c $(ARLIB)
 	$(CC) $(TEST_CFLAGS) -o $@ -g examples/simple_plpgsql.c $(ARLIB) $(TEST_LDFLAGS)
 
-TESTS = test/complex test/concurrency test/deparse test/fingerprint test/fingerprint_opts test/is_utility_stmt test/normalize test/normalize_utility test/parse test/parse_opts test/parse_protobuf test/parse_protobuf_opts test/parse_plpgsql test/scan test/split test/summary test/summary_truncate
+TESTS = test/complex test/concurrency test/deparse test/deparse_depth test/fingerprint test/fingerprint_opts test/is_utility_stmt test/normalize test/normalize_utility test/parse test/parse_opts test/parse_protobuf test/parse_protobuf_opts test/parse_plpgsql test/scan test/split test/stack_depth test/summary test/summary_truncate
 test: $(TESTS)
 ifeq ($(VALGRIND),1)
 	$(VALGRIND_MEMCHECK) test/complex || (cat test/valgrind.log && false)
 	$(VALGRIND_MEMCHECK) test/concurrency || (cat test/valgrind.log && false)
 	$(VALGRIND_MEMCHECK) test/deparse || (cat test/valgrind.log && false)
+ifneq ($(USE_PROTOBUF_CPP),1)
+	# Tests the upb encoder's depth limit, which the C++ backend doesn't have
+	$(VALGRIND_MEMCHECK) test/deparse_depth || (cat test/valgrind.log && false)
+endif
 	$(VALGRIND_MEMCHECK) test/fingerprint || (cat test/valgrind.log && false)
 	$(VALGRIND_MEMCHECK) test/fingerprint_opts || (cat test/valgrind.log && false)
 	$(VALGRIND_MEMCHECK) test/is_utility_stmt || (cat test/valgrind.log && false)
@@ -255,6 +289,7 @@ ifeq ($(VALGRIND),1)
 	$(VALGRIND_MEMCHECK) test/parse_protobuf_opts || (cat test/valgrind.log && false)
 	$(VALGRIND_MEMCHECK) test/scan || (cat test/valgrind.log && false)
 	$(VALGRIND_MEMCHECK) test/split || (cat test/valgrind.log && false)
+	$(VALGRIND_MEMCHECK) test/stack_depth || (cat test/valgrind.log && false)
 	$(VALGRIND_MEMCHECK) test/summary || (cat test/valgrind.log && false)
 	$(VALGRIND_MEMCHECK) test/summary_truncate || (cat test/valgrind.log && false)
 	# Output-based tests
@@ -264,6 +299,10 @@ else
 	test/complex
 	test/concurrency
 	test/deparse
+ifneq ($(USE_PROTOBUF_CPP),1)
+	# Tests the upb encoder's depth limit, which the C++ backend doesn't have
+	test/deparse_depth
+endif
 	test/fingerprint
 	test/fingerprint_opts
 	test/is_utility_stmt
@@ -275,6 +314,7 @@ else
 	test/parse_protobuf_opts
 	test/scan
 	test/split
+	test/stack_depth
 	test/summary
 	test/summary_truncate
 	# Output-based tests
@@ -292,6 +332,9 @@ test/concurrency: test/concurrency.c test/parse_tests.c $(ARLIB)
 test/deparse: test/deparse.c test/deparse_tests.c $(ARLIB)
 	$(CC) $(TEST_CFLAGS) -o $@ test/deparse.c $(ARLIB) $(TEST_LDFLAGS)
 
+test/deparse_depth: test/deparse_depth.c $(ARLIB)
+	$(CC) $(TEST_CFLAGS) -o $@ test/deparse_depth.c $(ARLIB) $(TEST_LDFLAGS)
+
 test/fingerprint: test/fingerprint.c test/fingerprint_tests.c $(ARLIB)
 	# We have "-Isrc/" because this test uses pg_query_fingerprint_with_opts
 	$(CC) $(TEST_CFLAGS) -o $@ -Isrc/ test/fingerprint.c $(ARLIB) $(TEST_LDFLAGS)
@@ -339,6 +382,9 @@ test/scan: test/scan.c test/scan_tests.c $(ARLIB)
 test/split: test/split.c test/split_tests.c $(ARLIB)
 	$(CC) $(TEST_CFLAGS) -o $@ test/split.c $(ARLIB) $(TEST_LDFLAGS)
 
+test/stack_depth: test/stack_depth.c $(ARLIB)
+	$(CC) $(TEST_CFLAGS) -o $@ test/stack_depth.c $(ARLIB) $(TEST_LDFLAGS)
+
 prefix = /usr/local
 libdir = $(prefix)/lib
 includedir = $(prefix)/include
@@ -352,4 +398,5 @@ install: $(ARLIB) $(SOLIB)
 	$(INSTALL) -d "$(DESTDIR)"$(includedir)/$(TARGET)
 	$(INSTALL) -m 644 pg_query.h "$(DESTDIR)"$(includedir)/pg_query.h
 	$(INSTALL) -m 644 postgres_deparse.h "$(DESTDIR)"$(includedir)/postgres_deparse.h
+	$(INSTALL) -m 644 pg_query_scan_tokens.h "$(DESTDIR)"$(includedir)/pg_query_scan_tokens.h
 	$(INSTALL) -m 644 protobuf/pg_query.proto "$(DESTDIR)"$(includedir)/$(TARGET)/pg_query.proto