From aea0d9a06b77fca1f329b45fc9b667db5c9df177 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sun, 1 Nov 2026 18:16:20 +0000 Subject: [PATCH 1/2] feat(database): estimate a collection's count from engine statistics An exact COUNT on Postgres reads the whole heap once scattered writes clear the visibility map: 3.5 GB and 30s of disk reads for a 3M-row table. The statistics collector already tracks live tuples per table, updated on every commit, so callers that only need a figure (usage sweeps) can read it in under a millisecond. Adapters without a usable estimate, and shared tables whose statistics span every tenant, return null so the caller counts. --- src/Database/Adapter.php | 11 +++++ src/Database/Adapter/Pool.php | 8 ++++ src/Database/Adapter/Postgres.php | 42 ++++++++++++++++ src/Database/Adapter/ReadWritePool.php | 1 + src/Database/Trait/Collections.php | 10 ++++ tests/e2e/Adapter/Scopes/CollectionTests.php | 30 ++++++++++++ .../Adapter/PostgresCollectionSizeTest.php | 48 ++++++++++++++++++- 7 files changed, 149 insertions(+), 1 deletion(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 7d6b0ff7c..f44429644 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -879,6 +879,17 @@ abstract public function getSizeOfCollection(string $collection): int; */ abstract public function getSizeOfCollectionOnDisk(string $collection): int; + /** + * Estimate the number of documents in a collection from engine statistics, without scanning it. + * Null when the engine keeps no usable estimate, so the caller counts instead. + * + * @throws DatabaseException + */ + public function estimateCount(string $collection): ?int + { + return null; + } + /** * Estimate maximum number of bytes required to store a document in $collection. * Byte requirement varies based on column type and size. diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 950ff23e2..2654813bf 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -904,6 +904,14 @@ public function count(Document $collection, array $queries = [], ?int $max = nul return $result; } + #[\Override] + public function estimateCount(string $collection): ?int + { + /** @var int|null $result */ + $result = $this->delegate(__FUNCTION__, \func_get_args()); + return $result; + } + #[\Override] public function getSizeOfCollection(string $collection): int { diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index c60e4c46b..566a61d60 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -397,6 +397,48 @@ public function getSizeOfCollectionOnDisk(string $collection): int return $size; } + /** + * Live tuples from the statistics collector, which every commit updates. Standbys and a + * statistics reset leave it at zero, so reltuples from the last vacuum or analyze stands in. + * Shared tables hold every tenant's rows, so they have no per-tenant estimate. + * + * @throws DatabaseException + */ + #[\Override] + public function estimateCount(string $collection): ?int + { + if ($this->sharedTables) { + return null; + } + + $name = $this->getTable($this->filter($collection)); + + $result = $this->dialectBuilder() + ->fromNone() + ->selectRaw( + 'COALESCE(NULLIF(stats.n_live_tup, 0), class.reltuples::bigint) FROM pg_class AS class + LEFT JOIN pg_stat_user_tables AS stats ON stats.relid = class.oid + WHERE class.oid = to_regclass(?)', + [$name] + ) + ->build(); + + $statement = $this->executeResult($result, Event::CollectionRead); + + try { + $this->execute($statement); + $estimate = $statement->fetchColumn(); + } catch (PDOException $e) { + throw new DatabaseException('Failed to estimate collection count: '.$e->getMessage()); + } + + if (! \is_numeric($estimate) || $estimate < 0) { + return null; + } + + return (int) $estimate; + } + /** * @throws DatabaseException */ diff --git a/src/Database/Adapter/ReadWritePool.php b/src/Database/Adapter/ReadWritePool.php index 28871425f..8ec9f5477 100644 --- a/src/Database/Adapter/ReadWritePool.php +++ b/src/Database/Adapter/ReadWritePool.php @@ -22,6 +22,7 @@ class ReadWritePool extends Pool 'getSchemaIndexes', 'getSizeOfCollection', 'getSizeOfCollectionOnDisk', + 'estimateCount', 'ping', 'id', ]; diff --git a/src/Database/Trait/Collections.php b/src/Database/Trait/Collections.php index 0054f7147..7e642b843 100644 --- a/src/Database/Trait/Collections.php +++ b/src/Database/Trait/Collections.php @@ -276,6 +276,16 @@ public function getSizeOfCollection(string $collection): int return $this->adapter->getSizeOfCollection($definition->getId()); } + /** + * @throws Exception + */ + public function estimateCount(string $collection): ?int + { + $definition = $this->silent(fn () => $this->getOwnCollection($collection)); + + return $this->adapter->estimateCount($definition->getId()); + } + /** * @throws DatabaseException * @throws NotFoundException diff --git a/tests/e2e/Adapter/Scopes/CollectionTests.php b/tests/e2e/Adapter/Scopes/CollectionTests.php index 5dc87d1b5..e6dd7fcf8 100644 --- a/tests/e2e/Adapter/Scopes/CollectionTests.php +++ b/tests/e2e/Adapter/Scopes/CollectionTests.php @@ -267,6 +267,36 @@ public function testSizeCollection(): void $this->assertLessThan($size2, $size3); } + public function testEstimateCount(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + $database->createCollection(Collection::create(id: 'estimateTest')); + $database->createAttribute('estimateTest', Attribute::string(key: 'name', size: 255, required: true)); + + $database->getAuthorization()->skip(function () use ($database) { + for ($i = 0; $i < 10; $i++) { + $database->createDocument('estimateTest', new Document([ + '$id' => 'doc'.$i, + 'name' => 'name'.$i, + ])); + } + }); + + $database->analyzeCollection('estimateTest'); + + $estimate = $database->estimateCount('estimateTest'); + + if (! $this->engineIs(Postgres::class) || $database->getAdapter()->hasSharedTables()) { + $this->assertNull($estimate); + + return; + } + + $this->assertSame(10, $estimate); + } + public function testSizeCollectionOnDisk(): void { $this->getDatabase()->createCollection(Collection::create(id: 'sizeTestDisk1')); diff --git a/tests/unit/Adapter/PostgresCollectionSizeTest.php b/tests/unit/Adapter/PostgresCollectionSizeTest.php index 1f018c019..a46ec692b 100644 --- a/tests/unit/Adapter/PostgresCollectionSizeTest.php +++ b/tests/unit/Adapter/PostgresCollectionSizeTest.php @@ -52,6 +52,52 @@ public function testAFailedSizeReadIsADatabaseError(string $method, string $func } } + public function testTheEstimateReadsTableStatistics(): void + { + $adapter = $this->adapter(['3000000']); + + $this->assertSame(3000000, $adapter->estimateCount('books')); + $this->assertCount(1, $this->statements); + $this->assertStringContainsString('n_live_tup', $this->statements[0]); + $this->assertStringContainsString('reltuples', $this->statements[0]); + } + + /** + * @return iterable + */ + public static function missingEstimates(): iterable + { + yield 'never analyzed' => ['-1']; + yield 'missing table' => [false]; + } + + #[DataProvider('missingEstimates')] + public function testNoStatisticsIsNoEstimate(string|false $estimate): void + { + $adapter = $this->adapter([$estimate]); + + $this->assertNull($adapter->estimateCount('books')); + } + + public function testSharedTablesHaveNoEstimate(): void + { + $adapter = $this->adapter(['3000000']); + $adapter->setSharedTables(true); + + $this->assertNull($adapter->estimateCount('books')); + $this->assertSame([], $this->statements); + } + + public function testAFailedEstimateIsADatabaseError(): void + { + $adapter = $this->adapter(['3000000'], new PDOException('SQLSTATE[08006]: connection failure')); + + $this->expectException(DatabaseException::class); + $this->expectExceptionMessage('Failed to estimate collection count: SQLSTATE[08006]: connection failure'); + + $adapter->estimateCount('books'); + } + private function readSize(Postgres $adapter, string $method): int { return $method === 'getSizeOfCollectionOnDisk' @@ -60,7 +106,7 @@ private function readSize(Postgres $adapter, string $method): int } /** - * @param list $sizes + * @param list $sizes */ private function adapter(array $sizes, ?PDOException $failure = null): Postgres { From 91a3363e2f6120f3d8aa26eaee31fc5d4898e7d6 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sun, 1 Nov 2026 18:48:39 +0000 Subject: [PATCH 2/2] fix(database): keep zero live tuples as the estimate NULLIF treated a primary's zero n_live_tup as missing statistics, so a table emptied since its last ANALYZE reported the stale reltuples. Only a standby, which collects no statistics, now falls back to reltuples. Statement preparation also moves inside the try, and the PDOException is kept as the previous exception so retry classification still sees it. --- src/Database/Adapter/Postgres.php | 12 ++++----- tests/e2e/Adapter/Scopes/CollectionTests.php | 25 ++++++++++++++----- .../Adapter/PostgresCollectionSizeTest.php | 11 +++++--- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 566a61d60..162cfdb1f 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -398,8 +398,8 @@ public function getSizeOfCollectionOnDisk(string $collection): int } /** - * Live tuples from the statistics collector, which every commit updates. Standbys and a - * statistics reset leave it at zero, so reltuples from the last vacuum or analyze stands in. + * Live tuples from the statistics collector, which every commit on the primary updates. + * A standby collects none, so it reads reltuples from the last vacuum or analyze instead. * Shared tables hold every tenant's rows, so they have no per-tenant estimate. * * @throws DatabaseException @@ -416,20 +416,20 @@ public function estimateCount(string $collection): ?int $result = $this->dialectBuilder() ->fromNone() ->selectRaw( - 'COALESCE(NULLIF(stats.n_live_tup, 0), class.reltuples::bigint) FROM pg_class AS class + 'CASE WHEN pg_is_in_recovery() THEN class.reltuples::bigint + ELSE COALESCE(stats.n_live_tup, class.reltuples::bigint) END FROM pg_class AS class LEFT JOIN pg_stat_user_tables AS stats ON stats.relid = class.oid WHERE class.oid = to_regclass(?)', [$name] ) ->build(); - $statement = $this->executeResult($result, Event::CollectionRead); - try { + $statement = $this->executeResult($result, Event::CollectionRead); $this->execute($statement); $estimate = $statement->fetchColumn(); } catch (PDOException $e) { - throw new DatabaseException('Failed to estimate collection count: '.$e->getMessage()); + throw new DatabaseException('Failed to estimate collection count: '.$e->getMessage(), $e->getCode(), $e); } if (! \is_numeric($estimate) || $estimate < 0) { diff --git a/tests/e2e/Adapter/Scopes/CollectionTests.php b/tests/e2e/Adapter/Scopes/CollectionTests.php index e6dd7fcf8..d27cac02a 100644 --- a/tests/e2e/Adapter/Scopes/CollectionTests.php +++ b/tests/e2e/Adapter/Scopes/CollectionTests.php @@ -284,17 +284,30 @@ public function testEstimateCount(): void } }); - $database->analyzeCollection('estimateTest'); - - $estimate = $database->estimateCount('estimateTest'); - if (! $this->engineIs(Postgres::class) || $database->getAdapter()->hasSharedTables()) { - $this->assertNull($estimate); + $this->assertNull($database->estimateCount('estimateTest')); return; } - $this->assertSame(10, $estimate); + // Postgres publishes a backend's pending statistics at most once a second, when a transaction ends, + // and adds any still pending on top of what ANALYZE records. + sleep(2); + $database->count('estimateTest'); + $database->analyzeCollection('estimateTest'); + + $this->assertSame(10, $database->estimateCount('estimateTest')); + + $database->getAuthorization()->skip(function () use ($database) { + for ($i = 0; $i < 10; $i++) { + $database->deleteDocument('estimateTest', 'doc'.$i); + } + }); + + sleep(2); + $database->count('estimateTest'); + + $this->assertSame(0, $database->estimateCount('estimateTest')); } public function testSizeCollectionOnDisk(): void diff --git a/tests/unit/Adapter/PostgresCollectionSizeTest.php b/tests/unit/Adapter/PostgresCollectionSizeTest.php index a46ec692b..19dd64dd2 100644 --- a/tests/unit/Adapter/PostgresCollectionSizeTest.php +++ b/tests/unit/Adapter/PostgresCollectionSizeTest.php @@ -92,10 +92,13 @@ public function testAFailedEstimateIsADatabaseError(): void { $adapter = $this->adapter(['3000000'], new PDOException('SQLSTATE[08006]: connection failure')); - $this->expectException(DatabaseException::class); - $this->expectExceptionMessage('Failed to estimate collection count: SQLSTATE[08006]: connection failure'); - - $adapter->estimateCount('books'); + try { + $adapter->estimateCount('books'); + $this->fail('An estimate that fails must reach the caller'); + } catch (DatabaseException $error) { + $this->assertSame('Failed to estimate collection count: SQLSTATE[08006]: connection failure', $error->getMessage()); + $this->assertInstanceOf(PDOException::class, $error->getPrevious()); + } } private function readSize(Postgres $adapter, string $method): int