From e2e22a0c8d759c2862d40481fd0d7684fe7cc8c4 Mon Sep 17 00:00:00 2001 From: Avtandil Kikabidze Date: Wed, 11 Nov 2026 21:51:14 +0400 Subject: [PATCH 01/11] Add schema diff and database preparation scripts --- composer.json | 9 +- tests/Unit/Utils/SchemaDiffTest.php | 113 ++++++++++++ utils/SchemaDiff.php | 265 ++++++++++++++++++++++++++++ utils/db-prepare.sh | 147 +++++++++++++++ utils/db-schema-diff-accepted.txt | 18 ++ utils/db-schema-diff.php | 183 +++++++++++++++++++ 6 files changed, 734 insertions(+), 1 deletion(-) create mode 100644 tests/Unit/Utils/SchemaDiffTest.php create mode 100644 utils/SchemaDiff.php create mode 100755 utils/db-prepare.sh create mode 100644 utils/db-schema-diff-accepted.txt create mode 100644 utils/db-schema-diff.php diff --git a/composer.json b/composer.json index 9f803201..a580a4ed 100644 --- a/composer.json +++ b/composer.json @@ -50,7 +50,8 @@ }, "autoload-dev": { "psr-4": { - "Longman\\TelegramBot\\Tests\\Unit\\": "tests/Unit" + "Longman\\TelegramBot\\Tests\\Unit\\": "tests/Unit", + "Longman\\TelegramBot\\Utils\\": "utils" } }, "scripts": { @@ -65,6 +66,12 @@ ], "test-cov-upload": [ "@php ocular.phar code-coverage:upload --format=php-clover clover.xml" + ], + "db-prepare": [ + "bash utils/db-prepare.sh" + ], + "schema-diff": [ + "@php utils/db-schema-diff.php telegrambot telegrambot_migrations" ] }, "config": { diff --git a/tests/Unit/Utils/SchemaDiffTest.php b/tests/Unit/Utils/SchemaDiffTest.php new file mode 100644 index 00000000..87022dc9 --- /dev/null +++ b/tests/Unit/Utils/SchemaDiffTest.php @@ -0,0 +1,113 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Longman\TelegramBot\Tests\Unit\Utils; + +use InvalidArgumentException; +use Longman\TelegramBot\Tests\Unit\TestCase; +use Longman\TelegramBot\Utils\SchemaDiff; + +/** + * @link https://github.com/php-telegram-bot/core + * @author Avtandil Kikabidze + * @copyright Avtandil Kikabidze + * @license http://opensource.org/licenses/mit-license.php The MIT License (MIT) + * @package TelegramTest + */ +class SchemaDiffTest extends TestCase +{ + public function testIdenticalSnapshotsHaveNoDifferences(): void + { + self::assertSame([], SchemaDiff::compare($this->snapshot(), $this->snapshot())); + } + + public function testTableOnOneSideYieldsOnlyTheTableKey(): void + { + $migrated = $this->snapshot(); + $migrated['botan_shortener'] = ['columns' => ['id' => 'bigint|NO||'], 'indexes' => ['1:id'], 'foreign_keys' => []]; + + self::assertSame(['table botan_shortener'], array_keys(SchemaDiff::compare($this->snapshot(), $migrated))); + } + + public function testColumnIndexAndForeignKeyDifferences(): void + { + $migrated = $this->snapshot(); + $migrated['user']['columns']['username'] = 'char(255)|YES||utf8mb4|utf8mb4_unicode_520_ci'; + $migrated['message']['columns']['new_chat_member'] = 'bigint|YES|||'; + $migrated['message']['indexes'][] = '0:new_chat_member'; + unset($migrated['message']['foreign_keys'][0]); + + self::assertSame( + ['column message.new_chat_member', 'column user.username', 'fk message(chat_id)', 'index message(new_chat_member)'], + array_keys(SchemaDiff::compare($this->snapshot(), $migrated)) + ); + } + + public function testUniquenessRepeatsAndReferencesAreComparedPerColumns(): void + { + $migrated = $this->snapshot(); + $migrated['user']['indexes'] = ['1:id', '1:username']; + $migrated['message']['indexes'][] = '0:user_id'; + $migrated['message']['foreign_keys'][1] = 'user_id>chat(id)'; + + self::assertSame( + [ + 'fk message(user_id)' => 'fresh user(id); migrated chat(id)', + 'index message(user_id)' => 'fresh non-unique; migrated non-unique + non-unique', + 'index user(username)' => 'fresh non-unique; migrated unique', + ], + SchemaDiff::compare($this->snapshot(), $migrated) + ); + } + + public function testEmptySnapshotIsAnError(): void + { + $this->expectException(InvalidArgumentException::class); + SchemaDiff::compare($this->snapshot(), []); + } + + public function testFilterSeparatesUnexpectedAndStale(): void + { + $accepted = SchemaDiff::parseAccepted("# unused, kept on purpose\ntable botan_shortener\n\ncolumn user.username\n"); + $result = SchemaDiff::filter(['table botan_shortener' => 'x', 'column chat.title' => 'y'], $accepted); + + self::assertSame(['column chat.title' => 'y'], $result['unexpected']); + self::assertSame(['column user.username'], $result['stale']); + } + + /** + * Two-table snapshot in the shape the CLI reads from information_schema. + * + * @return array, indexes: array, foreign_keys: array}> + */ + private function snapshot(): array + { + return [ + 'user' => [ + 'columns' => [ + 'id' => 'bigint|NO|||', + 'username' => 'char(191)|YES||utf8mb4|utf8mb4_unicode_520_ci', + ], + 'indexes' => ['1:id', '0:username'], + 'foreign_keys' => [], + ], + 'message' => [ + 'columns' => [ + 'chat_id' => 'bigint|NO|||', + 'id' => 'bigint unsigned|NO|||', + 'user_id' => 'bigint|YES|||', + ], + 'indexes' => ['1:chat_id,id', '0:user_id'], + 'foreign_keys' => ['chat_id>chat(id)', 'user_id>user(id)'], + ], + ]; + } +} diff --git a/utils/SchemaDiff.php b/utils/SchemaDiff.php new file mode 100644 index 00000000..ca230aeb --- /dev/null +++ b/utils/SchemaDiff.php @@ -0,0 +1,265 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Longman\TelegramBot\Utils; + +use InvalidArgumentException; + +/** + * Compare two database schema snapshots, a fresh one and a migrated one. + * + * A snapshot maps each table name to its columns, indexes and foreign keys: + * + * 'user' => [ + * 'columns' => ['username' => 'char(191)|YES|NULL|utf8mb4|utf8mb4_unicode_520_ci'], + * 'indexes' => ['1:id', '0:username'], + * 'foreign_keys' => ['chat_id>chat(id)'], + * ] + * + * A column is "||||", an index is + * ":" and a foreign key is ">()". + * Index and foreign key names are not part of a snapshot, so generated names never differ. + * + * Each difference is keyed "table ", "column .", "index ()" or + * "fk ()", the keys listed in the accepted differences file. + */ +class SchemaDiff +{ + /** + * Compare two snapshots and return every difference, sorted by key. + * + * A table present on one side only yields its "table" key and nothing for its columns. + * Indexes and foreign keys are grouped by their columns: two indexes over the same + * columns are compared as a list with repeats, and a uniqueness change or a changed + * reference is reported under the key of those columns. + * + * @param array, indexes: array, foreign_keys: array}> $fresh + * @param array, indexes: array, foreign_keys: array}> $migrated + * + * @return array Difference key to human-readable detail + * @throws InvalidArgumentException When either snapshot has no tables + */ + public static function compare(array $fresh, array $migrated): array + { + if ($fresh === []) { + throw new InvalidArgumentException('The fresh snapshot has no tables.'); + } + + if ($migrated === []) { + throw new InvalidArgumentException('The migrated snapshot has no tables.'); + } + + $differences = []; + + foreach (array_diff_key($fresh, $migrated) as $table => $definition) { + $differences['table ' . $table] = 'fresh only'; + } + + foreach (array_diff_key($migrated, $fresh) as $table => $definition) { + $differences['table ' . $table] = 'migrated only'; + } + + foreach (array_intersect_key($fresh, $migrated) as $table => $definition) { + $differences += self::compareColumns($table, $definition['columns'], $migrated[$table]['columns']); + $differences += self::compareGroups( + 'index ' . $table, + self::groupIndexes($definition['indexes']), + self::groupIndexes($migrated[$table]['indexes']) + ); + $differences += self::compareGroups( + 'fk ' . $table, + self::groupForeignKeys($definition['foreign_keys']), + self::groupForeignKeys($migrated[$table]['foreign_keys']) + ); + } + + ksort($differences, SORT_STRING); + + return $differences; + } + + /** + * Read the difference keys from the contents of the accepted differences file. + * + * Blank lines and lines starting with "#" are skipped; surrounding whitespace is trimmed. + * + * @param string $contents + * + * @return array + */ + public static function parseAccepted(string $contents): array + { + $keys = []; + + foreach (preg_split('/\R/', $contents) as $line) { + $line = trim($line); + + if ($line === '' || $line[0] === '#') { + continue; + } + + $keys[] = $line; + } + + return $keys; + } + + /** + * Split differences into the ones not accepted and the accepted keys that match no difference. + * + * @param array $differences + * @param array $accepted + * + * @return array{unexpected: array, stale: array} + */ + public static function filter(array $differences, array $accepted): array + { + $stale = array_filter($accepted, static function (string $key) use ($differences): bool { + return !array_key_exists($key, $differences); + }); + + return [ + 'unexpected' => array_diff_key($differences, array_flip($accepted)), + 'stale' => array_values($stale), + ]; + } + + /** + * Compare the columns of one table present on both sides. + * + * @param string $table + * @param array $fresh + * @param array $migrated + * + * @return array + */ + private static function compareColumns(string $table, array $fresh, array $migrated): array + { + $differences = []; + + foreach (array_keys($fresh + $migrated) as $column) { + $fresh_definition = $fresh[$column] ?? null; + $migrated_definition = $migrated[$column] ?? null; + + if ($fresh_definition === $migrated_definition) { + continue; + } + + $differences['column ' . $table . '.' . $column] = self::describe( + $fresh_definition === null ? [] : [$fresh_definition], + $migrated_definition === null ? [] : [$migrated_definition] + ); + } + + return $differences; + } + + /** + * Compare two groupings (columns to sorted list of values) and key each mismatch by its columns. + * + * @param string $prefix "index " or "fk " + * @param array> $fresh + * @param array> $migrated + * + * @return array + */ + private static function compareGroups(string $prefix, array $fresh, array $migrated): array + { + $differences = []; + + foreach (array_keys($fresh + $migrated) as $columns) { + $fresh_values = $fresh[$columns] ?? []; + $migrated_values = $migrated[$columns] ?? []; + + if ($fresh_values === $migrated_values) { + continue; + } + + $differences[$prefix . '(' . $columns . ')'] = self::describe($fresh_values, $migrated_values); + } + + return $differences; + } + + /** + * Group index entries by their columns, each group a sorted list of "unique" and "non-unique". + * + * @param array $indexes + * + * @return array> + */ + private static function groupIndexes(array $indexes): array + { + $groups = []; + + foreach ($indexes as $index) { + [$unique, $columns] = explode(':', $index, 2); + + $groups[$columns][] = $unique === '1' ? 'unique' : 'non-unique'; + } + + return self::sortGroups($groups); + } + + /** + * Group foreign key entries by their columns, each group a sorted list of references. + * + * @param array $foreign_keys + * + * @return array> + */ + private static function groupForeignKeys(array $foreign_keys): array + { + $groups = []; + + foreach ($foreign_keys as $foreign_key) { + [$columns, $reference] = explode('>', $foreign_key, 2); + + $groups[$columns][] = $reference; + } + + return self::sortGroups($groups); + } + + /** + * Sort the values inside each group, so the order the server listed them in does not matter. + * + * @param array> $groups + * + * @return array> + */ + private static function sortGroups(array $groups): array + { + foreach ($groups as $columns => $values) { + sort($values, SORT_STRING); + + $groups[$columns] = $values; + } + + return $groups; + } + + /** + * Describe both sides of a difference, "absent" for a side that has nothing. + * + * @param array $fresh + * @param array $migrated + * + * @return string + */ + private static function describe(array $fresh, array $migrated): string + { + $fresh_text = $fresh === [] ? 'absent' : implode(' + ', $fresh); + $migrated_text = $migrated === [] ? 'absent' : implode(' + ', $migrated); + + return 'fresh ' . $fresh_text . '; migrated ' . $migrated_text; + } +} diff --git a/utils/db-prepare.sh b/utils/db-prepare.sh new file mode 100755 index 00000000..c2764bdb --- /dev/null +++ b/utils/db-prepare.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# +# This file is part of the TelegramBot package. +# +# (c) Avtandil Kikabidze aka LONGMAN +# +# For the full copyright and license information, please view the LICENSE +# file that was distributed with this source code. +# +# Create the two databases the schema diff compares, on one server: +# telegrambot from the current structure.sql; +# telegrambot_migrations from the 0.44.1 structure.sql, then every file in +# utils/db-schema-update in ls order. +# Both are dropped and created again, so the script can run repeatedly on one +# server. The 0.44.1 schema is read through git, fetching the tag from origin +# when it is missing; the working tree is never touched. +# +# Connection over TCP from DB_HOST (default 127.0.0.1), DB_PORT (3306), +# DB_USER (root) and DB_PASSWORD; the client is DB_CLIENT (mysql). Waits up to +# 120 s for the server. Stops at the first failing file and names it on +# standard error. + +set -euo pipefail + +# Sort the migration files byte by byte, whatever the caller's locale. +export LC_ALL=C + +DB_HOST="${DB_HOST:-127.0.0.1}" +DB_PORT="${DB_PORT:-3306}" +DB_USER="${DB_USER:-root}" +DB_PASSWORD="${DB_PASSWORD:-}" +DB_CLIENT="${DB_CLIENT:-mysql}" + +baseline_tag=0.44.1 +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# Progress goes to standard error, redrawn in place, and only when it is a terminal. +progress() { + if [ -t 2 ]; then + printf '\r\033[K%s' "$1" >&2 + fi +} + +end_progress() { + if [ -t 2 ]; then + printf '\r\033[K' >&2 + fi +} + +fail() { + end_progress + echo "db-prepare: $1" >&2 + exit 1 +} + +# The password goes through an option file, so it shows neither in the process +# list nor in a client warning. --defaults-file keeps other option files out. +( + umask 077 + printf '[client]\npassword="%s"\n' "${DB_PASSWORD//\\/\\\\}" > "$work/client.cnf" +) + +db() { + "$DB_CLIENT" --defaults-file="$work/client.cnf" --protocol=TCP --host="$DB_HOST" --port="$DB_PORT" \ + --user="$DB_USER" --default-character-set=utf8mb4 "$@" +} + +# run_sql