diff --git a/CHANGELOG.md b/CHANGELOG.md index bc30d6f..7ed9f3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Latest * [#143](https://github.com/cleverage/ui-process-bundle/issues/143) Stop a running process execution from the UI: "Stop the process" action (POST, with confirmation) on the `started` executions, recording a stop request (`ProcessExecution::$stopRequestedAt`). The new `ProcessExecutionStopChecker` (a stop checker of cleverage/process-bundle#266, tag `cleverage_process.stop_checker`) reads it with DBAL before the task executions, at most every `process_execution.heartbeat_interval` seconds (new bundle configuration, default `5`): the process stops before its next task execution, with the new `stopped` status (`ProcessExecutionStatus::Stopped`, "Stop requested" displayed until then). New `stopped` notification status (`NotificationTrigger::Stopped`, not notified by default). Requires `cleverage/process-bundle` ^5.3. Add documentation and tests. * [#144](https://github.com/cleverage/ui-process-bundle/issues/144) Detect the process executions whose process has been killed: the `ProcessExecutionStopChecker` updates the heartbeat of the running execution (`ProcessExecution::$heartbeatAt`); a `started` execution without activity for more than `process_execution.stale_after` seconds (new bundle configuration, default `900`) is displayed as "Not responding", and can be marked as failed ("Mark as failed" action, POST, with confirmation; its end date is its last activity; its process is not stopped). It can still be stopped: a process blocked in a long task call stops when the call returns. New `ProcessExecutionCrudController` optional `$staleAfter` constructor argument. * [#143](https://github.com/cleverage/ui-process-bundle/issues/143) [#144](https://github.com/cleverage/ui-process-bundle/issues/144) `Version20261014120000` migration adding `process_execution.stop_requested_at` and `process_execution.heartbeat_at` (nullable). +* [#147](https://github.com/cleverage/ui-process-bundle/issues/147) Log retention: new `cleverage:ui-process:purge-logs` command (with `--dry-run`) deleting the log records (`log_record`, by batches) older than `logs.database_max_days` days and the log files (`/.log`) last modified more than `logs.file_max_days` days ago (new bundle configuration, default `null`: kept), both overridable by the new `logs` option of each process (`ProcessConfigurationsManager::getLogsOptions()`). The process executions are kept. Add documentation (including how to give the space back to the database after the first purge) and tests. v3.1.1 ------ diff --git a/config/services/command.yaml b/config/services/command.yaml index e8f5c9f..9339ca8 100644 --- a/config/services/command.yaml +++ b/config/services/command.yaml @@ -8,3 +8,13 @@ services: - '@validator' - '@security.user_password_hasher' - '@doctrine.orm.entity_manager' + + cleverage_ui_process.command.purge_logs: + class: CleverAge\UiProcessBundle\Command\PurgeLogsCommand + public: false + tags: + - { name: console.command } + arguments: + $entityManager: '@doctrine.orm.entity_manager' + $processConfigurationsManager: '@cleverage_ui_process.manager.process_configuration' + $logDirectory: '%kernel.logs_dir%' diff --git a/docs/index.md b/docs/index.md index f8cd1fc..6f19ab6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -140,6 +140,8 @@ clever_age_ui_process: database_level: Info # Debug on dev environment file_level: Info # Debug on dev environment report_increment_level: Warning + database_max_days: ~ # Log retention, see the cleverage:ui-process:purge-logs command + file_max_days: ~ design: logo_path: 'bundles/cleverageuiprocess/logo.jpg' ``` diff --git a/docs/reference/01-bundle_configuration.md b/docs/reference/01-bundle_configuration.md index aaea559..6978f81 100644 --- a/docs/reference/01-bundle_configuration.md +++ b/docs/reference/01-bundle_configuration.md @@ -16,6 +16,8 @@ clever_age_ui_process: database_level: Info # Debug when kernel.environment is "dev" file_level: Info # Debug when kernel.environment is "dev" report_increment_level: Warning + database_max_days: ~ + file_max_days: ~ design: logo_path: 'bundles/cleverageuiprocess/logo.jpg' notification: @@ -47,6 +49,10 @@ See [process executions & logs](04-process_executions_and_logs.md) for the whole | `database_level` | `string` | `Debug` on `dev` environment, else `Info` | Minimum Monolog level of the log records stored in database. | | `file_level` | `string` | `Debug` on `dev` environment, else `Info` | Minimum Monolog level of the log records written in the execution log file. | | `report_increment_level` | `string` | `Warning` | Minimum Monolog level from which a log record increments the execution report (one counter per level, e.g. `Warning: 3`, `Error: 1`). Only records passing `file_level` are counted. | +| `database_max_days` | `int`, `null` | `null` | Days after which the log records are deleted from database by the [`cleverage:ui-process:purge-logs`](07-console_commands.md#cleverageui-processpurge-logs) command. `null`: kept. Can be overridden by process. | +| `file_max_days` | `int`, `null` | `null` | Days after which the log files are deleted by the [`cleverage:ui-process:purge-logs`](07-console_commands.md#cleverageui-processpurge-logs) command. `null`: kept. Can be overridden by process. | + +See [log retention](04-process_executions_and_logs.md#log-retention). Levels are Monolog level names (case-insensitive): `Debug`, `Info`, `Notice`, `Warning`, `Error`, `Critical`, `Alert`, `Emergency`. diff --git a/docs/reference/02-process_ui_options.md b/docs/reference/02-process_ui_options.md index 82b766d..5421500 100644 --- a/docs/reference/02-process_ui_options.md +++ b/docs/reference/02-process_ui_options.md @@ -34,7 +34,8 @@ Every key is optional: a process without `options.ui` is displayed and launched keys, or invalid values, raise an options resolver error when the process list is displayed. The `notification` key, next to `ui`, configures the [notification](09-notifications.md) of the end of the process -executions. +executions. The `logs` key, next to `ui`, configures the [log retention](04-process_executions_and_logs.md#log-retention) +of the process. Options ------- diff --git a/docs/reference/04-process_executions_and_logs.md b/docs/reference/04-process_executions_and_logs.md index 979c665..6ff4eec 100644 --- a/docs/reference/04-process_executions_and_logs.md +++ b/docs/reference/04-process_executions_and_logs.md @@ -159,6 +159,71 @@ clever_age_ui_process: report_increment_level: Error ``` +### Log retention + +Neither the log records stored in database nor the log files are deleted by default. The +`cleverage:ui-process:purge-logs` command deletes those older than a retention, in days: + +| Option | Deletes | +|---------------------|-----------------------------------------------------------------------------------------------------------| +| `database_max_days` | The log records (`log_record` table) created more than `database_max_days` days ago. | +| `file_max_days` | The log files (`/.log` in `%kernel.logs_dir%`) last modified more than `file_max_days` days ago. | + +Both default to `null` (kept), in the [bundle configuration](01-bundle_configuration.md#logs), and can be overridden +by process under the `logs` key of its options (next to the [`ui` options](02-process_ui_options.md)). A key missing +(or `null`) in the process options is inherited from the bundle configuration; the executions of a process no longer +defined use the bundle configuration. + +```yaml +# config/packages/clever_age_ui_process.yaml +clever_age_ui_process: + logs: + database_max_days: 30 + file_max_days: 180 + +# A process whose logs are kept longer +clever_age_process: + configurations: + app.daily_import: + options: + logs: + database_max_days: 90 + file_max_days: 365 + tasks: + # ... +``` + +Schedule the command, e.g. every night with a cron: + +``` +0 3 * * * cd /path/to/project && bin/console cleverage:ui-process:purge-logs --no-debug +``` + +Run it with `--dry-run` first to count what would be deleted. + +Keep in mind that: +- The process executions are never modified nor deleted: the executions list, their statuses and reports stay + complete. Once its logs are purged, the "Show logs stored in database" and "Download log file" actions of an + execution are no longer displayed. +- The log file of an execution contains its log records (the `message` is not truncated), as long as `file_level` is + not higher than `database_level`: keep the files longer than the database logs to keep the logs readable. +- Only the log files of the bundle are deleted: files named `.log`, in the directory of a process code. +- The log records are deleted by batches of 1000. + +#### Database space + +Deleting rows does not shrink the `log_record` table on disk: the database reuses the freed space for the new log +records, so that with regular purges the table stops growing. To give the space back after the first purge, rebuild +the table during a maintenance window (no process running: the table is locked while it is rebuilt): + +| Database | Command | +|---------------------|-------------------------------| +| PostgreSQL | `VACUUM FULL log_record;` | +| MySQL / MariaDB | `OPTIMIZE TABLE log_record;` | +| SQLite | `VACUUM;` (whole database) | + +The command does not run them: they would block the running processes, which write their logs in this table. + ### Logs list The "Process > Logs" menu lists the database logs (250 per page), with their level, message, date and a "Has context diff --git a/docs/reference/07-console_commands.md b/docs/reference/07-console_commands.md index 3c18af8..a5058d8 100644 --- a/docs/reference/07-console_commands.md +++ b/docs/reference/07-console_commands.md @@ -26,6 +26,24 @@ bin/console cleverage:ui-process:user-create admin@example.com 'my-secret-passwo Values passed as arguments are not validated. The email must be unique. +cleverage:ui-process:purge-logs +------------------------------- + +Deletes the log records stored in database and the log files older than the configured retention +(`logs.database_max_days` and `logs.file_max_days`, overridable by process). The process executions are kept. See +[log retention](04-process_executions_and_logs.md#log-retention). + +```bash +bin/console cleverage:ui-process:purge-logs [--dry-run] +``` + +| Option | Description | +|-------------|-------------------------------------------------------------------------------| +| `--dry-run` | Counts the log records and the log files to delete, without deleting them. | + +It displays, for each process with a retention, the retention and the number of log records and log files deleted. +Without any retention configured, it does nothing. + Related commands ---------------- diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 1b92beb..eb9b1e3 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -45,6 +45,13 @@ When the `store_in_database` option is enabled with a low `database_level`, the See [process executions & logs](reference/04-process_executions_and_logs.md). +## The log_record table keeps growing + +Configure a log retention (`logs.database_max_days`) and schedule the `cleverage:ui-process:purge-logs` command. After +the first purge, the space of the deleted rows is reused by the new ones, but the table does not shrink on disk. + +See [log retention](reference/04-process_executions_and_logs.md#log-retention). + ## {"message":"Missing auth token."} response when launching a process via HTTP request If you use the Apache web server with PHP-FPM/FastCGI, the `Authorization` header may not be passed to PHP. Add (or diff --git a/src/Command/PurgeLogsCommand.php b/src/Command/PurgeLogsCommand.php new file mode 100644 index 0000000..9a4b780 --- /dev/null +++ b/src/Command/PurgeLogsCommand.php @@ -0,0 +1,179 @@ +.log. + */ + private const LOG_FILE_PATTERN = '/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.log$/'; + + public function __construct( + private readonly EntityManagerInterface $entityManager, + private readonly ProcessConfigurationsManager $processConfigurationsManager, + private readonly string $logDirectory, + private readonly ?int $databaseMaxDays = null, + private readonly ?int $fileMaxDays = null, + ) { + parent::__construct(); + } + + protected function configure(): void + { + $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Count the log records and the log files to delete, without deleting them'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $style = new SymfonyStyle($input, $output); + $dryRun = (bool) $input->getOption('dry-run'); + $connection = $this->entityManager->getConnection(); + + $rows = []; + $totalRecords = 0; + $totalFiles = 0; + foreach ($this->getProcessCodes($connection) as $code) { + $options = $this->processConfigurationsManager->getLogsOptions($code); + $databaseMaxDays = $options['database_max_days'] ?? $this->databaseMaxDays; + $fileMaxDays = $options['file_max_days'] ?? $this->fileMaxDays; + if (null === $databaseMaxDays && null === $fileMaxDays) { + continue; + } + + $records = null === $databaseMaxDays ? null : $this->purgeLogRecords($connection, $code, $databaseMaxDays, $dryRun); + $files = null === $fileMaxDays ? null : $this->purgeLogFiles($code, $fileMaxDays, $dryRun); + $totalRecords += $records ?? 0; + $totalFiles += $files ?? 0; + $rows[] = [$code, $databaseMaxDays ?? '-', $records ?? '-', $fileMaxDays ?? '-', $files ?? '-']; + } + + if ([] === $rows) { + $style->success('Nothing to purge: no retention configured ("logs.database_max_days", "logs.file_max_days").'); + + return Command::SUCCESS; + } + + $style->table( + ['Process', 'Database max days', $dryRun ? 'Log records to delete' : 'Log records deleted', 'File max days', $dryRun ? 'Log files to delete' : 'Log files deleted'], + $rows + ); + $style->success(\sprintf( + $dryRun ? '%d log record(s) and %d log file(s) to delete (dry run).' : '%d log record(s) and %d log file(s) deleted.', + $totalRecords, + $totalFiles + )); + + return Command::SUCCESS; + } + + /** + * Codes of the recorded process executions and of the defined processes, sorted. + * + * @return list + */ + private function getProcessCodes(Connection $connection): array + { + /** @var list $codes */ + $codes = $connection->fetchFirstColumn('SELECT DISTINCT code FROM process_execution'); + $configurations = [...$this->processConfigurationsManager->getPublicProcesses(), ...$this->processConfigurationsManager->getPrivateProcesses()]; + $codes = array_unique([ + ...$codes, + ...array_map(static fn (ProcessConfiguration $configuration): string => $configuration->getCode(), array_values($configurations)), + ]); + sort($codes); + + return $codes; + } + + private function purgeLogRecords(Connection $connection, string $code, int $maxDays, bool $dryRun): int + { + $limit = new \DateTimeImmutable(\sprintf('-%d days', $maxDays)); + $queryBuilder = $connection->createQueryBuilder() + ->from('log_record', 'lr') + ->innerJoin('lr', 'process_execution', 'pe', 'pe.id = lr.process_execution_id') + ->where('pe.code = :code') + ->andWhere('lr.created_at < :limit') + ->setParameter('code', $code) + ->setParameter('limit', $limit, Types::DATETIME_IMMUTABLE); + + if ($dryRun) { + /** @var int|string|false $count */ + $count = $queryBuilder->select('COUNT(lr.id)')->fetchOne(); + + return (int) $count; + } + + $queryBuilder->select('lr.id')->setMaxResults(self::BATCH_SIZE); + $deleted = 0; + do { + $ids = $queryBuilder->fetchFirstColumn(); + if ([] !== $ids) { + $deleted += (int) $connection->executeStatement( + 'DELETE FROM log_record WHERE id IN (?)', + [$ids], + [ArrayParameterType::INTEGER] + ); + } + } while (self::BATCH_SIZE === \count($ids)); + + return $deleted; + } + + private function purgeLogFiles(string $code, int $maxDays, bool $dryRun): int + { + $directory = $this->logDirectory.\DIRECTORY_SEPARATOR.$code; + // The code is the name of the directory of the log files: never look outside the log directory + if (\in_array($code, ['', '.', '..'], true) || str_contains($code, '/') || str_contains($code, '\\') || !is_dir($directory)) { + return 0; + } + + $limit = (new \DateTimeImmutable(\sprintf('-%d days', $maxDays)))->getTimestamp(); + $finder = (new Finder())->files()->in($directory)->depth(0)->name(self::LOG_FILE_PATTERN) + ->filter(static fn (\SplFileInfo $file): bool => $file->getMTime() < $limit); + $files = array_keys(iterator_to_array($finder)); + if (!$dryRun) { + (new Filesystem())->remove($files); + } + + return \count($files); + } +} diff --git a/src/DependencyInjection/CleverAgeUiProcessExtension.php b/src/DependencyInjection/CleverAgeUiProcessExtension.php index 720cd39..a790d6a 100644 --- a/src/DependencyInjection/CleverAgeUiProcessExtension.php +++ b/src/DependencyInjection/CleverAgeUiProcessExtension.php @@ -38,7 +38,7 @@ public function load(array $configs, ContainerBuilder $container): void $configuration = new Configuration($env); /** @var array{ * security: array{roles: list}, - * logs: array{store_in_database: bool, database_level: string, file_level: string, report_increment_level: string}, + * logs: array{store_in_database: bool, database_level: string, file_level: string, report_increment_level: string, database_max_days: ?int, file_max_days: ?int}, * design: array{logo_path: string}, * notification: array{enabled: bool, ...}, * process_execution: array{heartbeat_interval: int, stale_after: int}, @@ -59,6 +59,9 @@ public function load(array $configs, ContainerBuilder $container): void $container->getDefinition('cleverage_ui_process.monolog_handler.doctrine_process') ->addMethodCall('disable'); } + $container->getDefinition('cleverage_ui_process.command.purge_logs') + ->setArgument('$databaseMaxDays', $config['logs']['database_max_days']) + ->setArgument('$fileMaxDays', $config['logs']['file_max_days']); $container->getDefinition(ProcessDashboardController::class) ->setArgument('$logoPath', $config['design']['logo_path']); diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index 79baab9..41a2f76 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -64,6 +64,8 @@ protected function addLogSection(ArrayNodeDefinition $node): void ->scalarNode('database_level')->defaultValue($defaultLevel)->end() // min log level to store log record in database ->scalarNode('file_level')->defaultValue($defaultLevel)->end() // min log level to store log record in file ->scalarNode('report_increment_level')->defaultValue(Level::Warning->name)->end() // min log level to increment process execution report + ->integerNode('database_max_days')->defaultNull()->min(1)->end() // days after which the log records are deleted from database by the purge command, null to keep them, can be overridden by process + ->integerNode('file_max_days')->defaultNull()->min(1)->end() // days after which the log files are deleted by the purge command, null to keep them, can be overridden by process ->end() ->end() ->end() diff --git a/src/Manager/ProcessConfigurationsManager.php b/src/Manager/ProcessConfigurationsManager.php index 4572a4b..d2e36d8 100644 --- a/src/Manager/ProcessConfigurationsManager.php +++ b/src/Manager/ProcessConfigurationsManager.php @@ -38,6 +38,10 @@ * 'channels': ?list, * 'recipients': ?array * } + * @phpstan-type LogsOptions array{ + * 'database_max_days': ?int, + * 'file_max_days': ?int + * } */ final readonly class ProcessConfigurationsManager { @@ -87,10 +91,26 @@ public function getNotificationOptions(string $processCode): ?array return $this->resolveOptions($configuration->getOptions())['notification']; } + /** + * The "logs" option of the process, null values are inherited from the bundle configuration. + * + * @return LogsOptions|null + */ + public function getLogsOptions(string $processCode): ?array + { + if (false === $this->registry->hasProcessConfiguration($processCode)) { + return null; + } + + $configuration = $this->registry->getProcessConfiguration($processCode); + + return $this->resolveOptions($configuration->getOptions())['logs']; + } + /** * @param array $options * - * @return array{'ui': UiOptions, 'notification': NotificationOptions} + * @return array{'ui': UiOptions, 'notification': NotificationOptions, 'logs': LogsOptions} */ private function resolveOptions(array $options): array { @@ -135,6 +155,18 @@ private function resolveOptions(array $options): array return $notificationResolver->resolve($notification); }); + $resolver->setDefault('logs', []); + $resolver->setAllowedTypes('logs', 'array'); + $resolver->setNormalizer('logs', static function (Options $options, array $logs): array { + $logsResolver = new OptionsResolver(); + $logsResolver->setDefaults(['database_max_days' => null, 'file_max_days' => null]); + foreach (['database_max_days', 'file_max_days'] as $option) { + $logsResolver->setAllowedTypes($option, ['null', 'int']); + $logsResolver->setAllowedValues($option, static fn (?int $days): bool => null === $days || $days >= 1); + } + + return $logsResolver->resolve($logs); + }); $resolver->setDefault('ui', []); $resolver->setAllowedTypes('ui', 'array'); $resolver->setNormalizer('ui', static function (Options $options, array $ui): array { @@ -176,7 +208,7 @@ private function resolveOptions(array $options): array return $uiResolver->resolve($ui); }); /** - * @var array{'ui': UiOptions, 'notification': NotificationOptions} $options + * @var array{'ui': UiOptions, 'notification': NotificationOptions, 'logs': LogsOptions} $options */ $options = $resolver->resolve($options); diff --git a/tests/App/config/packages.yaml b/tests/App/config/packages.yaml index 9c965ed..600e425 100644 --- a/tests/App/config/packages.yaml +++ b/tests/App/config/packages.yaml @@ -130,6 +130,15 @@ clever_age_process: tasks: data: service: '@CleverAge\ProcessBundle\Task\ConstantOutputTask' + test.logs_retention: + public: false + options: + logs: + database_max_days: 90 + file_max_days: 365 + tasks: + data: + service: '@CleverAge\ProcessBundle\Task\DummyTask' twig_component: anonymous_template_directory: 'components/' diff --git a/tests/DependencyInjection/CleverAgeUiProcessExtensionTest.php b/tests/DependencyInjection/CleverAgeUiProcessExtensionTest.php index 08b0104..7f27ee5 100644 --- a/tests/DependencyInjection/CleverAgeUiProcessExtensionTest.php +++ b/tests/DependencyInjection/CleverAgeUiProcessExtensionTest.php @@ -55,6 +55,10 @@ public function testLoadWithDefaultConfiguration(): void self::assertSame('Info', $databaseHandler->getArgument('$level')); self::assertFalse($databaseHandler->hasMethodCall('disable')); + $purgeLogsCommand = $container->getDefinition('cleverage_ui_process.command.purge_logs'); + self::assertNull($purgeLogsCommand->getArgument('$databaseMaxDays')); + self::assertNull($purgeLogsCommand->getArgument('$fileMaxDays')); + self::assertSame( 'bundles/cleverageuiprocess/logo.jpg', $container->getDefinition(ProcessDashboardController::class)->getArgument('$logoPath') @@ -111,6 +115,8 @@ public function testLoadWithCustomConfiguration(): void 'database_level' => 'Error', 'file_level' => 'Notice', 'report_increment_level' => 'Critical', + 'database_max_days' => 30, + 'file_max_days' => 180, ], 'design' => ['logo_path' => 'images/my-logo.png'], 'process_execution' => ['heartbeat_interval' => 10, 'stale_after' => 3600], @@ -132,6 +138,10 @@ public function testLoadWithCustomConfiguration(): void $calls = $databaseHandler->getMethodCalls(); self::assertSame(['disable', []], end($calls)); + $purgeLogsCommand = $container->getDefinition('cleverage_ui_process.command.purge_logs'); + self::assertSame(30, $purgeLogsCommand->getArgument('$databaseMaxDays')); + self::assertSame(180, $purgeLogsCommand->getArgument('$fileMaxDays')); + self::assertSame( 'images/my-logo.png', $container->getDefinition(ProcessDashboardController::class)->getArgument('$logoPath') diff --git a/tests/DependencyInjection/ConfigurationTest.php b/tests/DependencyInjection/ConfigurationTest.php index db69655..52b8831 100644 --- a/tests/DependencyInjection/ConfigurationTest.php +++ b/tests/DependencyInjection/ConfigurationTest.php @@ -49,6 +49,8 @@ public function testDefaultConfiguration(string $env, string $expectedLevel): vo 'database_level' => $expectedLevel, 'file_level' => $expectedLevel, 'report_increment_level' => 'Warning', + 'database_max_days' => null, + 'file_max_days' => null, ], 'design' => [ 'logo_path' => 'bundles/cleverageuiprocess/logo.jpg', @@ -88,6 +90,8 @@ public function testCustomConfiguration(): void 'database_level' => 'Error', 'file_level' => 'Notice', 'report_increment_level' => 'Critical', + 'database_max_days' => 30, + 'file_max_days' => 180, ], 'design' => [ 'logo_path' => 'images/my-logo.png', @@ -113,6 +117,8 @@ public function testCustomConfiguration(): void 'database_level' => 'Error', 'file_level' => 'Notice', 'report_increment_level' => 'Critical', + 'database_max_days' => 30, + 'file_max_days' => 180, ], 'design' => ['logo_path' => 'images/my-logo.png'], 'notification' => [ @@ -136,6 +142,8 @@ public function testPartialConfigurationKeepsOtherDefaults(): void 'store_in_database' => true, 'database_level' => 'Debug', 'report_increment_level' => 'Warning', + 'database_max_days' => null, + 'file_max_days' => null, ], $config['logs'] ); @@ -166,6 +174,14 @@ public static function provideInvalidConfigurations(): iterable ['logs' => ['store_in_database' => 'yes']], 'clever_age_ui_process.logs.store_in_database', ]; + yield 'database_max_days lower than 1' => [ + ['logs' => ['database_max_days' => 0]], + 'clever_age_ui_process.logs.database_max_days', + ]; + yield 'file_max_days not an integer' => [ + ['logs' => ['file_max_days' => '30 days']], + 'clever_age_ui_process.logs.file_max_days', + ]; yield 'roles not an array' => [['security' => ['roles' => 'ROLE_ADMIN']], 'clever_age_ui_process.security.roles']; yield 'role not a scalar' => [['security' => ['roles' => [['ROLE_ADMIN']]]], 'clever_age_ui_process.security.roles']; yield 'logo_path not a scalar' => [['design' => ['logo_path' => ['a.png']]], 'clever_age_ui_process.design.logo_path']; diff --git a/tests/Functional/PurgeLogsCommandTest.php b/tests/Functional/PurgeLogsCommandTest.php new file mode 100644 index 0000000..ab7d1c6 --- /dev/null +++ b/tests/Functional/PurgeLogsCommandTest.php @@ -0,0 +1,260 @@ + */ + private array $createdFiles = []; + + protected function tearDown(): void + { + (new Filesystem())->remove($this->createdFiles); + $this->createdFiles = []; + parent::tearDown(); + } + + public function testService(): void + { + $command = static::getContainer()->get('cleverage_ui_process.command.purge_logs'); + + self::assertInstanceOf(PurgeLogsCommand::class, $command); + self::assertSame('cleverage:ui-process:purge-logs', $command->getName()); + self::assertFalse($command->getDefinition()->getOption('dry-run')->acceptValue()); + } + + public function testNothingToPurgeWithoutRetention(): void + { + $execution = $this->createExecution('test.process'); + $this->createLogRecord($execution, 400); + $file = $this->createLogFile('test.process', 400); + + // No process with a "logs" option + $tester = $this->execute(null, null, [], new ProcessConfigurationsManager(new ProcessConfigurationRegistry([], 'stop'))); + + $tester->assertCommandIsSuccessful(); + self::assertStringContainsString('Nothing to purge', $tester->getDisplay()); + self::assertSame(1, $this->countLogRecords()); + self::assertFileExists($file); + } + + public function testPurge(): void + { + // Bundle retention: 30 days in database, 60 days for the files + $default = $this->createExecution('test.process'); + $this->createLogRecord($default, 40); + $this->createLogRecord($default, 1); + $defaultOldFile = $this->createLogFile('test.process', 70); + $defaultRecentFile = $this->createLogFile('test.process', 1); + // Not written by the bundle: kept + $otherFile = $this->createLogFile('test.process', 70, 'notes.log'); + + // Retention of the process: 90 days in database, 365 days for the files + $overridden = $this->createExecution('test.logs_retention'); + $this->createLogRecord($overridden, 100); + $this->createLogRecord($overridden, 40); + $overriddenOldFile = $this->createLogFile('test.logs_retention', 400); + $overriddenRecentFile = $this->createLogFile('test.logs_retention', 70); + + // Process no longer defined: bundle retention + $undefined = $this->createExecution('test.removed'); + $this->createLogRecord($undefined, 40); + $undefinedOldFile = $this->createLogFile('test.removed', 70); + + $tester = $this->execute(30, 60); + + $tester->assertCommandIsSuccessful(); + self::assertStringContainsString('3 log record(s) and 3 log file(s) deleted.', $tester->getDisplay()); + self::assertSame(2, $this->countLogRecords()); + self::assertSame(1, $this->countLogRecords($default)); + self::assertSame(1, $this->countLogRecords($overridden)); + self::assertSame(0, $this->countLogRecords($undefined)); + self::assertFileDoesNotExist($defaultOldFile); + self::assertFileExists($defaultRecentFile); + self::assertFileExists($otherFile); + self::assertFileDoesNotExist($overriddenOldFile); + self::assertFileExists($overriddenRecentFile); + self::assertFileDoesNotExist($undefinedOldFile); + // The process executions are kept + self::assertSame(3, $this->getEntityManager()->getRepository(ProcessExecution::class)->count([])); + } + + public function testDryRun(): void + { + $execution = $this->createExecution('test.process'); + $this->createLogRecord($execution, 40); + $file = $this->createLogFile('test.process', 70); + + $tester = $this->execute(30, 60, ['--dry-run' => true]); + + $tester->assertCommandIsSuccessful(); + self::assertStringContainsString('1 log record(s) and 1 log file(s) to delete (dry run).', $tester->getDisplay()); + self::assertSame(1, $this->countLogRecords()); + self::assertFileExists($file); + } + + public function testRetentionOfTheProcessOnly(): void + { + // No bundle retention: only the process with a "logs" option is purged + $default = $this->createExecution('test.process'); + $this->createLogRecord($default, 400); + $defaultFile = $this->createLogFile('test.process', 400); + $overridden = $this->createExecution('test.logs_retention'); + $this->createLogRecord($overridden, 100); + $overriddenFile = $this->createLogFile('test.logs_retention', 400); + + $tester = $this->execute(null, null); + + $tester->assertCommandIsSuccessful(); + self::assertSame(1, $this->countLogRecords($default)); + self::assertSame(0, $this->countLogRecords($overridden)); + self::assertFileExists($defaultFile); + self::assertFileDoesNotExist($overriddenFile); + self::assertStringNotContainsString('test.process ', $tester->getDisplay()); + } + + public function testDatabaseRetentionOnly(): void + { + $execution = $this->createExecution('test.process'); + $this->createLogRecord($execution, 40); + $file = $this->createLogFile('test.process', 400); + + $tester = $this->execute(30, null); + + $tester->assertCommandIsSuccessful(); + self::assertSame(0, $this->countLogRecords()); + self::assertFileExists($file); + } + + public function testRecordsAreDeletedByBatches(): void + { + $execution = $this->createExecution('test.process'); + $connection = $this->getEntityManager()->getConnection(); + $connection->beginTransaction(); + for ($i = 0; $i < 2001; ++$i) { + $connection->insert('log_record', [ + 'channel' => 'cleverage_process', + 'level' => Level::Info->value, + 'message' => 'Message '.$i, + 'context' => '[]', + 'created_at' => (new \DateTimeImmutable('-40 days'))->format('Y-m-d H:i:s'), + 'process_execution_id' => $execution->getId(), + ]); + } + $connection->commit(); + + $tester = $this->execute(30, null); + + $tester->assertCommandIsSuccessful(); + self::assertStringContainsString('2001 log record(s) and 0 log file(s) deleted.', $tester->getDisplay()); + self::assertSame(0, $this->countLogRecords()); + } + + /** + * @param array $input + */ + private function execute( + ?int $databaseMaxDays, + ?int $fileMaxDays, + array $input = [], + ?ProcessConfigurationsManager $processConfigurationsManager = null, + ): CommandTester { + $processConfigurationsManager ??= $this->getProcessConfigurationsManager(); + /** @var string $logDirectory */ + $logDirectory = static::getContainer()->getParameter('kernel.logs_dir'); + $command = new PurgeLogsCommand( + $this->getEntityManager(), + $processConfigurationsManager, + $logDirectory, + $databaseMaxDays, + $fileMaxDays + ); + $tester = new CommandTester($command); + $tester->execute($input); + + return $tester; + } + + private function getProcessConfigurationsManager(): ProcessConfigurationsManager + { + /** @var ProcessConfigurationsManager $processConfigurationsManager */ + $processConfigurationsManager = static::getContainer()->get('cleverage_ui_process.manager.process_configuration'); + + return $processConfigurationsManager; + } + + private function createExecution(string $code): ProcessExecution + { + $execution = new ProcessExecution($code, Uuid::v4().'.log'); + $execution->end(); + $this->getEntityManager()->persist($execution); + $this->getEntityManager()->flush(); + + return $execution; + } + + private function createLogRecord(ProcessExecution $execution, int $daysAgo): void + { + $this->getEntityManager()->persist(new LogRecord( + new \Monolog\LogRecord(new \DateTimeImmutable(\sprintf('-%d days', $daysAgo)), 'cleverage_process', Level::Info, 'Message'), + $execution + )); + $this->getEntityManager()->flush(); + } + + private function createLogFile(string $code, int $daysAgo, ?string $filename = null): string + { + /** @var string $logDirectory */ + $logDirectory = static::getContainer()->getParameter('kernel.logs_dir'); + $path = \sprintf('%s/%s/%s', $logDirectory, $code, $filename ?? Uuid::v4().'.log'); + (new Filesystem())->dumpFile($path, 'log'); + touch($path, (new \DateTimeImmutable(\sprintf('-%d days', $daysAgo)))->getTimestamp()); + $this->createdFiles[] = $path; + + return $path; + } + + private function countLogRecords(?ProcessExecution $execution = null): int + { + $criteria = $execution instanceof ProcessExecution ? ['processExecution' => $execution] : []; + + return $this->getEntityManager()->getRepository(LogRecord::class)->count($criteria); + } +} diff --git a/tests/Manager/ProcessConfigurationsManagerTest.php b/tests/Manager/ProcessConfigurationsManagerTest.php index 027e7e7..82141d5 100644 --- a/tests/Manager/ProcessConfigurationsManagerTest.php +++ b/tests/Manager/ProcessConfigurationsManagerTest.php @@ -239,6 +239,58 @@ public function testInvalidNotificationOptions(array $options, string $exception $manager->getNotificationOptions('test.process'); } + public function testLogsOptionsOfAnUnknownProcess(): void + { + self::assertNull($this->createManager(['test.process' => $this->rawProcess()])->getLogsOptions('unknown')); + } + + public function testDefaultLogsOptions(): void + { + self::assertSame( + ['database_max_days' => null, 'file_max_days' => null], + $this->createManager(['test.process' => $this->rawProcess()])->getLogsOptions('test.process') + ); + } + + public function testConfiguredLogsOptions(): void + { + $manager = $this->createManager(['test.process' => $this->rawProcess([ + 'ui' => ['source' => 'ERP'], + 'notification' => ['enabled' => true], + 'logs' => ['database_max_days' => 30, 'file_max_days' => 365], + ])]); + + self::assertSame(['database_max_days' => 30, 'file_max_days' => 365], $manager->getLogsOptions('test.process')); + // The "logs" option is accepted next to the "ui" and "notification" ones + self::assertSame('ERP', $manager->getUiOptions('test.process')['source'] ?? null); + self::assertTrue($manager->getNotificationOptions('test.process')['enabled'] ?? null); + } + + /** + * @return iterable, class-string<\Throwable>}> + */ + public static function provideInvalidLogsOptions(): iterable + { + yield 'not an array' => [['logs' => 30], InvalidOptionsException::class]; + yield 'unknown key' => [['logs' => ['max_days' => 30]], UndefinedOptionsException::class]; + yield 'database_max_days not an integer' => [['logs' => ['database_max_days' => '30']], InvalidOptionsException::class]; + yield 'database_max_days lower than 1' => [['logs' => ['database_max_days' => 0]], InvalidOptionsException::class]; + yield 'file_max_days lower than 1' => [['logs' => ['file_max_days' => -1]], InvalidOptionsException::class]; + } + + /** + * @param array $options + * @param class-string<\Throwable> $exception + */ + #[DataProvider('provideInvalidLogsOptions')] + public function testInvalidLogsOptions(array $options, string $exception): void + { + $manager = $this->createManager(['test.process' => $this->rawProcess($options)]); + + $this->expectException($exception); + $manager->getLogsOptions('test.process'); + } + /** * @param array> $rawConfiguration */