From 58f43f096a9545afc155d1e9776b67b38e1e1111 Mon Sep 17 00:00:00 2001 From: Ondrej Vlach Date: Fri, 2 Aug 2024 14:47:23 +0200 Subject: [PATCH] feat(command): add command for refresh database --- .env | 6 + composer.json | 3 + composer.lock | 184 +++++++++++++++++- config/packages/lock.yaml | 2 + config/services.yaml | 3 + src/Command/RefreshDatabaseCommand.php | 148 ++++++++++++++ src/Repository/Post/PostRepository.php | 1 + src/Repository/Users/CompanyRepository.php | 1 + src/Repository/Users/UserRepository.php | 1 + symfony.lock | 12 ++ .../Command/RefreshDatabaseCommandTest.php | 44 +++++ 11 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 config/packages/lock.yaml create mode 100644 src/Command/RefreshDatabaseCommand.php create mode 100644 tests/Integration/Command/RefreshDatabaseCommandTest.php diff --git a/.env b/.env index 1fdabde..8cff316 100644 --- a/.env +++ b/.env @@ -39,3 +39,9 @@ MESSENGER_TRANSPORT_DSN=doctrine://default?auto_setup=0 ###> symfony/mailer ### # MAILER_DSN=null://null ###< symfony/mailer ### + +###> symfony/lock ### +# Choose one of the stores below +# postgresql+advisory://db_user:db_password@localhost/db_name +LOCK_DSN=flock +###< symfony/lock ### diff --git a/composer.json b/composer.json index 35b4d22..e9cd72f 100644 --- a/composer.json +++ b/composer.json @@ -11,6 +11,8 @@ "doctrine/doctrine-bundle": "^2.12", "doctrine/doctrine-migrations-bundle": "^3.3", "doctrine/orm": "^3.2", + "fakerphp/faker": "^1.23", + "nubium/this-should-never-happen-exception": "^1.0", "phpdocumentor/reflection-docblock": "^5.4", "phpstan/phpdoc-parser": "^1.29", "symfony/asset": "7.1.*", @@ -24,6 +26,7 @@ "symfony/framework-bundle": "7.1.*", "symfony/http-client": "7.1.*", "symfony/intl": "7.1.*", + "symfony/lock": "7.1.*", "symfony/mailer": "7.1.*", "symfony/mime": "7.1.*", "symfony/monolog-bundle": "^3.0", diff --git a/composer.lock b/composer.lock index efa770c..b2dab9b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7a7438d096ed310a67fe200d001e5bf4", + "content-hash": "131c365156daf6fd97b5e15f509f17eb", "packages": [ { "name": "composer/semver", @@ -1376,6 +1376,69 @@ ], "time": "2023-10-06T06:47:41+00:00" }, + { + "name": "fakerphp/faker", + "version": "v1.23.1", + "source": { + "type": "git", + "url": "https://github.com/FakerPHP/Faker.git", + "reference": "bfb4fe148adbf78eff521199619b93a52ae3554b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/FakerPHP/Faker/zipball/bfb4fe148adbf78eff521199619b93a52ae3554b", + "reference": "bfb4fe148adbf78eff521199619b93a52ae3554b", + "shasum": "" + }, + "require": { + "php": "^7.4 || ^8.0", + "psr/container": "^1.0 || ^2.0", + "symfony/deprecation-contracts": "^2.2 || ^3.0" + }, + "conflict": { + "fzaninotto/faker": "*" + }, + "require-dev": { + "bamarni/composer-bin-plugin": "^1.4.1", + "doctrine/persistence": "^1.3 || ^2.0", + "ext-intl": "*", + "phpunit/phpunit": "^9.5.26", + "symfony/phpunit-bridge": "^5.4.16" + }, + "suggest": { + "doctrine/orm": "Required to use Faker\\ORM\\Doctrine", + "ext-curl": "Required by Faker\\Provider\\Image to download images.", + "ext-dom": "Required by Faker\\Provider\\HtmlLorem for generating random HTML.", + "ext-iconv": "Required by Faker\\Provider\\ru_RU\\Text::realText() for generating real Russian text.", + "ext-mbstring": "Required for multibyte Unicode string functionality." + }, + "type": "library", + "autoload": { + "psr-4": { + "Faker\\": "src/Faker/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "François Zaninotto" + } + ], + "description": "Faker is a PHP library that generates fake data for you.", + "keywords": [ + "data", + "faker", + "fixtures" + ], + "support": { + "issues": "https://github.com/FakerPHP/Faker/issues", + "source": "https://github.com/FakerPHP/Faker/tree/v1.23.1" + }, + "time": "2024-01-02T13:46:09+00:00" + }, { "name": "monolog/monolog", "version": "3.7.0", @@ -1477,6 +1540,47 @@ ], "time": "2024-06-28T09:40:51+00:00" }, + { + "name": "nubium/this-should-never-happen-exception", + "version": "v1.0", + "source": { + "type": "git", + "url": "https://github.com/nubium/this-should-never-happen-exception.git", + "reference": "3ed1b6f725881c527050c235e2503a8300427b86" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/nubium/this-should-never-happen-exception/zipball/3ed1b6f725881c527050c235e2503a8300427b86", + "reference": "3ed1b6f725881c527050c235e2503a8300427b86", + "shasum": "" + }, + "require-dev": { + "jakub-onderka/php-parallel-lint": "~1.0", + "phpstan/phpstan": "~0.9" + }, + "type": "library", + "autoload": { + "psr-4": { + "Nubium\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jiri Travnicek", + "email": "jiri.travnicek@nubium.cz" + } + ], + "description": "Extend this exception and throw it anytime something unexpected happens.", + "support": { + "issues": "https://github.com/nubium/this-should-never-happen-exception/issues", + "source": "https://github.com/nubium/this-should-never-happen-exception/tree/master" + }, + "time": "2018-03-27T10:16:09+00:00" + }, { "name": "phpdocumentor/reflection-common", "version": "2.2.0", @@ -4152,6 +4256,84 @@ ], "time": "2024-05-31T14:57:53+00:00" }, + { + "name": "symfony/lock", + "version": "v7.1.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/lock.git", + "reference": "1f8c941f1270dee046e09a826bcdd3b2ebada45e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/lock/zipball/1f8c941f1270dee046e09a826bcdd3b2ebada45e", + "reference": "1f8c941f1270dee046e09a826bcdd3b2ebada45e", + "shasum": "" + }, + "require": { + "php": ">=8.2", + "psr/log": "^1|^2|^3" + }, + "conflict": { + "doctrine/dbal": "<3.6", + "symfony/cache": "<6.4" + }, + "require-dev": { + "doctrine/dbal": "^3.6|^4", + "predis/predis": "^1.1|^2.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Lock\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jérémy Derussé", + "email": "jeremy@derusse.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Creates and manages locks, a mechanism to provide exclusive access to a shared resource", + "homepage": "https://symfony.com", + "keywords": [ + "cas", + "flock", + "locking", + "mutex", + "redlock", + "semaphore" + ], + "support": { + "source": "https://github.com/symfony/lock/tree/v7.1.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-05-31T14:57:53+00:00" + }, { "name": "symfony/mailer", "version": "v7.1.2", diff --git a/config/packages/lock.yaml b/config/packages/lock.yaml new file mode 100644 index 0000000..574879f --- /dev/null +++ b/config/packages/lock.yaml @@ -0,0 +1,2 @@ +framework: + lock: '%env(LOCK_DSN)%' diff --git a/config/services.yaml b/config/services.yaml index 2d6a76f..1e61849 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -20,5 +20,8 @@ services: - '../src/Entity/' - '../src/Kernel.php' + App\Command\RefreshDatabaseCommand: + arguments: + $name: "brilo:refresh-database" # add more service definitions when explicit configuration is needed # please note that last definitions always *replace* previous ones diff --git a/src/Command/RefreshDatabaseCommand.php b/src/Command/RefreshDatabaseCommand.php new file mode 100644 index 0000000..7bc60b6 --- /dev/null +++ b/src/Command/RefreshDatabaseCommand.php @@ -0,0 +1,148 @@ +lock = $lock->createLock('refresh-database'); + parent::__construct($name); + } + + protected function configure(): void + { + $this->setHelp('Refresh the database from the Brilo API'); + $this->setDescription('Import new data from Brilo API and import it into the database.'); + } + + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->lock->acquire(true); + + try { + return $this->runImport(); + } finally { + $this->lock->release(); + } + } + /** + * @return int + * @throws \Exception + */ + private function runImport(): int + { + $returnCode = 0; + + // First import users, then posts and finally comments + + $data = $this->remoteUsersConvertor->convert($this->remoteUsers->getUsers()); + + $code = $this->runGeneratorInTransaction($data, function (User $user) { + $this->logger->info("Processing user: {$user->id}"); + $this->em->persist($user->address); + $this->em->persist($user->company); + $this->em->persist($user); + }); + + if ($code != 0) { + $returnCode = 1; + } + + $data = $this->remotePostsConvertor->convert($this->remotePosts->getPosts()); + + $code = $this->runGeneratorInTransaction($data, function (Post $post) { + $this->logger->info("Processing post: {$post->id}"); + $this->em->persist($post); + }); + + if ($code != 0) { + $returnCode = 1; + } + + $data = $this->remoteCommentsConvertor->convert($this->remoteComments->getComments()); + + $code = $this->runGeneratorInTransaction($data, function (Comment $comment) { + $this->logger->info("Processing comment: {$comment->id}"); + $this->em->persist($comment); + }); + + if ($code != 0) { + $returnCode = 1; + } + + $this->companyRepository->deleteOrphanRecords(); + $this->logger->info("Refreshing database completed"); + return $returnCode; + } + + /** + * Run every yielded item from generator in separate transaction and call $callback with item from generator + * @param \Generator $data + * @param callable $callback + * @return int + */ + private function runGeneratorInTransaction(\Generator $data, callable $callback): int + { + $returnCode = 0; + + while ($data->valid()) { /* normalni foreach nepouzivam kvuli microtranskaci ... + v tomhle pripade mi prijde lepsi mit v databazi neco nez nic (nebo stara data) v pripade jakekoliv chyby, + navic "velke" transkace nejsou dobre pro databazi a tohle je api 3-ti strany... + ale finalni slovo by mel mit zadavatel, ktereho "nemam" */ + $this->em->beginTransaction(); + + try { + $callback($data->current()); + $this->em->flush(); + $this->em->commit(); + } catch (\Exception $e) { + $returnCode = 1; + $this->em->rollback(); + $this->logger->error("Database transaction failed", ['exception' => $e]); + continue; + } finally { + $data->next(); + } + } + + return $returnCode; + } +} diff --git a/src/Repository/Post/PostRepository.php b/src/Repository/Post/PostRepository.php index e675952..2c59446 100644 --- a/src/Repository/Post/PostRepository.php +++ b/src/Repository/Post/PostRepository.php @@ -9,6 +9,7 @@ use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository; use Doctrine\Persistence\ManagerRegistry; /** + * In case of deleting records, you must modify RefreshDatabaseCommand transaction logic * @extends ServiceEntityRepository */ class PostRepository extends ServiceEntityRepository diff --git a/src/Repository/Users/CompanyRepository.php b/src/Repository/Users/CompanyRepository.php index 288e3eb..01393c3 100644 --- a/src/Repository/Users/CompanyRepository.php +++ b/src/Repository/Users/CompanyRepository.php @@ -9,6 +9,7 @@ use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository; use Doctrine\Persistence\ManagerRegistry; /** + * In case of deleting records, you must modify RefreshDatabaseCommand transaction logic * @extends ServiceEntityRepository */ class CompanyRepository extends ServiceEntityRepository diff --git a/src/Repository/Users/UserRepository.php b/src/Repository/Users/UserRepository.php index a68ed58..7872a36 100644 --- a/src/Repository/Users/UserRepository.php +++ b/src/Repository/Users/UserRepository.php @@ -9,6 +9,7 @@ use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository; use Doctrine\Persistence\ManagerRegistry; /** + * In case of deleting records, you must modify RefreshDatabaseCommand transaction logic * @extends ServiceEntityRepository */ class UserRepository extends ServiceEntityRepository diff --git a/symfony.lock b/symfony.lock index 283e3b4..b4aa190 100644 --- a/symfony.lock +++ b/symfony.lock @@ -134,6 +134,18 @@ "src/Kernel.php" ] }, + "symfony/lock": { + "version": "7.1", + "recipe": { + "repo": "github.com/symfony/recipes", + "branch": "main", + "version": "5.2", + "ref": "8e937ff2b4735d110af1770f242c1107fdab4c8e" + }, + "files": [ + "config/packages/lock.yaml" + ] + }, "symfony/mailer": { "version": "7.1", "recipe": { diff --git a/tests/Integration/Command/RefreshDatabaseCommandTest.php b/tests/Integration/Command/RefreshDatabaseCommandTest.php new file mode 100644 index 0000000..47c0048 --- /dev/null +++ b/tests/Integration/Command/RefreshDatabaseCommandTest.php @@ -0,0 +1,44 @@ +bootDatabase(); + } + + public function testCommandIsSuccessful(): void + { + $app = new Application(self::$kernel ?? throw new ThisShouldNeverHappenException("Kernel not booted yet")); + $command = $app->find('brilo:refresh-database'); + $commandTester = new CommandTester($command); + + $commandTester->execute([]); + + $this->assertSame(0, $commandTester->getStatusCode()); + // Tady bych otestoval ze jsou tam nejake data ktere tam vzdy maji byt ... nicmene o zadnych takovych datech nevim + $this->assertGreaterThan(0, $this->getEntityCount(User::class)); + $this->assertGreaterThan(0, $this->getEntityCount(Company::class)); + $this->assertGreaterThan(0, $this->getEntityCount(Address::class)); + $this->assertGreaterThan(0, $this->getEntityCount(Post::class)); + $this->assertGreaterThan(0, $this->getEntityCount(Comment::class)); + } +}