pohovor-public/brilo-example
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(command): add command for refresh database

Browse Source
This commit is contained in:
Ondrej Vlach 2024-08-02 14:47:23 +02:00
parent 31155efabe
commit 58f43f096a
No known key found for this signature in database
GPG Key ID: 7F141CDACEDEE2DE
11 changed files with 404 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.env
View File

@ -39,3 +39,9 @@ MESSENGER_TRANSPORT_DSN=doctrine://default?auto_setup=0
###> symfony/mailer ### ###> symfony/mailer ###
# MAILER_DSN=null://null # MAILER_DSN=null://null
###< symfony/mailer ### ###< symfony/mailer ###
###> symfony/lock ###
# Choose one of the stores below
# postgresql+advisory://db_user:db_password@localhost/db_name
LOCK_DSN=flock
###< symfony/lock ###

3
composer.json
View File

@ -11,6 +11,8 @@
"doctrine/doctrine-bundle": "^2.12", "doctrine/doctrine-bundle": "^2.12",
"doctrine/doctrine-migrations-bundle": "^3.3", "doctrine/doctrine-migrations-bundle": "^3.3",
"doctrine/orm": "^3.2", "doctrine/orm": "^3.2",
"fakerphp/faker": "^1.23",
"nubium/this-should-never-happen-exception": "^1.0",
"phpdocumentor/reflection-docblock": "^5.4", "phpdocumentor/reflection-docblock": "^5.4",
"phpstan/phpdoc-parser": "^1.29", "phpstan/phpdoc-parser": "^1.29",
"symfony/asset": "7.1.*", "symfony/asset": "7.1.*",
@ -24,6 +26,7 @@
"symfony/framework-bundle": "7.1.*", "symfony/framework-bundle": "7.1.*",
"symfony/http-client": "7.1.*", "symfony/http-client": "7.1.*",
"symfony/intl": "7.1.*", "symfony/intl": "7.1.*",
"symfony/lock": "7.1.*",
"symfony/mailer": "7.1.*", "symfony/mailer": "7.1.*",
"symfony/mime": "7.1.*", "symfony/mime": "7.1.*",
"symfony/monolog-bundle": "^3.0", "symfony/monolog-bundle": "^3.0",

184
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically" "This file is @generated automatically"
], ],
"content-hash": "7a7438d096ed310a67fe200d001e5bf4", "content-hash": "131c365156daf6fd97b5e15f509f17eb",
"packages": [ "packages": [
{ {
"name": "composer/semver", "name": "composer/semver",
@ -1376,6 +1376,69 @@
], ],
"time": "2023-10-06T06:47:41+00:00" "time": "2023-10-06T06:47:41+00:00"
}, },
{
"name": "fakerphp/faker",
"version": "v1.23.1",
"source": {
"type": "git",
"url": "https://github.com/FakerPHP/Faker.git",
"reference": "bfb4fe148adbf78eff521199619b93a52ae3554b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/FakerPHP/Faker/zipball/bfb4fe148adbf78eff521199619b93a52ae3554b",
"reference": "bfb4fe148adbf78eff521199619b93a52ae3554b",
"shasum": ""
},
"require": {
"php": "^7.4 || ^8.0",
"psr/container": "^1.0 || ^2.0",
"symfony/deprecation-contracts": "^2.2 || ^3.0"
},
"conflict": {
"fzaninotto/faker": "*"
},
"require-dev": {
"bamarni/composer-bin-plugin": "^1.4.1",
"doctrine/persistence": "^1.3 || ^2.0",
"ext-intl": "*",
"phpunit/phpunit": "^9.5.26",
"symfony/phpunit-bridge": "^5.4.16"
},
"suggest": {
"doctrine/orm": "Required to use Faker\\ORM\\Doctrine",
"ext-curl": "Required by Faker\\Provider\\Image to download images.",
"ext-dom": "Required by Faker\\Provider\\HtmlLorem for generating random HTML.",
"ext-iconv": "Required by Faker\\Provider\\ru_RU\\Text::realText() for generating real Russian text.",
"ext-mbstring": "Required for multibyte Unicode string functionality."
},
"type": "library",
"autoload": {
"psr-4": {
"Faker\\": "src/Faker/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "François Zaninotto"
}
],
"description": "Faker is a PHP library that generates fake data for you.",
"keywords": [
"data",
"faker",
"fixtures"
],
"support": {
"issues": "https://github.com/FakerPHP/Faker/issues",
"source": "https://github.com/FakerPHP/Faker/tree/v1.23.1"
},
"time": "2024-01-02T13:46:09+00:00"
},
{ {
"name": "monolog/monolog", "name": "monolog/monolog",
"version": "3.7.0", "version": "3.7.0",
@ -1477,6 +1540,47 @@
], ],
"time": "2024-06-28T09:40:51+00:00" "time": "2024-06-28T09:40:51+00:00"
}, },
{
"name": "nubium/this-should-never-happen-exception",
"version": "v1.0",
"source": {
"type": "git",
"url": "https://github.com/nubium/this-should-never-happen-exception.git",
"reference": "3ed1b6f725881c527050c235e2503a8300427b86"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/nubium/this-should-never-happen-exception/zipball/3ed1b6f725881c527050c235e2503a8300427b86",
"reference": "3ed1b6f725881c527050c235e2503a8300427b86",
"shasum": ""
},
"require-dev": {
"jakub-onderka/php-parallel-lint": "~1.0",
"phpstan/phpstan": "~0.9"
},
"type": "library",
"autoload": {
"psr-4": {
"Nubium\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Jiri Travnicek",
"email": "jiri.travnicek@nubium.cz"
}
],
"description": "Extend this exception and throw it anytime something unexpected happens.",
"support": {
"issues": "https://github.com/nubium/this-should-never-happen-exception/issues",
"source": "https://github.com/nubium/this-should-never-happen-exception/tree/master"
},
"time": "2018-03-27T10:16:09+00:00"
},
{ {
"name": "phpdocumentor/reflection-common", "name": "phpdocumentor/reflection-common",
"version": "2.2.0", "version": "2.2.0",
@ -4152,6 +4256,84 @@
], ],
"time": "2024-05-31T14:57:53+00:00" "time": "2024-05-31T14:57:53+00:00"
}, },
{
"name": "symfony/lock",
"version": "v7.1.1",
"source": {
"type": "git",
"url": "https://github.com/symfony/lock.git",
"reference": "1f8c941f1270dee046e09a826bcdd3b2ebada45e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/lock/zipball/1f8c941f1270dee046e09a826bcdd3b2ebada45e",
"reference": "1f8c941f1270dee046e09a826bcdd3b2ebada45e",
"shasum": ""
},
"require": {
"php": ">=8.2",
"psr/log": "^1|^2|^3"
},
"conflict": {
"doctrine/dbal": "<3.6",
"symfony/cache": "<6.4"
},
"require-dev": {
"doctrine/dbal": "^3.6|^4",
"predis/predis": "^1.1|^2.0"
},
"type": "library",
"autoload": {
"psr-4": {
"Symfony\\Component\\Lock\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Jérémy Derussé",
"email": "jeremy@derusse.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Creates and manages locks, a mechanism to provide exclusive access to a shared resource",
"homepage": "https://symfony.com",
"keywords": [
"cas",
"flock",
"locking",
"mutex",
"redlock",
"semaphore"
],
"support": {
"source": "https://github.com/symfony/lock/tree/v7.1.1"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2024-05-31T14:57:53+00:00"
},
{ {
"name": "symfony/mailer", "name": "symfony/mailer",
"version": "v7.1.2", "version": "v7.1.2",

2
config/packages/lock.yaml Normal file
View File

@ -0,0 +1,2 @@
framework:
lock: '%env(LOCK_DSN)%'

3
config/services.yaml
View File

@ -20,5 +20,8 @@ services:
- '../src/Entity/' - '../src/Entity/'
- '../src/Kernel.php' - '../src/Kernel.php'
App\Command\RefreshDatabaseCommand:
arguments:
$name: "brilo:refresh-database"
# add more service definitions when explicit configuration is needed # add more service definitions when explicit configuration is needed
# please note that last definitions always *replace* previous ones # please note that last definitions always *replace* previous ones

148
src/Command/RefreshDatabaseCommand.php Normal file
View File

@ -0,0 +1,148 @@
<?php
declare(strict_types=1);
namespace App\Command;
use App\Entity\Database\Comments\Comment;
use App\Entity\Database\Posts\Post;
use App\Entity\Database\Users\User;
use App\Repository\Users\CompanyRepository;
use App\Service\Persist\RemoteCommentConvertor;
use App\Service\Persist\RemotePostsConvertor;
use App\Service\Persist\RemoteUsersConvertor;
use App\Service\Remote\BriloApiComments;
use App\Service\Remote\BriloApiPosts;
use App\Service\Remote\BriloApiUsers;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Lock\LockFactory;
use Symfony\Component\Lock\SharedLockInterface;
/**
* Download new data from Brilo API and import it into the database.
*/
final class RefreshDatabaseCommand extends Command
{
private readonly SharedLockInterface $lock;
public function __construct(
?string $name,
private readonly BriloApiComments $remoteComments,
private readonly BriloApiPosts $remotePosts,
private readonly BriloApiUsers $remoteUsers,
private readonly LoggerInterface $logger,
private readonly RemoteUsersConvertor $remoteUsersConvertor,
private readonly RemotePostsConvertor $remotePostsConvertor,
private readonly RemoteCommentConvertor $remoteCommentsConvertor,
private readonly CompanyRepository $companyRepository,
private readonly EntityManagerInterface $em,
LockFactory $lock,
) {
$this->lock = $lock->createLock('refresh-database');
parent::__construct($name);
}
protected function configure(): void
{
$this->setHelp('Refresh the database from the Brilo API');
$this->setDescription('Import new data from Brilo API and import it into the database.');
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
$this->lock->acquire(true);
try {
return $this->runImport();
} finally {
$this->lock->release();
}
}
/**
* @return int
* @throws \Exception
*/
private function runImport(): int
{
$returnCode = 0;
// First import users, then posts and finally comments
$data = $this->remoteUsersConvertor->convert($this->remoteUsers->getUsers());
$code = $this->runGeneratorInTransaction($data, function (User $user) {
$this->logger->info("Processing user: {$user->id}");
$this->em->persist($user->address);
$this->em->persist($user->company);
$this->em->persist($user);
});
if ($code != 0) {
$returnCode = 1;
}
$data = $this->remotePostsConvertor->convert($this->remotePosts->getPosts());
$code = $this->runGeneratorInTransaction($data, function (Post $post) {
$this->logger->info("Processing post: {$post->id}");
$this->em->persist($post);
});
if ($code != 0) {
$returnCode = 1;
}
$data = $this->remoteCommentsConvertor->convert($this->remoteComments->getComments());
$code = $this->runGeneratorInTransaction($data, function (Comment $comment) {
$this->logger->info("Processing comment: {$comment->id}");
$this->em->persist($comment);
});
if ($code != 0) {
$returnCode = 1;
}
$this->companyRepository->deleteOrphanRecords();
$this->logger->info("Refreshing database completed");
return $returnCode;
}
/**
* Run every yielded item from generator in separate transaction and call $callback with item from generator
* @param \Generator $data
* @param callable $callback
* @return int
*/
private function runGeneratorInTransaction(\Generator $data, callable $callback): int
{
$returnCode = 0;
while ($data->valid()) { /* normalni foreach nepouzivam kvuli microtranskaci ...
v tomhle pripade mi prijde lepsi mit v databazi neco nez nic (nebo stara data) v pripade jakekoliv chyby,
navic "velke" transkace nejsou dobre pro databazi a tohle je api 3-ti strany...
ale finalni slovo by mel mit zadavatel, ktereho "nemam" */
$this->em->beginTransaction();
try {
$callback($data->current());
$this->em->flush();
$this->em->commit();
} catch (\Exception $e) {
$returnCode = 1;
$this->em->rollback();
$this->logger->error("Database transaction failed", ['exception' => $e]);
continue;
} finally {
$data->next();
}
}
return $returnCode;
}
}

1
src/Repository/Post/PostRepository.php
View File

@ -9,6 +9,7 @@ use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
use Doctrine\Persistence\ManagerRegistry; use Doctrine\Persistence\ManagerRegistry;
/** /**
* In case of deleting records, you must modify RefreshDatabaseCommand transaction logic
* @extends ServiceEntityRepository<Post> * @extends ServiceEntityRepository<Post>
*/ */
class PostRepository extends ServiceEntityRepository class PostRepository extends ServiceEntityRepository

1
src/Repository/Users/CompanyRepository.php
View File

@ -9,6 +9,7 @@ use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
use Doctrine\Persistence\ManagerRegistry; use Doctrine\Persistence\ManagerRegistry;
/** /**
* In case of deleting records, you must modify RefreshDatabaseCommand transaction logic
* @extends ServiceEntityRepository<Company> * @extends ServiceEntityRepository<Company>
*/ */
class CompanyRepository extends ServiceEntityRepository class CompanyRepository extends ServiceEntityRepository

1
src/Repository/Users/UserRepository.php
View File

@ -9,6 +9,7 @@ use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
use Doctrine\Persistence\ManagerRegistry; use Doctrine\Persistence\ManagerRegistry;
/** /**
* In case of deleting records, you must modify RefreshDatabaseCommand transaction logic
* @extends ServiceEntityRepository<User> * @extends ServiceEntityRepository<User>
*/ */
class UserRepository extends ServiceEntityRepository class UserRepository extends ServiceEntityRepository

12
symfony.lock
View File

@ -134,6 +134,18 @@
"src/Kernel.php" "src/Kernel.php"
] ]
}, },
"symfony/lock": {
"version": "7.1",
"recipe": {
"repo": "github.com/symfony/recipes",
"branch": "main",
"version": "5.2",
"ref": "8e937ff2b4735d110af1770f242c1107fdab4c8e"
},
"files": [
"config/packages/lock.yaml"
]
},
"symfony/mailer": { "symfony/mailer": {
"version": "7.1", "version": "7.1",
"recipe": { "recipe": {

44
tests/Integration/Command/RefreshDatabaseCommandTest.php Normal file
View File

@ -0,0 +1,44 @@
<?php
declare(strict_types=1);
namespace App\Tests\Integration\Command;
use App\Entity\Database\Comments\Comment;
use App\Entity\Database\Posts\Post;
use App\Entity\Database\Users\Address;
use App\Entity\Database\Users\Company;
use App\Entity\Database\Users\User;
use App\Tests\Common\DatabaseTestTrait;
use Nubium\Exception\ThisShouldNeverHappenException;
use Symfony\Bundle\FrameworkBundle\Console\Application;
use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
use Symfony\Component\Console\Tester\CommandTester;
class RefreshDatabaseCommandTest extends KernelTestCase
{
use DatabaseTestTrait;
public function setUp(): void
{
parent::bootKernel();
$this->bootDatabase();
}
public function testCommandIsSuccessful(): void
{
$app = new Application(self::$kernel ?? throw new ThisShouldNeverHappenException("Kernel not booted yet"));
$command = $app->find('brilo:refresh-database');
$commandTester = new CommandTester($command);
$commandTester->execute([]);
$this->assertSame(0, $commandTester->getStatusCode());
// Tady bych otestoval ze jsou tam nejake data ktere tam vzdy maji byt ... nicmene o zadnych takovych datech nevim
$this->assertGreaterThan(0, $this->getEntityCount(User::class));
$this->assertGreaterThan(0, $this->getEntityCount(Company::class));
$this->assertGreaterThan(0, $this->getEntityCount(Address::class));
$this->assertGreaterThan(0, $this->getEntityCount(Post::class));
$this->assertGreaterThan(0, $this->getEntityCount(Comment::class));
}
}