diff --git a/composer.lock b/composer.lock index 12839ec..35d376d 100644 --- a/composer.lock +++ b/composer.lock @@ -2999,16 +2999,16 @@ }, { "name": "goteo/benzina", - "version": "0.5.1", + "version": "0.5.2", "source": { "type": "git", "url": "https://github.com/goteo/benzina.git", - "reference": "3744833b3233a42fa8fb261fdf02958eb615876f" + "reference": "5fafb74117704e5b778232d39811b6e66884ba56" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/goteo/benzina/zipball/3744833b3233a42fa8fb261fdf02958eb615876f", - "reference": "3744833b3233a42fa8fb261fdf02958eb615876f", + "url": "https://api.github.com/repos/goteo/benzina/zipball/5fafb74117704e5b778232d39811b6e66884ba56", + "reference": "5fafb74117704e5b778232d39811b6e66884ba56", "shasum": "" }, "require": { @@ -3054,9 +3054,9 @@ "description": "Migration tool to pump v3 data into a v4 schema", "support": { "issues": "https://github.com/goteo/benzina/issues", - "source": "https://github.com/goteo/benzina/tree/0.5.1" + "source": "https://github.com/goteo/benzina/tree/0.5.2" }, - "time": "2025-10-01T12:52:00+00:00" + "time": "2025-10-21T10:06:39+00:00" }, { "name": "jolicode/automapper", diff --git a/migrations/Version20260119132933.php b/migrations/Version20260119132933.php new file mode 100644 index 0000000..8b8273f --- /dev/null +++ b/migrations/Version20260119132933.php @@ -0,0 +1,28 @@ +addSql('ALTER TABLE user ADD deduped TINYINT(1) NOT NULL, ADD deduped_ids JSON NOT NULL COMMENT \'(DC2Type:json)\''); + } + + public function down(Schema $schema): void + { + // this down() migration is auto-generated, please modify it to your needs + $this->addSql('ALTER TABLE user DROP deduped, DROP deduped_ids'); + } +} diff --git a/src/Benzina/InvestRewardsPump.php b/src/Benzina/InvestRewardsPump.php index 9148c25..83acb14 100644 --- a/src/Benzina/InvestRewardsPump.php +++ b/src/Benzina/InvestRewardsPump.php @@ -11,6 +11,7 @@ use App\Money\MoneyService; use App\Repository\Gateway\ChargeRepository; use App\Repository\Project\RewardRepository; use App\Repository\User\UserRepository; +use Doctrine\Common\Collections\Criteria; use Goteo\Benzina\Pump\ArrayPumpTrait; use Goteo\Benzina\Pump\DoctrinePumpTrait; use Goteo\Benzina\Pump\PumpInterface; @@ -136,15 +137,21 @@ class InvestRewardsPump implements PumpInterface return $charge; } - private function getUser(array $invest): ?User + private function getUser(array $record): ?User { - $id = $invest['user']; + $id = $record['user']; if (isset($this->userCache[$id])) { return $this->userRepository->find($this->userCache[$id]); } - $user = $this->userRepository->findOneBy(['migratedId' => $id]); + $criteria = new Criteria(); + $criteria + ->orWhere($criteria->expr()->eq('migratedId', $id)) + ->orWhere($criteria->expr()->contains('dedupedIds', $id)) + ->setMaxResults(1); + + $user = $this->userRepository->matching($criteria)->first(); $this->userCache[$id] = $user->getId(); diff --git a/src/Benzina/InvestsPump.php b/src/Benzina/InvestsPump.php index 1bade07..78d6944 100644 --- a/src/Benzina/InvestsPump.php +++ b/src/Benzina/InvestsPump.php @@ -29,6 +29,7 @@ use App\Repository\Project\SupportRepository; use App\Repository\TipjarRepository; use App\Repository\User\UserRepository; use App\Service\Gateway\CheckoutService; +use Doctrine\Common\Collections\Criteria; use Goteo\Benzina\Pump\ArrayPumpTrait; use Goteo\Benzina\Pump\DoctrinePumpTrait; use Goteo\Benzina\Pump\PumpInterface; @@ -214,7 +215,13 @@ class InvestsPump implements PumpInterface return $this->userRepository->find($this->userCache[$id]); } - $user = $this->userRepository->findOneBy(['migratedId' => $id]); + $criteria = new Criteria(); + $criteria + ->orWhere($criteria->expr()->eq('migratedId', $id)) + ->orWhere($criteria->expr()->contains('dedupedIds', $id)) + ->setMaxResults(1); + + $user = $this->userRepository->matching($criteria)->first(); $this->userCache[$id] = $user->getId(); diff --git a/src/Benzina/ProjectsPump.php b/src/Benzina/ProjectsPump.php index 01568d6..be1167d 100644 --- a/src/Benzina/ProjectsPump.php +++ b/src/Benzina/ProjectsPump.php @@ -15,6 +15,7 @@ use App\Repository\User\UserRepository; use App\Service\Embed\EmbedService; use App\Service\Project\TerritoryService; use Doctrine\Common\Collections\ArrayCollection; +use Doctrine\Common\Collections\Criteria; use Goteo\Benzina\Pump\ArrayPumpTrait; use Goteo\Benzina\Pump\DoctrinePumpTrait; use Goteo\Benzina\Pump\PumpInterface; @@ -111,7 +112,13 @@ class ProjectsPump implements PumpInterface private function getProjectOwner(array $record): ?User { - return $this->userRepository->findOneBy(['migratedId' => $record['owner']]); + $criteria = new Criteria(); + $criteria + ->orWhere($criteria->expr()->eq('migratedId', $record['owner'])) + ->orWhere($criteria->expr()->contains('dedupedIds', $record['owner'])) + ->setMaxResults(1); + + return $this->userRepository->matching($criteria)->first() ?? null; } private function getProjectDescription(array $record): string diff --git a/src/Benzina/UsersPump.php b/src/Benzina/UsersPump.php index f7b0892..2e93851 100644 --- a/src/Benzina/UsersPump.php +++ b/src/Benzina/UsersPump.php @@ -6,7 +6,9 @@ use App\Entity\User\Organization; use App\Entity\User\Person; use App\Entity\User\User; use App\Entity\User\UserType; +use App\Repository\User\UserRepository; use App\Service\UserService; +use Doctrine\Persistence\ManagerRegistry; use Goteo\Benzina\Pump\ArrayPumpTrait; use Goteo\Benzina\Pump\DoctrinePumpTrait; use Goteo\Benzina\Pump\PumpInterface; @@ -17,7 +19,10 @@ class UsersPump implements PumpInterface use DoctrinePumpTrait; use UsersPumpTrait; - private int $userCount = 0; + public function __construct( + private UserRepository $userRepository, + private ManagerRegistry $managerRegistry, + ) {} public function supports(mixed $sample): bool { @@ -31,6 +36,39 @@ class UsersPump implements PumpInterface public function pump(mixed $record, array $context): void { $user = new User(); + $user = $this->processUser($user, $record); + + try { + $this->persist($user, $context); + } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) { + $this->managerRegistry->resetManager(); + $em = $this->managerRegistry->getManager(); + $em->clear(); + $this->setEntityManager($em); + + $user = $this->userRepository->findOneBy(['email' => $record['email']]); + + if ($user) { + $user->setDeduped(true); + $user->addDedupedId($record['id']); + + $this->persist($user, $context); + + return; + } + + $user = new User(); + $user = $this->processUser($user, $record); + $user->setHandle(UserService::asHandle($record['id'], 16, 255)); + + $this->persist($user, $context); + + return; + } + } + + private function processUser(User $user, array $record): User + { $user->setHandle($this->buildHandle($record)); $user->setPassword($record['password'] ?? ''); $user->setEmail($record['email']); @@ -47,19 +85,18 @@ class UsersPump implements PumpInterface UserType::Organization => $user = $this->setUserOrganization($record, $user), }; - $this->persist($user, $context); - ++$this->userCount; + return $user; } private function buildHandle(array $record): string { try { - $handle = UserService::asHandle($record['id']); + $handle = UserService::asHandle($record['id'], 8, 255); } catch (\Exception $e) { - $handle = UserService::asHandle($record['email']); + $handle = UserService::asHandle($record['email'], 8, 255); } - return \sprintf('%s_%02d', $handle, $this->userCount % 100); + return $handle; } private function getDateCreated(array $record): \DateTime diff --git a/src/Entity/Trait/DedupEntityTrait.php b/src/Entity/Trait/DedupEntityTrait.php new file mode 100644 index 0000000..17c9f46 --- /dev/null +++ b/src/Entity/Trait/DedupEntityTrait.php @@ -0,0 +1,47 @@ +deduped; + } + + public function setDeduped(bool $deduped): static + { + $this->deduped = $deduped; + + return $this; + } + + public function getDedupedIds(): ?array + { + return $this->dedupedIds; + } + + public function addDedupedId(string $dedupedId): static + { + $this->dedupedIds = \array_unique([...$this->dedupedIds, $dedupedId]); + + return $this; + } +} diff --git a/src/Entity/User/User.php b/src/Entity/User/User.php index 3e12ccc..053b73a 100644 --- a/src/Entity/User/User.php +++ b/src/Entity/User/User.php @@ -5,6 +5,7 @@ namespace App\Entity\User; use App\Entity\Accounting\Accounting; use App\Entity\Interface\AccountingOwnerInterface; use App\Entity\Project\Project; +use App\Entity\Trait\DedupEntityTrait; use App\Entity\Trait\MigratedEntity; use App\Entity\Trait\TimestampedCreationEntity; use App\Entity\Trait\TimestampedUpdationEntity; @@ -35,6 +36,7 @@ use Symfony\Component\Security\Core\User\UserInterface; class User implements UserInterface, PasswordAuthenticatedUserInterface, AccountingOwnerInterface { use MigratedEntity; + use DedupEntityTrait; use TimestampedCreationEntity; use TimestampedUpdationEntity; diff --git a/src/Service/UserService.php b/src/Service/UserService.php index f9f2044..eaa7c8a 100644 --- a/src/Service/UserService.php +++ b/src/Service/UserService.php @@ -7,30 +7,34 @@ class UserService /** * Normalizes a string into a handle-valid string. * + * @param string $value The input string to be normalized into a valid handle + * @param int $min The shortest possible length of the output string + * @param int $max The longest possible length of the output string + * * @return string A handle-valid string * * @throws \Exception If the given value could not be normalized into a compliant handle string */ - public static function asHandle(string $value): string + public static function asHandle(string $value, int $min = 4, int $max = 32): string { + $nval = \strtolower($value); + // If email remove provider - if (\str_contains($value, '@') && \preg_match('/^[^@]+/', $value, $matches)) { - $value = $matches[0]; - } + $nval = \preg_replace('/@.*[\.-](com|net|org|es)/', '', $nval); + + // Replace known subs + $nval = \str_replace(['&', '@', 'ñ', '=', '≠'], ['and', 'at', 'nn', 'eq', 'nq'], $nval); // Only lowercase a-z, numbers, underscore and middle dots in user handles - $value = \preg_replace('/[^a-z0-9_.]|^\.|\.$/', '_', \strtolower($value)); + $nval = \preg_replace('/[^a-z0-9_.]|^\.|\.$/', '_', \strtolower($nval)); - // Min length 4 - $value = \str_pad($value, 4, '_'); + $nval = \str_pad($nval, $min, \sprintf('#%s', \hash('md5', $value))); + $nval = \substr($nval, 0, $max); - // Max length 30 - $value = \substr($value, 0, 30); - - if (strlen(str_replace('_', '', $value)) < 1) { + if (strlen(str_replace('_', '', $nval)) < 1) { throw new \Exception(\sprintf('The string \'%s\' could not be safely normalized for a handle', $value)); } - return $value; + return $nval; } }