DemocraticTech.Fund/crowd-fund-api
0
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #184 from goteo/fix/dedup-pumpings

Browse Source 
[fix] Dedup pumpings
This commit is contained in:
daniel.subiabre
2026-02-02 12:07:13 +01:00
committed by GitHub
parent 91e9ecfcaf e835908f01
commit 60a2bfdc75
9 changed files with 168 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
composer.lock
Generated
+6 -6
View File
@@ -2999,16 +2999,16 @@
},
{
"name": "goteo/benzina",
"version": "0.5.1",
"version": "0.5.2",
"source": {
"type": "git",
"url": "https://github.com/goteo/benzina.git",
"reference": "3744833b3233a42fa8fb261fdf02958eb615876f"
"reference": "5fafb74117704e5b778232d39811b6e66884ba56"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/goteo/benzina/zipball/3744833b3233a42fa8fb261fdf02958eb615876f",
"reference": "3744833b3233a42fa8fb261fdf02958eb615876f",
"url": "https://api.github.com/repos/goteo/benzina/zipball/5fafb74117704e5b778232d39811b6e66884ba56",
"reference": "5fafb74117704e5b778232d39811b6e66884ba56",
"shasum": ""
},
"require": {
@@ -3054,9 +3054,9 @@
"description": "Migration tool to pump v3 data into a v4 schema",
"support": {
"issues": "https://github.com/goteo/benzina/issues",
"source": "https://github.com/goteo/benzina/tree/0.5.1"
"source": "https://github.com/goteo/benzina/tree/0.5.2"
},
"time": "2025-10-01T12:52:00+00:00"
"time": "2025-10-21T10:06:39+00:00"
},
{
"name": "jolicode/automapper",
migrations/Version20260119132933.php
+28
View File
@@ -0,0 +1,28 @@
<?php
declare(strict_types=1);
namespace DoctrineMigrations;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\Migrations\AbstractMigration;
final class Version20260119132933 extends AbstractMigration
{
public function getDescription(): string
{
return 'Add deduped entity schema';
}
public function up(Schema $schema): void
{
// this up() migration is auto-generated, please modify it to your needs
$this->addSql('ALTER TABLE user ADD deduped TINYINT(1) NOT NULL, ADD deduped_ids JSON NOT NULL COMMENT \'(DC2Type:json)\'');
}
public function down(Schema $schema): void
{
// this down() migration is auto-generated, please modify it to your needs
$this->addSql('ALTER TABLE user DROP deduped, DROP deduped_ids');
}
}
src/Benzina/InvestRewardsPump.php
+10 -3
View File
@@ -11,6 +11,7 @@ use App\Money\MoneyService;
use App\Repository\Gateway\ChargeRepository;
use App\Repository\Project\RewardRepository;
use App\Repository\User\UserRepository;
use Doctrine\Common\Collections\Criteria;
use Goteo\Benzina\Pump\ArrayPumpTrait;
use Goteo\Benzina\Pump\DoctrinePumpTrait;
use Goteo\Benzina\Pump\PumpInterface;
@@ -136,15 +137,21 @@ class InvestRewardsPump implements PumpInterface
return $charge;
}
private function getUser(array $invest): ?User
private function getUser(array $record): ?User
{
$id = $invest['user'];
$id = $record['user'];
if (isset($this->userCache[$id])) {
return $this->userRepository->find($this->userCache[$id]);
}
$user = $this->userRepository->findOneBy(['migratedId' => $id]);
$criteria = new Criteria();
$criteria
->orWhere($criteria->expr()->eq('migratedId', $id))
->orWhere($criteria->expr()->contains('dedupedIds', $id))
->setMaxResults(1);
$user = $this->userRepository->matching($criteria)->first();
$this->userCache[$id] = $user->getId();
src/Benzina/InvestsPump.php
+8 -1
View File
@@ -29,6 +29,7 @@ use App\Repository\Project\SupportRepository;
use App\Repository\TipjarRepository;
use App\Repository\User\UserRepository;
use App\Service\Gateway\CheckoutService;
use Doctrine\Common\Collections\Criteria;
use Goteo\Benzina\Pump\ArrayPumpTrait;
use Goteo\Benzina\Pump\DoctrinePumpTrait;
use Goteo\Benzina\Pump\PumpInterface;
@@ -214,7 +215,13 @@ class InvestsPump implements PumpInterface
return $this->userRepository->find($this->userCache[$id]);
}
$user = $this->userRepository->findOneBy(['migratedId' => $id]);
$criteria = new Criteria();
$criteria
->orWhere($criteria->expr()->eq('migratedId', $id))
->orWhere($criteria->expr()->contains('dedupedIds', $id))
->setMaxResults(1);
$user = $this->userRepository->matching($criteria)->first();
$this->userCache[$id] = $user->getId();
src/Benzina/ProjectsPump.php
+8 -1
View File
@@ -15,6 +15,7 @@ use App\Repository\User\UserRepository;
use App\Service\Embed\EmbedService;
use App\Service\Project\TerritoryService;
use Doctrine\Common\Collections\ArrayCollection;
use Doctrine\Common\Collections\Criteria;
use Goteo\Benzina\Pump\ArrayPumpTrait;
use Goteo\Benzina\Pump\DoctrinePumpTrait;
use Goteo\Benzina\Pump\PumpInterface;
@@ -111,7 +112,13 @@ class ProjectsPump implements PumpInterface
private function getProjectOwner(array $record): ?User
{
return $this->userRepository->findOneBy(['migratedId' => $record['owner']]);
$criteria = new Criteria();
$criteria
->orWhere($criteria->expr()->eq('migratedId', $record['owner']))
->orWhere($criteria->expr()->contains('dedupedIds', $record['owner']))
->setMaxResults(1);
return $this->userRepository->matching($criteria)->first() ?? null;
}
private function getProjectDescription(array $record): string
src/Benzina/UsersPump.php
+43 -6
View File
@@ -6,7 +6,9 @@ use App\Entity\User\Organization;
use App\Entity\User\Person;
use App\Entity\User\User;
use App\Entity\User\UserType;
use App\Repository\User\UserRepository;
use App\Service\UserService;
use Doctrine\Persistence\ManagerRegistry;
use Goteo\Benzina\Pump\ArrayPumpTrait;
use Goteo\Benzina\Pump\DoctrinePumpTrait;
use Goteo\Benzina\Pump\PumpInterface;
@@ -17,7 +19,10 @@ class UsersPump implements PumpInterface
use DoctrinePumpTrait;
use UsersPumpTrait;
private int $userCount = 0;
public function __construct(
private UserRepository $userRepository,
private ManagerRegistry $managerRegistry,
) {}
public function supports(mixed $sample): bool
{
@@ -31,6 +36,39 @@ class UsersPump implements PumpInterface
public function pump(mixed $record, array $context): void
{
$user = new User();
$user = $this->processUser($user, $record);
try {
$this->persist($user, $context);
} catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) {
$this->managerRegistry->resetManager();
$em = $this->managerRegistry->getManager();
$em->clear();
$this->setEntityManager($em);
$user = $this->userRepository->findOneBy(['email' => $record['email']]);
if ($user) {
$user->setDeduped(true);
$user->addDedupedId($record['id']);
$this->persist($user, $context);
return;
}
$user = new User();
$user = $this->processUser($user, $record);
$user->setHandle(UserService::asHandle($record['id'], 16, 255));
$this->persist($user, $context);
return;
}
}
private function processUser(User $user, array $record): User
{
$user->setHandle($this->buildHandle($record));
$user->setPassword($record['password'] ?? '');
$user->setEmail($record['email']);
@@ -47,19 +85,18 @@ class UsersPump implements PumpInterface
UserType::Organization => $user = $this->setUserOrganization($record, $user),
};
$this->persist($user, $context);
++$this->userCount;
return $user;
}
private function buildHandle(array $record): string
{
try {
$handle = UserService::asHandle($record['id']);
$handle = UserService::asHandle($record['id'], 8, 255);
} catch (\Exception $e) {
$handle = UserService::asHandle($record['email']);
$handle = UserService::asHandle($record['email'], 8, 255);
}
return \sprintf('%s_%02d', $handle, $this->userCount % 100);
return $handle;
}
private function getDateCreated(array $record): \DateTime
src/Entity/Trait/DedupEntityTrait.php
+47
View File
@@ -0,0 +1,47 @@
<?php
namespace App\Entity\Trait;
use ApiPlatform\Metadata as API;
use Doctrine\ORM\Mapping as ORM;
trait DedupEntityTrait
{
/**
* Entity came duplicated from the Goteo v3 platform.
*/
#[API\ApiProperty(writable: false)]
#[ORM\Column()]
protected ?bool $deduped = false;
/**
* Previous IDs of the duplicated entities in the Goteo v3 platform.
*/
#[API\ApiProperty(writable: false)]
#[ORM\Column()]
protected array $dedupedIds = [];
public function isDeduped(): ?bool
{
return $this->deduped;
}
public function setDeduped(bool $deduped): static
{
$this->deduped = $deduped;
return $this;
}
public function getDedupedIds(): ?array
{
return $this->dedupedIds;
}
public function addDedupedId(string $dedupedId): static
{
$this->dedupedIds = \array_unique([...$this->dedupedIds, $dedupedId]);
return $this;
}
}
src/Entity/User/User.php
+2
View File
@@ -5,6 +5,7 @@ namespace App\Entity\User;
use App\Entity\Accounting\Accounting;
use App\Entity\Interface\AccountingOwnerInterface;
use App\Entity\Project\Project;
use App\Entity\Trait\DedupEntityTrait;
use App\Entity\Trait\MigratedEntity;
use App\Entity\Trait\TimestampedCreationEntity;
use App\Entity\Trait\TimestampedUpdationEntity;
@@ -35,6 +36,7 @@ use Symfony\Component\Security\Core\User\UserInterface;
class User implements UserInterface, PasswordAuthenticatedUserInterface, AccountingOwnerInterface
{
use MigratedEntity;
use DedupEntityTrait;
use TimestampedCreationEntity;
use TimestampedUpdationEntity;
src/Service/UserService.php
+16 -12
View File
@@ -7,30 +7,34 @@ class UserService
/**
* Normalizes a string into a handle-valid string.
*
* @param string $value The input string to be normalized into a valid handle
* @param int $min The shortest possible length of the output string
* @param int $max The longest possible length of the output string
*
* @return string A handle-valid string
*
* @throws \Exception If the given value could not be normalized into a compliant handle string
*/
public static function asHandle(string $value): string
public static function asHandle(string $value, int $min = 4, int $max = 32): string
{
$nval = \strtolower($value);
// If email remove provider
if (\str_contains($value, '@') && \preg_match('/^[^@]+/', $value, $matches)) {
$value = $matches[0];
}
$nval = \preg_replace('/@.*[\.-](com|net|org|es)/', '', $nval);
// Replace known subs
$nval = \str_replace(['&', '@', 'ñ', '=', '≠'], ['and', 'at', 'nn', 'eq', 'nq'], $nval);
// Only lowercase a-z, numbers, underscore and middle dots in user handles
$value = \preg_replace('/[^a-z0-9_.]|^\.|\.$/', '_', \strtolower($value));
$nval = \preg_replace('/[^a-z0-9_.]|^\.|\.$/', '_', \strtolower($nval));
// Min length 4
$value = \str_pad($value, 4, '_');
$nval = \str_pad($nval, $min, \sprintf('#%s', \hash('md5', $value)));
$nval = \substr($nval, 0, $max);
// Max length 30
$value = \substr($value, 0, 30);
if (strlen(str_replace('_', '', $value)) < 1) {
if (strlen(str_replace('_', '', $nval)) < 1) {
throw new \Exception(\sprintf('The string \'%s\' could not be safely normalized for a handle', $value));
}
return $value;
return $nval;
}
}