<commit_before><?php

namespace Oro\Bundle\ImportExportBundle\Tests\Functional;

use Oro\Bundle\ImportExportBundle\Async\Topics;
use Oro\Bundle\ImportExportBundle\Configuration\ImportExportConfigurationInterface;
use Oro\Bundle\ImportExportBundle\File\FileManager;
use Oro\Bundle\ImportExportBundle\Job\JobExecutor;
use Oro\Bundle\ImportExportBundle\Processor\ProcessorRegistry;
use Oro\Bundle\MessageQueueBundle\Entity\Job;
use Oro\Bundle\MessageQueueBundle\Test\Functional\MessageQueueExtension;
use Oro\Bundle\SecurityBundle\Authentication\Token\UsernamePasswordOrganizationToken;
use Oro\Bundle\TestFrameworkBundle\Test\WebTestCase;
use Oro\Bundle\UserBundle\Entity\User;
use Oro\Component\MessageQueue\Client\Message;
use Oro\Component\MessageQueue\Consumption\MessageProcessorInterface;
use Oro\Component\MessageQueue\Transport\Null\NullMessage;
use Oro\Component\MessageQueue\Transport\SessionInterface;
use Symfony\Component\HttpFoundation\File\UploadedFile;

abstract class AbstractImportExportTest extends WebTestCase
{
    use MessageQueueExtension;

    protected function setUp()
    {
        $this->initClient([], static::generateBasicAuthHeader());
        $this->client->useHashNavigation(true);
    }

    /**
     * @param ImportExportConfigurationInterface $configuration
     * @param string                             $expectedCsvFilePath
     */
    protected function assertExportTemplateWorks(
        ImportExportConfigurationInterface $configuration,
        string $expectedCsvFilePath
    ) {
        $this->client->request(
            'GET',
            $this->getUrl('oro_importexport_export_template', [
                'processorAlias' => $configuration->getExportProcessorAlias(),
                'exportTemplateJob' => $configuration->getExportTemplateJobName(),
            ])
        );

        $this->client->followRedirect();

        static::assertContains(
            $this->getFileContent($expectedCsvFilePath),
            $this->client->getResponse()->getContent()
        );

        $this->deleteImportExportFile($this->client->getRequest()->attributes->get('fileName'));
    }

    /**
     * @param ImportExportConfigurationInterface $configuration
     * @param string                             $expectedCsvFilePath
     * @param array                              $columnToIgnoreInExportFile
     */
    protected function assertExportWorks(
        ImportExportConfigurationInterface $configuration,
        string $expectedCsvFilePath,
        array $columnToIgnoreInExportFile = []
    ) {
        $this->assertPreExportActionExecuted($configuration);

        $preExportMessageData = $this->getOneSentMessageWithTopic(Topics::PRE_EXPORT);
        $this->clearMessageCollector();

        $this->assertMessageProcessorExecuted(
            'oro_importexport.async.pre_export',
            $preExportMessageData
        );

        static::assertMessageSent(Topics::EXPORT);

        $exportMessageData = $this->getOneSentMessageWithTopic(Topics::EXPORT);
        $jobId = $exportMessageData['jobId'];
        $this->clearMessageCollector();

        $this->assertMessageProcessorExecuted(
            'oro_importexport.async.export',
            $exportMessageData
        );

        $job = $this->findJob($jobId);
        $exportedFilename = $job->getData()['file'];

        $exportFileContent = $this->getImportExportFileContent($exportedFilename);
        $this->deleteImportExportFile($exportedFilename);

        if (empty($columnToIgnoreInExportFile)) {
            static::assertContains(
                $this->getFileContent($expectedCsvFilePath),
                $exportFileContent
            );
        } else {
            $expectedData = $this->getParsedDataFromCSVFile($expectedCsvFilePath);
            $exportedData = $this->getParsedDataFromCSVContent($exportFileContent);
            $this->removedIgnoredColumnsFromData(
                $exportedData,
                $columnToIgnoreInExportFile
            );

            static::assertEquals(
                $expectedData,
                $exportedData
            );
        }
    }

    /**
     * @param ImportExportConfigurationInterface $configuration
     * @param string                             $importFilePath
     */
    protected function assertImportWorks(
        ImportExportConfigurationInterface $configuration,
        string $importFilePath
    ) {
        $this->assertPreImportActionExecuted($configuration, $importFilePath);<commit_msg>CRM-8711: CSV Import removes data from entities - Added functional tests<commit_after><?php

namespace Oro\Bundle\ImportExportBundle\Tests\Functional;

use Oro\Bundle\ImportExportBundle\Async\Topics;
use Oro\Bundle\ImportExportBundle\Configuration\ImportExportConfigurationInterface;
use Oro\Bundle\ImportExportBundle\File\FileManager;
use Oro\Bundle\ImportExportBundle\Job\JobExecutor;
use Oro\Bundle\ImportExportBundle\Processor\ProcessorRegistry;
use Oro\Bundle\MessageQueueBundle\Entity\Job;
use Oro\Bundle\MessageQueueBundle\Test\Functional\MessageQueueExtension;
use Oro\Bundle\SecurityBundle\Authentication\Token\UsernamePasswordOrganizationToken;
use Oro\Bundle\TestFrameworkBundle\Test\WebTestCase;
use Oro\Bundle\UserBundle\Entity\User;
use Oro\Component\MessageQueue\Client\Message;
use Oro\Component\MessageQueue\Consumption\MessageProcessorInterface;
use Oro\Component\MessageQueue\Transport\Null\NullMessage;
use Oro\Component\MessageQueue\Transport\SessionInterface;
use Symfony\Component\HttpFoundation\File\UploadedFile;

abstract class AbstractImportExportTest extends WebTestCase
{
    use MessageQueueExtension;

    protected function setUp()
    {
        $this->initClient([], static::generateBasicAuthHeader());
        $this->client->useHashNavigation(true);
    }

    /**
     * @param ImportExportConfigurationInterface $configuration
     * @param string                             $expectedCsvFilePath
     */
    protected function assertExportTemplateWorks(
        ImportExportConfigurationInterface $configuration,
        string $expectedCsvFilePath
    ) {
        $this->client->request(
            'GET',
            $this->getUrl('oro_importexport_export_template', [
                'processorAlias' => $configuration->getExportProcessorAlias(),
                'exportTemplateJob' => $configuration->getExportTemplateJobName(),
            ])
        );

        $this->client->followRedirect();

        static::assertContains(
            $this->getFileContent($expectedCsvFilePath),
            $this->client->getResponse()->getContent()
        );

        $this->deleteImportExportFile($this->client->getRequest()->attributes->get('fileName'));
    }

    /**
     * @param ImportExportConfigurationInterface $configuration
     * @param string                             $expectedCsvFilePath
     * @param array                              $skippedColumns
     */
    protected function assertExportWorks(
        ImportExportConfigurationInterface $configuration,
        string $expectedCsvFilePath,
        array $skippedColumns = []
    ) {
        $this->assertPreExportActionExecuted($configuration);

        $preExportMessageData = $this->getOneSentMessageWithTopic(Topics::PRE_EXPORT);
        $this->clearMessageCollector();

        $this->assertMessageProcessorExecuted(
            'oro_importexport.async.pre_export',
            $preExportMessageData
        );

        static::assertMessageSent(Topics::EXPORT);

        $exportMessageData = $this->getOneSentMessageWithTopic(Topics::EXPORT);
        $jobId = $exportMessageData['jobId'];
        $this->clearMessageCollector();

        $this->assertMessageProcessorExecuted(
            'oro_importexport.async.export',
            $exportMessageData
        );

        $job = $this->findJob($jobId);
        $exportedFilename = $job->getData()['file'];

        $exportFileContent = $this->getImportExportFileContent($exportedFilename);
        $this->deleteImportExportFile($exportedFilename);

        if (empty($skippedColumns)) {
            static::assertContains(
                $this->getFileContent($expectedCsvFilePath),
                $exportFileContent
            );
        } else {
            $expectedData = $this->getParsedDataFromCSVFile($expectedCsvFilePath);
            $exportedData = $this->getParsedDataFromCSVContent($exportFileContent);
            $this->removedIgnoredColumnsFromData(
                $exportedData,
                $skippedColumns
            );

            static::assertEquals(
                $expectedData,
                $exportedData
            );
        }
    }

    /**
     * @param ImportExportConfigurationInterface $configuration
     * @param string                             $importFilePath
     */
    protected function assertImportWorks(
        ImportExportConfigurationInterface $configuration,
        string $importFilePath
    ) {
        $this->assertPreImportActionExecuted($configuration, $importFilePath);
{'fraction_non_alphanumeric': 0.1003221864237307, 'fraction_numerical': 0.00044439506721475393, 'mean_word_length': 2.8952834270878407, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '379169', 'n_tokens_mistral': 2247, 'n_tokens_neox': 2189, 'n_words': 294}