<commit_before><?php
/**
 * This file is part of the data-transfer-bundle
 *
 * (c) Kuborgh GmbH
 *
 * For the full copyright and license information, please view the LICENSE.txt
 * file that was distributed with this source code.
 */

namespace Kuborgh\DataTransferBundle\Command;

use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Process\Process;

/**
 * Command to fetch live data according to the configured parameters
 */
class FetchCommand extends AbstractCommand
{
    /**
     * Regex to check if the remote dump is ok
     * must start with '-- MySQL dump'
     */
    const VALID_DUMP_REGEX_1 = '/^\-\- MySQL dump/';

    /**
     * Regex to check if the remote dump is ok
     * must end with '-- Dump completed'
     */
    const VALID_DUMP_REGEX_2 = '/\-\- Dump completed on\s+\d*\-\d*\-\d*\s+\d+\:\d+\:\d+[\r\n\s\t]*$/';

    /**
     * Configure command
     */
    protected function configure()
    {
        $this
            ->setName('data-transfer:fetch')
            ->setDescription('Fetch remote database and files from configured system.')
            ->addOption('db-only', 'db-only', InputOption::VALUE_NONE, 'Only transfer the database, not the files.')
            ->addOption(
                'files-only',
                'files-only',
                InputOption::VALUE_NONE,
                'Only transfer the files, not the database.'
            );
    }

    /**
     * Execute the command
     *
     * @param \Symfony\Component\Console\Input\InputInterface   $input  Input
     * @param \Symfony\Component\Console\Output\OutputInterface $output Output
     *
     * @return void
     */
    protected function execute(InputInterface $input, OutputInterface $output)
    {
        parent::execute($input, $output);

        // Fetch and import live database
        if (!$input->getOption('files-only')) {
            try {
                $this->fetchDatabase();
            } catch (\Exception $exc) {
                $this->progressErr($exc->getMessage());
            }
            $this->progressDone();
        }

        // fetch live data files
        if (!$input->getOption('db-only')) {
            try {
                $this->fetchFiles();
            } catch (\Exception $exc) {
                $this->progressErr($exc->getMessage());
            }
            $this->progressDone();
        }

    }

    /**
     * Log in to the remote server and dump the database.
     */
    protected function fetchDatabase()
    {
        $this->output->writeln('Fetching database');

        // Prepare remote command
        $remoteHost = $this->getParam('remote.host');
        $remoteUser = $this->getParam('remote.user');
        $remoteDir = $this->getParam('remote.dir');
        $remoteEnv = $this->getParam('remote.env');
        $consoleCmd = $this->getParam('console_script');
        $options = $this->getParam('ssh.options');

        // Check for ssh proxy
        $sshProxyString = $this->getSshProxyOption();
        if ($sshProxyString) {
            $options[] = $sshProxyString;
        }

        $exportCmd = sprintf(
            'ssh %s %s@%s "cd %s ; %s %s data-transfer:export 2>&1"',
            implode(' ', $options),
            $remoteUser,
            $remoteHost,
            $remoteDir,
            $consoleCmd,
            $remoteEnv ? '--env=' . $remoteEnv : ''
        );
        $this->progress();

        // Execute command
        $process = new Process($exportCmd);
        $process->setTimeout(null);
        $bytes = 0;
        // Update status for each megabyte
        $process->run(
            function ($type, $buffer) use (&$bytes) {
                $bytes += strlen($buffer);
                if ($bytes / 1024 / 1024 >= 1) {
                    $this->progress();
                    $bytes = 0;
                }
            }
        );

        // Check for error
        if (!$process->isSuccessful()) {
            throw new \Exception(sprintf(
                'Cannot connect to remote host: %s %s',
                $process->getOutput(),
                $process->getErrorOutput()
            ));
        }
        $this->progressOk();

        // Check if we have a valid dump in our output
        // first line must start with '-- MySQL dump' and end with '-- Dump completed'
        $sqlDump = $process->getOutput();
        if (!preg_match(self::VALID_DUMP_REGEX_1, $sqlDump) || !preg_match(self::VALID_DUMP_REGEX_2, $sqlDump)) {
            throw new \Exception(sprintf('Error on remote host: %s', $process->getOutput()));
        }
        $this->progressOk();

        // Save dump to temporary file
        $tmpFile = $this->getContainer()->getParameter('kernel.cache_dir') . '/data-transfer.sql';
        file_put_contents($tmpFile, $sqlDump);
        $this->progressDone();

        // Import database
        $this->output->writeln('Importing database');

        // Fetch db connection data
        $siteaccess = $this->getContainer()->getParameter('data_transfer_bundle.siteaccess');
        $dbParams = $this->getContainer()->getParameter(sprintf('ezsettings.%s.database.params', $siteaccess));
        $dbName = $dbParams['database'];
        $dbUser = $dbParams['user'];
        $dbPass = $dbParams['password'];
        $dbHost = $dbParams['host'];

        // Import Dump
        $importCmd = sprintf(
            'mysql %s --user=%s --password=%s --host=%s < %s 2>&1',
            escapeshellarg($dbName),
            escapeshellarg($dbUser),
            escapeshellarg($dbPass),
            escapeshellarg($dbHost),
            escapeshellarg($tmpFile)
        );
        $this->progress();

        $process = new Process($importCmd);
        $process->setTimeout(null);
        // Update status for each megabyte
        $process->run();
        $this->progress();

        if (!$process->isSuccessful()) {
            throw new \Exception(sprintf(
                'Error importing database: %s %s',
                $process->getOutput(),
                $process->getErrorOutput()
            ));
        }
        $this->progressOk();
<commit_msg>fixed database parameter ez publish >= 2014.05<commit_after><?php
/**
 * This file is part of the data-transfer-bundle
 *
 * (c) Kuborgh GmbH
 *
 * For the full copyright and license information, please view the LICENSE.txt
 * file that was distributed with this source code.
 */

namespace Kuborgh\DataTransferBundle\Command;

use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Process\Process;

/**
 * Command to fetch live data according to the configured parameters
 */
class FetchCommand extends AbstractCommand
{
    /**
     * Regex to check if the remote dump is ok
     * must start with '-- MySQL dump'
     */
    const VALID_DUMP_REGEX_1 = '/^\-\- MySQL dump/';

    /**
     * Regex to check if the remote dump is ok
     * must end with '-- Dump completed'
     */
    const VALID_DUMP_REGEX_2 = '/\-\- Dump completed on\s+\d*\-\d*\-\d*\s+\d+\:\d+\:\d+[\r\n\s\t]*$/';

    /**
     * Configure command
     */
    protected function configure()
    {
        $this
            ->setName('data-transfer:fetch')
            ->setDescription('Fetch remote database and files from configured system.')
            ->addOption('db-only', 'db-only', InputOption::VALUE_NONE, 'Only transfer the database, not the files.')
            ->addOption(
                'files-only',
                'files-only',
                InputOption::VALUE_NONE,
                'Only transfer the files, not the database.'
            );
    }

    /**
     * Execute the command
     *
     * @param \Symfony\Component\Console\Input\InputInterface   $input  Input
     * @param \Symfony\Component\Console\Output\OutputInterface $output Output
     *
     * @return void
     */
    protected function execute(InputInterface $input, OutputInterface $output)
    {
        parent::execute($input, $output);

        // Fetch and import live database
        if (!$input->getOption('files-only')) {
            try {
                $this->fetchDatabase();
            } catch (\Exception $exc) {
                $this->progressErr($exc->getMessage());
            }
            $this->progressDone();
        }

        // fetch live data files
        if (!$input->getOption('db-only')) {
            try {
                $this->fetchFiles();
            } catch (\Exception $exc) {
                $this->progressErr($exc->getMessage());
            }
            $this->progressDone();
        }

    }

    /**
     * Log in to the remote server and dump the database.
     */
    protected function fetchDatabase()
    {
        $this->output->writeln('Fetching database');

        // Prepare remote command
        $remoteHost = $this->getParam('remote.host');
        $remoteUser = $this->getParam('remote.user');
        $remoteDir = $this->getParam('remote.dir');
        $remoteEnv = $this->getParam('remote.env');
        $consoleCmd = $this->getParam('console_script');
        $options = $this->getParam('ssh.options');

        // Check for ssh proxy
        $sshProxyString = $this->getSshProxyOption();
        if ($sshProxyString) {
            $options[] = $sshProxyString;
        }

        $exportCmd = sprintf(
            'ssh %s %s@%s "cd %s ; %s %s data-transfer:export 2>&1"',
            implode(' ', $options),
            $remoteUser,
            $remoteHost,
            $remoteDir,
            $consoleCmd,
            $remoteEnv ? '--env=' . $remoteEnv : ''
        );
        $this->progress();

        // Execute command
        $process = new Process($exportCmd);
        $process->setTimeout(null);
        $bytes = 0;
        // Update status for each megabyte
        $process->run(
            function ($type, $buffer) use (&$bytes) {
                $bytes += strlen($buffer);
                if ($bytes / 1024 / 1024 >= 1) {
                    $this->progress();
                    $bytes = 0;
                }
            }
        );

        // Check for error
        if (!$process->isSuccessful()) {
            throw new \Exception(sprintf(
                'Cannot connect to remote host: %s %s',
                $process->getOutput(),
                $process->getErrorOutput()
            ));
        }
        $this->progressOk();

        // Check if we have a valid dump in our output
        // first line must start with '-- MySQL dump' and end with '-- Dump completed'
        $sqlDump = $process->getOutput();
        if (!preg_match(self::VALID_DUMP_REGEX_1, $sqlDump) || !preg_match(self::VALID_DUMP_REGEX_2, $sqlDump)) {
            throw new \Exception(sprintf('Error on remote host: %s', $process->getOutput()));
        }
        $this->progressOk();

        // Save dump to temporary file
        $tmpFile = $this->getContainer()->getParameter('kernel.cache_dir') . '/data-transfer.sql';
        file_put_contents($tmpFile, $sqlDump);
        $this->progressDone();

        // Import database
        $this->output->writeln('Importing database');

        // Fetch db connection data
        $siteaccess = $this->getContainer()->getParameter('data_transfer_bundle.siteaccess');
        $repository = $this->getContainer()->getParameter(sprintf('ezsettings.%s.repository', $siteaccess));
        $repositories = $this->getContainer()->getParameter('ezpublish.repositories');
        $connection = $repositories[$repository]['connection'];
        /** @var $dbalConnection Connection */
        $dbalConnection = $this->getContainer()->get(sprintf('doctrine.dbal.%s_connection', $connection));

        $dbName = $dbalConnection->getDatabase();
        $dbUser = $dbalConnection->getUsername();
        $dbPass = $<PASSWORD>();
        $dbHost = $dbalConnection->getHost();

        // Import Dump
        $importCmd = sprintf(
            'mysql %s --user=%s --password=%s --host=%s < %s 2>&1',
            escapeshellarg($dbName),
            escapeshellarg($dbUser),
            escapeshellarg($dbPass),
            escapeshellarg($dbHost),
            escapeshellarg($tmpFile)
        );
        $this->progress();

        $process = new Process($importCmd);
        $process->setTimeout(null);
        // Update status for each megabyte
        $process->run();
        $this->progress();

        if (!$process->isSuccessful()) {
            throw new \Exception(sprintf(
                'Error importing database: %s %s',
                $process->getOutput(),
                $process->getErrorOutput()
            ));
        }
        $this->progressOk();

{'fraction_non_alphanumeric': 0.14351815543878912, 'fraction_numerical': 0.003450709748255039, 'mean_word_length': 1.9294739260280267, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 120, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2097580', 'n_tokens_mistral': 3575, 'n_tokens_neox': 3460, 'n_words': 805}