<commit_before><?php

namespace opensixt\SxTranslateBundle\Command;

use Doctrine\DBAL\Configuration;
use Doctrine\DBAL\DriverManager;

use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

use opensixt\BikiniTranslateBundle\Entity\Text;
use opensixt\BikiniTranslateBundle\Entity\Resource;
use opensixt\BikiniTranslateBundle\Entity\Language;
use opensixt\BikiniTranslateBundle\Entity\User;
use opensixt\BikiniTranslateBundle\Entity\Role;
use opensixt\BikiniTranslateBundle\Entity\Group;

/**
 * Description of MigrateCommand
 *
 * @author <NAME> <<EMAIL>>
 */
class MigrateCommand extends ContainerAwareCommand
{
    private $res = array();

    private $locale = array();

    private $user = array();

    private $role = array();

    private $group = array();

    private $resToGroup = array();

    /**
     *
     */
    protected function configure()
    {
        $this
            ->setName('sxtranslate:import')
            ->setDescription('Import live data from gtxt')
            ->addArgument('dsn', InputArgument::REQUIRED, 'Database DSN (mysql://username:password@host/database)')
            ->addArgument('max_rows', InputArgument::OPTIONAL, 'How many rows do you want to import');
    }

    /**
     * @param \Symfony\Component\Console\Input\InputInterface $input
     * @param \Symfony\Component\Console\Output\OutputInterface $output
     * @return int|void
     */
    protected function execute(InputInterface $input, OutputInterface $output)
    {
        $dsn = $input->getArgument('dsn');

        $manager = $this->getContainer()->get('doctrine.orm.entity_manager');

        $conn = $this->getConnection($dsn);

        $this->loadGroupsAndResources($conn, $manager);

        $this->loadResourcesAndLocales($conn, $manager);

        // admin role
        $adminRole = $this->getRoleAdmin();
        $manager->persist($adminRole);
        $manager->flush();

        $this->role['admin'] = $adminRole;

        // user role
        $userRole = $this->getRoleUser();
        $manager->persist($userRole);
        $manager->flush();

        $this->role['user'] = $userRole;

        // default group
        $resourceDefault = $this->getResourceDefault();
        $manager->persist($resourceDefault);
        $manager->flush();

        $this->res['default'] = $resourceDefault;

        // default group
        $groupDefault = $this->getGroupDefault();
        $groupDefault->setResources(array($resourceDefault));
        $manager->persist($groupDefault);
        $manager->flush();

        $this->group['default'] = $groupDefault;

        // admin user
        $admin = $this->getUserAdmin();
        $manager->persist($admin);
        $manager->flush();

        $this->user['admin'] = $admin;

        // no user
        $nouser = $this->getUserNoUser();
        $manager->persist($nouser);
        $manager->flush();

        $this->user[''] = $nouser;

        $i = 0;
        gc_enable(); // Enable Garbage Collector

        $max_rows = intval($input->getArgument('max_rows') ?: 0);
        $sql = "SELECT * FROM gtxt" . ($max_rows > 0 ? " LIMIT {$max_rows}" : "");

        $stmt = $conn->query($sql);

        while ($row = $stmt->fetch()) {
            if (isset($this->user[$row['user']])) {
                $user = $this->user[$row['user']];
                $user->addUserLanguage($this->locale[$row['locale']]);
            } else {
                $user = $this->getUserUser(
                    $row['user'],
                    $row['user'],
                    $row['user'] . '@sixt.de',
                    $this->locale[$row['locale']]
                );

                $this->user[$row['user']] = $user;
            }

            // get the group by resource name
            $userGroup = $this->resToGroup[$row['module']];
            $user->addUserGroup($userGroup);

            $manager->persist($user);

            $text = $this->getText($row);
            $manager->persist($text);

            // flush every 500th
            if ($i > 0 && $i % 500 === 0) {
                echo "{$i} rows imported\n";
                $manager->flush();
                gc_collect_cycles();
            }

            $i++;
        }

        $manager->flush();<commit_msg>add acl for new user<commit_after><?php

namespace opensixt\SxTranslateBundle\Command;

use Doctrine\DBAL\Configuration;
use Doctrine\DBAL\DriverManager;

use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

use opensixt\BikiniTranslateBundle\Entity\Text;
use opensixt\BikiniTranslateBundle\Entity\Resource;
use opensixt\BikiniTranslateBundle\Entity\Language;
use opensixt\BikiniTranslateBundle\Entity\User;
use opensixt\BikiniTranslateBundle\Entity\Role;
use opensixt\BikiniTranslateBundle\Entity\Group;

/**
 * Description of MigrateCommand
 *
 * @author <NAME> <<EMAIL>>
 */
class MigrateCommand extends ContainerAwareCommand
{
    private $res = array();

    private $locale = array();

    private $user = array();

    private $role = array();

    private $group = array();

    private $resToGroup = array();

    /**
     *
     */
    protected function configure()
    {
        $this
            ->setName('sxtranslate:import')
            ->setDescription('Import live data from gtxt')
            ->addArgument('dsn', InputArgument::REQUIRED, 'Database DSN (mysql://username:password@host/database)')
            ->addArgument('max_rows', InputArgument::OPTIONAL, 'How many rows do you want to import');
    }

    /**
     * @param \Symfony\Component\Console\Input\InputInterface $input
     * @param \Symfony\Component\Console\Output\OutputInterface $output
     * @return int|void
     */
    protected function execute(InputInterface $input, OutputInterface $output)
    {
        $dsn = $input->getArgument('dsn');

        $manager = $this->getContainer()->get('doctrine.orm.entity_manager');

        $user_permissions = $this->getContainer()->get('opensixt.bikini_translate.acl.user_permissions');

        $conn = $this->getConnection($dsn);

        $this->loadGroupsAndResources($conn, $manager);

        $this->loadResourcesAndLocales($conn, $manager);

        // admin role
        $adminRole = $this->getRoleAdmin();
        $manager->persist($adminRole);
        $manager->flush();

        $this->role['admin'] = $adminRole;

        // user role
        $userRole = $this->getRoleUser();
        $manager->persist($userRole);
        $manager->flush();

        $this->role['user'] = $userRole;

        // default group
        $resourceDefault = $this->getResourceDefault();
        $manager->persist($resourceDefault);
        $manager->flush();

        $this->res['default'] = $resourceDefault;

        // default group
        $groupDefault = $this->getGroupDefault();
        $groupDefault->setResources(array($resourceDefault));
        $manager->persist($groupDefault);
        $manager->flush();

        $this->group['default'] = $groupDefault;

        // admin user
        $admin = $this->getUserAdmin();
        $manager->persist($admin);
        $manager->flush();

        $this->user['admin'] = $admin;

        $user_permissions->initAclForNewUser($admin);

        // no user
        $nouser = $this->getUserNoUser();
        $manager->persist($nouser);
        $manager->flush();

        $this->user[''] = $nouser;

        $user_permissions->initAclForNewUser($nouser);

        $i = 0;
        gc_enable(); // Enable Garbage Collector

        $max_rows = intval($input->getArgument('max_rows') ?: 0);
        $sql = "SELECT * FROM gtxt" . ($max_rows > 0 ? " LIMIT {$max_rows}" : "");

        $stmt = $conn->query($sql);

        $usersToAcl = array();

        while ($row = $stmt->fetch()) {
            if (isset($this->user[$row['user']])) {
                $user = $this->user[$row['user']];
                $user->addUserLanguage($this->locale[$row['locale']]);
            } else {
                $user = $this->getUserUser(
                    $row['user'],
                    $row['user'],
                    $row['user'] . '@sixt.de',
                    $this->locale[$row['locale']]
                );

                $usersToAcl[] = $user;
                $this->user[$row['user']] = $user;
            }

            // get the group by resource name
            $userGroup = $this->resToGroup[$row['module']];
            $user->addUserGroup($userGroup);

            $manager->persist($user);

            $text = $this->getText($row);
            $manager->persist($text);

            // flush every 500th
            if ($i > 0 && $i % 500 === 0) {
                echo "{$i} rows imported\n";
                $manager->flush();

                foreach ($usersToAcl as $user) {
                    $user_permissions->initAclForNewUser($user);
                }
                $usersToAcl = array();

                gc_collect_cycles();
            }

            $i++;
        }

        $manager->flush();
{'fraction_non_alphanumeric': 0.15713522180652056, 'fraction_numerical': 0.0023516835916622125, 'mean_word_length': 2.3083451202263086, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 122, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4737538', 'n_tokens_mistral': 2767, 'n_tokens_neox': 2643, 'n_words': 462}