<commit_before><?php
class Vps_Controller_Action_Cli_ImportController extends Vps_Controller_Action_Cli_Abstract
{
    public function indexAction()
    {
        $ownConfig = Vps_Registry::get('config');

        $dbConfig = Zend_Registry::get('db')->getConfig();

        $mysqlLocalOptions = "--host=$dbConfig[host] ";
        //auskommentiert weil: es muss in ~/.my.cnf ein benutzer der das machen darf eingestellt sein!
        //ansonsten gibt es probleme für das erstellen von triggers, dazu benötigt man SUPER priviliges
        // -> scheiß mysql
        //$mysqlLocalOptions .= "--user={$dbConfig->username} --password={$dbConfig->password} ";

        $server = $this->_getParam('server');
        $config = new Zend_Config_Ini('application/config.ini', $server);
        if (!$config->server || !$config->server->host) {
            throw new Vps_ClientException("kein server konfiguriert");
        }
        $this->_sshHost = $config->server->user.'@'.$config->server->host;
        $this->_sshDir = $config->server->dir;

        if (Vps_Registry::get('config')->application->id != 'service') {
            $this->_copyServiceUsers();
        }

        if ($ownConfig->server->host == $config->server->host) {
            $cmd = "cd {$config->server->dir} && php bootstrap.php import get-update-revision";
        } else {
            $cmd = "sudo -u vps sshvps $this->_sshHost $this->_sshDir import get-update-revision";
        }
        if ($this->_getParam('debug')) echo $cmd."\n";
        exec($cmd, $onlineRevision, $ret);
        if ($ret != 0) throw new Vps_ClientException();
        $onlineRevision = implode('', $onlineRevision);

        if (!$onlineRevision) {
            throw new Vps_ClientException("Can't get onlineRevision");
        }

        if ($config->uploads && $ownConfig->uploads) {
            echo "kopiere uploads...\n";
            if ($ownConfig->server->host == $config->server->host) {
                if ($ownConfig->uploads == $config->uploads) {
                    throw new Vps_ClientException("Uplodas-Pfade für beide Server sind gleich!");
                }
                $this->_systemCheckRet("rsync --progress --delete --times --exclude=cache/ --recursive {$config->uploads} {$ownConfig->uploads}");
            } else {
                $this->_systemSshVps('copy-uploads '.$ownConfig->uploads.'/', $config->uploads);
            }
        }

        if ($config->server->import && $config->server->import->dirs) {
            foreach ($config->server->import->dirs as $dir) {
                echo "importing $dir...\n";
                $ig = simplexml_load_string(`svn propget --recursive --xml svn:ignore $dir`);
                $ignores = array();
                foreach ($ig->target as $t) {
                    $p = explode("\n", trim((string)$t->property));
                    foreach ($p as $i) {
                        $ignores[] = (string)$t['path'] . '/' . trim($i);
                    }
                }
                if (!$ignores) continue;

                $includes = array();
                foreach ($ignores as $i) {
                    $p = '';
                    foreach (explode('/', $i) as $j) {
                        $p .= $j.'/';
                        $e = trim($p, '/');
                        if (substr($e, -1) == '*') $e .= '*';
                        if (!in_array($e, $includes)) {
                            $includes[] = $e;
                        }
                    }
                }

                $cmd = "sudo -u vps sshvps $this->_sshHost $this->_sshDir copy-files";
                $cmd .= " --includes=\"".implode(',', $includes)."\"";
                if ($this->_getParam('debug')) $cmd .= " --debug";
                if ($this->_getParam('debug')) echo $cmd."\n";
                passthru($cmd);
            }
        }

        echo "erstelle datenbank-backup...\n";

        $cacheTables = Vps_Util_ClearCache::getInstance()->getDbCacheTables();
        if ($config->import && $config->import->ignoreTables) {
            foreach ($config->import->ignoreTables as $t) {
                $cacheTables[] = $t;
            }
        }

        $tables = Zend_Registry::get('db')->fetchCol('SHOW TABLES');
        $keepTables = array();
        if ($config->server->import && $config->server->import->keepTables) {
            foreach ($config->server->import->keepTables as $t) {
                if (substr($t, -1) == '*') {
                    foreach ($tables as $table) {
                        if (substr($table, 0, strlen($t)-1) == substr($t, 0, -1)) {
                            $keepTables[] = $table;
                        }
                    }
                } else {
                    $keepTables[] = $t;
                }
            }
        }

        $dumpname = $this->_backupDb(array_merge($cacheTables, $keepTables));
        $backupSize = filesize($dumpname);
        $this->_systemCheckRet("bzip2 --fast $dumpname");
        echo $dumpname.".bz2\n";


        if ($keepTables) {
            echo "erstelle dump fr KeepTables...\n";
            $keepTablesDump = tempnam('/tmp', 'importkeep');
            $cmd = "mysqldump --add-drop-table=false --no-create-info=true $mysqlLocalOptions $dbConfig[dbname] ".implode(' ', $keepTables).">> $keepTablesDump";
            if ($this->_getParam('debug')) file_put_contents('php://stderr', "$cmd\n");
            $this->_systemCheckRet($cmd);
        }

        echo "loesche lokale datenbank...\n";
        $this->_systemCheckRet("echo \"DROP DATABASE \`$dbConfig[dbname]\`\" | mysql $mysqlLocalOptions");

        echo "erstelle neue datenbank...\n";
        $this->_systemCheckRet("echo \"CREATE DATABASE \`$dbConfig[dbname]\`\" | mysql $mysqlLocalOptions");


        echo "importiere datenbank...\n";
        if ($ownConfig->server->host == $config->server->host) {
            $otherDbConfig = new Zend_Config_Ini($config->server->dir.'/application/config.db.ini', 'database');
            $otherDbConfig = $otherDbConfig->web;
            $cmd = $this->_getDumpCommand($otherDbConfig, array_merge($cacheTables, $keepTables));
        } else {
            $ignoreTables = '';
            if (!$this->_getParam('include-cache')) {
                $ignoreTables = Vps_Util_ClearCache::getInstance()->getDbCacheTables();
                if ($config->server->import && $config->server->import->ignoreTables) {
                    foreach ($config->server->import->ignoreTables as $t) {
                        $ignoreTables[] = $t;
                    }
                }
                $ignoreTables = implode(',', array_merge($ignoreTables, $keepTables));
                if ($ignoreTables) $ignoreTables = " --ignore-tables=$ignoreTables";
            }
            $cmd = "sudo -u vps sshvps $this->_sshHost $this->_sshDir db-dump";
            $cmd .= "$ignoreTables";
            if ($this->_getParam('debug')) $cmd .= " --debug";<commit_msg>fixes import<commit_after><?php
class Vps_Controller_Action_Cli_ImportController extends Vps_Controller_Action_Cli_Abstract
{
    public function indexAction()
    {
        $ownConfig = Vps_Registry::get('config');

        $dbConfig = Zend_Registry::get('db')->getConfig();

        $mysqlLocalOptions = "--host=$dbConfig[host] ";
        //auskommentiert weil: es muss in ~/.my.cnf ein benutzer der das machen darf eingestellt sein!
        //ansonsten gibt es probleme für das erstellen von triggers, dazu benötigt man SUPER priviliges
        // -> scheiß mysql
        //$mysqlLocalOptions .= "--user={$dbConfig->username} --password={$dbConfig->password} ";

        $server = $this->_getParam('server');
        $config = new Zend_Config_Ini('application/config.ini', $server);
        if (!$config->server || !$config->server->host) {
            throw new Vps_ClientException("kein server konfiguriert");
        }
        $this->_sshHost = $config->server->user.'@'.$config->server->host;
        $this->_sshDir = $config->server->dir;

        if (Vps_Registry::get('config')->application->id != 'service') {
            $this->_copyServiceUsers();
        }

        if ($ownConfig->server->host == $config->server->host) {
            $cmd = "cd {$config->server->dir} && php bootstrap.php import get-update-revision";
        } else {
            $cmd = "sudo -u vps sshvps $this->_sshHost $this->_sshDir import get-update-revision";
        }
        if ($this->_getParam('debug')) echo $cmd."\n";
        exec($cmd, $onlineRevision, $ret);
        if ($ret != 0) throw new Vps_ClientException();
        $onlineRevision = implode('', $onlineRevision);

        if (!$onlineRevision) {
            throw new Vps_ClientException("Can't get onlineRevision");
        }

        if ($config->uploads && $ownConfig->uploads) {
            echo "kopiere uploads...\n";
            if ($ownConfig->server->host == $config->server->host) {
                if ($ownConfig->uploads == $config->uploads) {
                    throw new Vps_ClientException("Uplodas-Pfade für beide Server sind gleich!");
                }
                $this->_systemCheckRet("rsync --progress --delete --times --exclude=cache/ --recursive {$config->uploads} {$ownConfig->uploads}");
            } else {
                $this->_systemSshVps('copy-uploads '.$ownConfig->uploads.'/', $config->uploads);
            }
        }

        if ($config->server->import && $config->server->import->dirs) {
            foreach ($config->server->import->dirs as $dir) {
                echo "importing $dir...\n";
                $ig = simplexml_load_string(`svn propget --recursive --xml svn:ignore $dir`);
                $ignores = array();
                foreach ($ig->target as $t) {
                    $p = explode("\n", trim((string)$t->property));
                    foreach ($p as $i) {
                        $ignores[] = (string)$t['path'] . '/' . trim($i);
                    }
                }
                if (!$ignores) continue;

                $includes = array();
                foreach ($ignores as $i) {
                    $p = '';
                    foreach (explode('/', $i) as $j) {
                        $p .= $j.'/';
                        $e = trim($p, '/');
                        if (substr($e, -1) == '*') $e .= '*';
                        if (!in_array($e, $includes)) {
                            $includes[] = $e;
                        }
                    }
                }

                $cmd = "sudo -u vps sshvps $this->_sshHost $this->_sshDir copy-files";
                $cmd .= " --includes=\"".implode(',', $includes)."\"";
                if ($this->_getParam('debug')) $cmd .= " --debug";
                if ($this->_getParam('debug')) echo $cmd."\n";
                passthru($cmd);
            }
        }

        echo "erstelle datenbank-backup...\n";

        $tables = Zend_Registry::get('db')->fetchCol('SHOW TABLES');

        $cacheTables = Vps_Util_ClearCache::getInstance()->getDbCacheTables();

        if ($config->server->import && $config->server->import->ignoreTables) {
            foreach ($config->server->import->ignoreTables as $t) {
                if (substr($t, -1) == '*') {
                    foreach ($tables as $table) {
                        if (substr($table, 0, strlen($t)-1) == substr($t, 0, -1)) {
                            $cacheTables[] = $table;
                        }
                    }
                } else {
                    $cacheTables[] = $t;
                }
            }
        }

        $keepTables = array();
        if ($config->server->import && $config->server->import->keepTables) {
            foreach ($config->server->import->keepTables as $t) {
                if (substr($t, -1) == '*') {
                    foreach ($tables as $table) {
                        if (substr($table, 0, strlen($t)-1) == substr($t, 0, -1)) {
                            $keepTables[] = $table;
                        }
                    }
                } else {
                    $keepTables[] = $t;
                }
            }
        }

        $dumpname = $this->_backupDb(array_merge($cacheTables, $keepTables));
        $backupSize = filesize($dumpname);
        $this->_systemCheckRet("bzip2 --fast $dumpname");
        echo $dumpname.".bz2\n";


        if ($keepTables) {
            echo "erstelle dump fr KeepTables...\n";
            $keepTablesDump = tempnam('/tmp', 'importkeep');
            $cmd = "mysqldump --add-drop-table=false --no-create-info=true $mysqlLocalOptions $dbConfig[dbname] ".implode(' ', $keepTables).">> $keepTablesDump";
            if ($this->_getParam('debug')) file_put_contents('php://stderr', "$cmd\n");
            $this->_systemCheckRet($cmd);
        }

        echo "loesche lokale datenbank...\n";
        $this->_systemCheckRet("echo \"DROP DATABASE \`$dbConfig[dbname]\`\" | mysql $mysqlLocalOptions");

        echo "erstelle neue datenbank...\n";
        $this->_systemCheckRet("echo \"CREATE DATABASE \`$dbConfig[dbname]\`\" | mysql $mysqlLocalOptions");


        echo "importiere datenbank...\n";
        if ($ownConfig->server->host == $config->server->host) {
            $otherDbConfig = new Zend_Config_Ini($config->server->dir.'/application/config.db.ini', 'database');
            $otherDbConfig = $otherDbConfig->web;
            $cmd = $this->_getDumpCommand($otherDbConfig, array_merge($cacheTables, $keepTables));
        } else {
            $ignoreTables = '';
            if (!$this->_getParam('include-cache')) {
                $ignoreTables = implode(',', array_merge($cacheTables, $keepTables));
                if ($ignoreTables) $ignoreTables = " --ignore-tables=$ignoreTables";
            }
            $cmd = "sudo -u vps sshvps $this->_sshHost $this->_sshDir db-dump";
            $cmd .= "$ignoreTables";
            if ($this->_getParam('debug')) $cmd .= " --debug";
{'fraction_non_alphanumeric': 0.16969871287840657, 'fraction_numerical': 0.0016538433882217589, 'mean_word_length': 1.7694145758661888, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 197, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5099104', 'n_tokens_mistral': 4051, 'n_tokens_neox': 3812, 'n_words': 760}