<commit_before><?php
/**
 * DB package
 *
 * Classes for work with database
 *
 * @package DB
 * @since Version 1.0
 *
 */
namespace DB;
use \Exception;

/**
 * Class Transfer
 * @package DB
 * @version 1.0
 * @since 1.0
 * @author <NAME> <<EMAIL>>
 */
class Transfer {

    /** @var BasePDO $fromDB */
    private $fromDB = null;
    /** @var BasePDO $toDB */
    private $toDB = null;

    private $portion = 10;

    private static $availableActions = [
        'rewrite' => 'rewrite',
        'rewriteOrClear' => 'rewriteOrClear',
        'skip' => 'skip',
        'stop' => 'stop',
        'append' => 'append',
    ];

    /**
     * What should be done if table exists
     * @var string
     */
    private $existsAction = 'rewrite';

    /**
     * @return string
     */
    public function getExistsAction() {
        return $this->existsAction;
    }

    /**
     * @param string $existsAction
     * @throws \Exception
     */
    public function setExistsAction($existsAction) {
        if (isset(self::$availableActions[strtolower($existsAction)])) {
            $this->existsAction = self::$availableActions[strtolower($existsAction)];
        } else {
            throw new Exception('Undefined action, action can be ' . implode(',', self::$availableActions));
        }
    }

    /**
     * Creates new instance
     * @param BasePDO|array $fromDB
     * @param BasePDO|array $toDB
     * @throws \Exception
     */
    public function __construct($fromDB, $toDB) {
        if ($fromDB instanceof BasePDO) {
            $this->fromDB = $fromDB;
        } elseif(is_array($fromDB)) {
            $this->fromDB = BasePDO::construct($fromDB);
        } else {
            throw new \Exception('Incorrect fromDB param!');
        }

        if ($toDB instanceof BasePDO) {
            $this->toDB = $toDB;
        } elseif(is_array($toDB)) {
            $this->toDB = BasePDO::construct($toDB);
        } else {
            throw new \Exception('Incorrect toDB param!');
        }
    }

    /**
     * Processes each field if necessary by handler at params array and PDO quote
     * @param array $row
     * @param array $columnsMap [oldFieldName => newFieldName]
     * @param array $params [handler => callable, sourceColumns => array]
     * @return array
     */
    private function processFields($row, $columnsMap, $params) {
        $newRow = [];
        foreach($row as $name => &$field) {
            if ((false !== $columnsMap) && !isset($columnsMap[$name])) {
                continue;
            }
            $newName = ((false !== $columnsMap) && isset($columnsMap[$name])) ? $columnsMap[$name] : $name;
            if (isset($params['handler'])) {
                $type = isset($params['sourceColumns'][$name]['type']) ? $params['sourceColumns'][$name]['type'] : false;
                $field = call_user_func($params['handler'], $name, $type, $field);
            }
            $newNames = is_array($newName) ? $newName : [$newName];
            foreach ($newNames as $newName) {
                $newRow[$newName] = $this->toDB->quote($field);
            }
        }
        return $newRow;
    }

    /**
     * Prepare columns for create new table
     * @param array $columns
     * @param array $params
     * @return mixed
     */
    public function processColumns($columns, $params) {
        $rules = ['default' => 'default', 'whitelist' => 'whitelist', 'blacklist' => 'blacklist'];
        if (isset($params['fields']) && is_array($params['fields'])) {
            $fieldsRule = isset($params['fields'][0]) && isset($rules[strtolower($params['fields'][0])]) ?
                $rules[strtolower($params['fields'][0])] : reset($rules);
            unset($params['fields'][0]);
        } elseif(isset($params['fields']) && is_callable($params['fields'])) {
            $fieldsRule = $params['fields'];
        }

        if (isset($params['types']) && is_array($params['types'])) {
            $typesRule = isset($params['types'][0]) && isset($rules[strtolower($params['types'][0])]) ?
                $rules[strtolower($params['types'][0])] : reset($rules);
            unset($params['types'][0]);
        }

        $processedColumns = [];
        $ind = 0;
        foreach ($columns as $name => $column) {
            if (isset($fieldsRule) && is_callable($fieldsRule)) {
                if ($result = call_user_func($fieldsRule, $ind, $name, $column)) {
                    $processedColumns = array_merge($result, $processedColumns);
                }
            } elseif (isset($fieldsRule)) {
                if ('whitelist' === $fieldsRule) {
                    if (isset($params['fields'][$name])) {
                        $processedColumns[$params['fields'][$name]] = $column;
                    }
                } elseif('blacklist' === $fieldsRule) {
                    if (!in_array($name, $params['fields'])) {
                        $processedColumns[$params['fields'][$name]] = $column;
                    }
                } elseif (isset($params['fields'][$name])) {
                    $processedColumns[$params['fields'][$name]] = $column;
                }
            }

            if (isset($typesRule)) {
                if ('whitelist' === $typesRule) {
                    $found = false;
                    foreach ($params['types'] as $oldType => $newType) {
                        if (strstr($column['type'], $oldType)) {
                            $found = true;
                            $columns[$name]['type'] = $newType;
                            break;
                        }
                    }
                    if (!$found) {
                        unset($columns[$name]);
                    }
                } elseif('blacklist' === $typesRule) {
                    $found = false;
                    foreach ($params['types'] as $oldType => $newType) {
                        if (strstr($column['type'], $oldType)) {
                            $found = true;
                            break;
                        }
                    }
                    if ($found) {
                        unset($columns[$name]);
                    }
                } else {
                    foreach ($params['types'] as $oldType => $newType) {
                        if (strstr($column['type'], $oldType)) {
                            $columns[$name]['type'] = $newType;
                            break;
                        }
                    }
                }
            }
            ++$ind;
        }

        return $processedColumns;
    }

    /**
     * Creates array of columns map, that consist of pairs oldFieldName => newFieldName
     * @param array $columns
     * @param array $params
     * @return array|false
     */
    public function createColumnsMap($columns, $params) {
        $columnsMap = [];

        $rules = ['default' => 'default', 'whitelist' => 'whitelist', 'blacklist' => 'blacklist'];
        if (isset($params['fields']) && is_array($params['fields'])) {
            $fieldsRule = isset($params['fields'][0]) && isset($rules[strtolower($params['fields'][0])]) ?
                $rules[strtolower($params['fields'][0])] : reset($rules);
            unset($params['fields'][0]);
        } elseif(isset($params['fields']) && is_callable($params['fields'])) {
            $fieldsRule = $params['fields'];
        }

        if (isset($params['types']) && is_array($params['types'])) {
            $typesRule = isset($params['types'][0]) && isset($rules[strtolower($params['types'][0])]) ?
                $rules[strtolower($params['types'][0])] : reset($rules);
            unset($params['types'][0]);
        }

        if (!isset($fieldsRule) && !isset($typesRule)) {
            return false;
        }<commit_msg>Remove array_merge effect<commit_after><?php
/**
 * DB package
 *
 * Classes for work with database
 *
 * @package DB
 * @since Version 1.0
 *
 */
namespace DB;
use \Exception;

/**
 * Class Transfer
 * @package DB
 * @version 1.0
 * @since 1.0
 * @author <NAME> <<EMAIL>>
 */
class Transfer {

    /** @var BasePDO $fromDB */
    private $fromDB = null;
    /** @var BasePDO $toDB */
    private $toDB = null;

    private $portion = 10;

    private static $availableActions = [
        'rewrite' => 'rewrite',
        'rewriteOrClear' => 'rewriteOrClear',
        'skip' => 'skip',
        'stop' => 'stop',
        'append' => 'append',
    ];

    /**
     * What should be done if table exists
     * @var string
     */
    private $existsAction = 'rewrite';

    /**
     * @return string
     */
    public function getExistsAction() {
        return $this->existsAction;
    }

    /**
     * @param string $existsAction
     * @throws \Exception
     */
    public function setExistsAction($existsAction) {
        if (isset(self::$availableActions[strtolower($existsAction)])) {
            $this->existsAction = self::$availableActions[strtolower($existsAction)];
        } else {
            throw new Exception('Undefined action, action can be ' . implode(',', self::$availableActions));
        }
    }

    /**
     * Creates new instance
     * @param BasePDO|array $fromDB
     * @param BasePDO|array $toDB
     * @throws \Exception
     */
    public function __construct($fromDB, $toDB) {
        if ($fromDB instanceof BasePDO) {
            $this->fromDB = $fromDB;
        } elseif(is_array($fromDB)) {
            $this->fromDB = BasePDO::construct($fromDB);
        } else {
            throw new \Exception('Incorrect fromDB param!');
        }

        if ($toDB instanceof BasePDO) {
            $this->toDB = $toDB;
        } elseif(is_array($toDB)) {
            $this->toDB = BasePDO::construct($toDB);
        } else {
            throw new \Exception('Incorrect toDB param!');
        }
    }

    /**
     * Processes each field if necessary by handler at params array and PDO quote
     * @param array $row
     * @param array $columnsMap [oldFieldName => newFieldName]
     * @param array $params [handler => callable, sourceColumns => array]
     * @return array
     */
    private function processFields($row, $columnsMap, $params) {
        $newRow = [];
        foreach($row as $name => &$field) {
            if ((false !== $columnsMap) && !isset($columnsMap[$name])) {
                continue;
            }
            $newName = ((false !== $columnsMap) && isset($columnsMap[$name])) ? $columnsMap[$name] : $name;
            if (isset($params['handler'])) {
                $type = isset($params['sourceColumns'][$name]['type']) ? $params['sourceColumns'][$name]['type'] : false;
                $field = call_user_func($params['handler'], $name, $type, $field);
            }
            $newNames = is_array($newName) ? $newName : [$newName];
            foreach ($newNames as $newName) {
                $newRow[$newName] = $this->toDB->quote($field);
            }
        }
        return $newRow;
    }

    /**
     * Prepare columns for create new table
     * @param array $columns
     * @param array $params
     * @return mixed
     */
    public function processColumns($columns, $params) {
        $rules = ['default' => 'default', 'whitelist' => 'whitelist', 'blacklist' => 'blacklist'];
        if (isset($params['fields']) && is_array($params['fields'])) {
            $fieldsRule = isset($params['fields'][0]) && isset($rules[strtolower($params['fields'][0])]) ?
                $rules[strtolower($params['fields'][0])] : reset($rules);
            unset($params['fields'][0]);
        } elseif(isset($params['fields']) && is_callable($params['fields'])) {
            $fieldsRule = $params['fields'];
        }

        if (isset($params['types']) && is_array($params['types'])) {
            $typesRule = isset($params['types'][0]) && isset($rules[strtolower($params['types'][0])]) ?
                $rules[strtolower($params['types'][0])] : reset($rules);
            unset($params['types'][0]);
        }

        $processedColumns = [];
        $ind = 0;
        foreach ($columns as $name => $column) {
            if (isset($fieldsRule) && is_callable($fieldsRule)) {
                if ($result = call_user_func($fieldsRule, $ind, $name, $column)) {
                    $processedColumns = array_merge($result, $processedColumns);
                }
            } elseif (isset($fieldsRule)) {
                if ('whitelist' === $fieldsRule) {
                    if (isset($params['fields'][$name])) {
                        $processedColumns[$params['fields'][$name]] = $column;
                    }
                } elseif('blacklist' === $fieldsRule) {
                    if (!in_array($name, $params['fields'])) {
                        $processedColumns[$params['fields'][$name]] = $column;
                    }
                } elseif (isset($params['fields'][$name])) {
                    $processedColumns[$params['fields'][$name]] = $column;
                }
            }

            if (isset($typesRule)) {
                if ('whitelist' === $typesRule) {
                    $found = false;
                    foreach ($params['types'] as $oldType => $newType) {
                        if (strstr($column['type'], $oldType)) {
                            $found = true;
                            $columns[$name]['type'] = $newType;
                            break;
                        }
                    }
                    if (!$found) {
                        unset($columns[$name]);
                    }
                } elseif('blacklist' === $typesRule) {
                    $found = false;
                    foreach ($params['types'] as $oldType => $newType) {
                        if (strstr($column['type'], $oldType)) {
                            $found = true;
                            break;
                        }
                    }
                    if ($found) {
                        unset($columns[$name]);
                    }
                } else {
                    foreach ($params['types'] as $oldType => $newType) {
                        if (strstr($column['type'], $oldType)) {
                            $columns[$name]['type'] = $newType;
                            break;
                        }
                    }
                }
            }
            ++$ind;
        }

        return array_reverse($processedColumns);
    }

    /**
     * Creates array of columns map, that consist of pairs oldFieldName => newFieldName
     * @param array $columns
     * @param array $params
     * @return array|false
     */
    public function createColumnsMap($columns, $params) {
        $columnsMap = [];

        $rules = ['default' => 'default', 'whitelist' => 'whitelist', 'blacklist' => 'blacklist'];
        if (isset($params['fields']) && is_array($params['fields'])) {
            $fieldsRule = isset($params['fields'][0]) && isset($rules[strtolower($params['fields'][0])]) ?
                $rules[strtolower($params['fields'][0])] : reset($rules);
            unset($params['fields'][0]);
        } elseif(isset($params['fields']) && is_callable($params['fields'])) {
            $fieldsRule = $params['fields'];
        }

        if (isset($params['types']) && is_array($params['types'])) {
            $typesRule = isset($params['types'][0]) && isset($rules[strtolower($params['types'][0])]) ?
                $rules[strtolower($params['types'][0])] : reset($rules);
            unset($params['types'][0]);
        }

        if (!isset($fieldsRule) && !isset($typesRule)) {
            return false;
        }
{'fraction_non_alphanumeric': 0.16644191870545175, 'fraction_numerical': 0.0032106851602131895, 'mean_word_length': 1.7855482024682525, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1415105', 'n_tokens_mistral': 4302, 'n_tokens_neox': 4222, 'n_words': 840}