<commit_before><?php

/*
 * This file is part of the Access to Memory (AtoM) software.
 *
 * Access to Memory (AtoM) is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Access to Memory (AtoM) is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Access to Memory (AtoM).  If not, see <http://www.gnu.org/licenses/>.
 */

/**
 * Check csv data.
 *
 * @author     <NAME> <<EMAIL>>
 * @author     <NAME> <<EMAIL>>
 */
class csvCheckImportTask extends arBaseTask
{
    protected $verbose = false;

    /**
     * @see sfTask
     *
     * @param mixed $arguments
     * @param mixed $options
     */
    public function execute($arguments = [], $options = [])
    {
        parent::execute($arguments, $options);

        $validatorOptions = $this->setOptions($options);

        if (!empty($options['verbose'])) {
            $this->verbose = true;
        }

        $filenames = $this->setCsvValidatorFilenames($arguments['filename']);

        $validator = new CsvImportValidator(
            $this->context, $this->getDbConnection(), $validatorOptions
        );

        $validator->setShowDisplayProgress(true);
        $validator->setFilenames($filenames);
        $results = $validator->validate();
        $output = $results->renderResultsAsText($this->verbose);
        echo $output;

        unset($validator);
    }

    protected function configure()
    {
        $this->addArguments([
            new sfCommandArgument('filename', sfCommandArgument::REQUIRED,
              'The input file name (csv format).'),
        ]);

        $this->addOptions([
            new sfCommandOption(
                'application',
                null,
                sfCommandOption::PARAMETER_OPTIONAL,
                'The application name',
                'qubit'
            ),
            new sfCommandOption(
                'env',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'The environment',
                'cli'
            ),
            new sfCommandOption(
                'connection',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'The connection name',
                'propel'
            ),
            new sfCommandOption(
                'verbose',
                'i',
                sfCommandOption::PARAMETER_NONE,
                'Provide detailed information regarding each test.'
            ),
            new sfCommandOption(
                'source',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'Source name for validating parentId matching against previous imports. If not set, parentId validation against AtoM\'s database will be skipped.'
            ),
            new sfCommandOption(
                'class-name',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'Qubit object type contained in CSV.',
                'QubitInformationObject'
            ),
            new sfCommandOption(
                'specific-tests',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'Specific test classes to run.'
            ),
            new sfCommandOption(
                'path-to-digital-objects',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'Path to root of digital object folder that will match digitalObjectPath in CSV.'
            ),
        ]);

        $this->namespace = 'csv';
        $this->name = 'check-import';
        $this->briefDescription = 'Check CSV data, providing diagnostic info.';
        $this->detailedDescription = <<<'EOF'
    Check CSV data, providing information about it.
EOF;
    }

    protected function getDbConnection()
    {
        $databaseManager = new sfDatabaseManager($this->configuration);

        return $databaseManager->getDatabase('propel')->getConnection();
    }

    protected function setCsvValidatorFilenames($filenameString)
    {
        // Could be a comma separated list of filenames or just one.
        foreach (explode(',', $filenameString) as $filename) {
            CsvImportValidator::validateFileName($filename);
            // The validator expects an associative array of files
            // where displayname => filename
            $filenames[$filename] = $filename;
        }

        return $filenames;
    }

    protected function setOptions($options = [])
    {
        $opts = [];

        $keymap = [
            'source' => 'source',
            'class-name' => 'className',
            'escape' => 'escape',
            'specific-tests' => 'specificTests',
            'path-to-digital-objects' => 'pathToDigitalObjects',
        ];

        foreach ($keymap as $oldkey => $newkey) {
            if (empty($options[$oldkey])) {
                continue;
            }

            $opts[$newkey] = $options[$oldkey];
        }

        return $opts;
    }
}<commit_msg>Add CSV task check dir. of files, refs #13561<commit_after><?php

/*
 * This file is part of the Access to Memory (AtoM) software.
 *
 * Access to Memory (AtoM) is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Access to Memory (AtoM) is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Access to Memory (AtoM).  If not, see <http://www.gnu.org/licenses/>.
 */

/**
 * Check csv data.
 *
 * @author     <NAME> <<EMAIL>>
 * @author     <NAME> <<EMAIL>>
 */
class csvCheckImportTask extends arBaseTask
{
    protected $verbose = false;

    /**
     * @see sfTask
     *
     * @param mixed $arguments
     * @param mixed $options
     */
    public function execute($arguments = [], $options = [])
    {
        parent::execute($arguments, $options);

        $validatorOptions = $this->setOptions($options);

        if (!empty($options['verbose'])) {
            $this->verbose = true;
        }

        $filenames = $this->setCsvValidatorFilenames($arguments['filename']);

        $validator = new CsvImportValidator(
            $this->context, $this->getDbConnection(), $validatorOptions
        );

        $validator->setShowDisplayProgress(true);
        $validator->setFilenames($filenames);
        $results = $validator->validate();
        $output = $results->renderResultsAsText($this->verbose);
        echo $output;

        unset($validator);
    }

    protected function configure()
    {
        $this->addArguments([
            new sfCommandArgument('filename', sfCommandArgument::REQUIRED,
              'The input file name (CSV format) or directory of CSV files.'),
        ]);

        $this->addOptions([
            new sfCommandOption(
                'application',
                null,
                sfCommandOption::PARAMETER_OPTIONAL,
                'The application name',
                'qubit'
            ),
            new sfCommandOption(
                'env',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'The environment',
                'cli'
            ),
            new sfCommandOption(
                'connection',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'The connection name',
                'propel'
            ),
            new sfCommandOption(
                'verbose',
                'i',
                sfCommandOption::PARAMETER_NONE,
                'Provide detailed information regarding each test.'
            ),
            new sfCommandOption(
                'source',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'Source name for validating parentId matching against previous imports. If not set, parentId validation against AtoM\'s database will be skipped.'
            ),
            new sfCommandOption(
                'class-name',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'Qubit object type contained in CSV.',
                'QubitInformationObject'
            ),
            new sfCommandOption(
                'specific-tests',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'Specific test classes to run.'
            ),
            new sfCommandOption(
                'path-to-digital-objects',
                null,
                sfCommandOption::PARAMETER_REQUIRED,
                'Path to root of digital object folder that will match digitalObjectPath in CSV.'
            ),
        ]);

        $this->namespace = 'csv';
        $this->name = 'check-import';
        $this->briefDescription = 'Check CSV data, providing diagnostic info.';
        $this->detailedDescription = <<<'EOF'
    Check CSV data, providing information about it.
EOF;
    }

    protected function getDbConnection()
    {
        $databaseManager = new sfDatabaseManager($this->configuration);

        return $databaseManager->getDatabase('propel')->getConnection();
    }

    protected function setCsvValidatorFilenames($filenameString)
    {
        // If a directory's provided return an array of file paths
        if (is_dir($filenameString) && is_readable($filenameString)) {
            return $this->getFilePathsFromDirectory($filenameString);
        }

        // Could be a comma separated list of filenames or just one.
        foreach (explode(',', $filenameString) as $filename) {
            CsvImportValidator::validateFileName($filename);
            // The validator expects an associative array of files
            // where displayname => filename
            $filenames[$filename] = $filename;
        }

        return $filenames;
    }

    protected function getFilePathsFromDirectory($directory)
    {
        $filePaths = [];

        if (!is_dir($directory) || !is_readable($directory)) {
            $errorMessage = sprintf('%s requires a readable directory.', __FUNCTION__);

            throw new UnexpectedValueException($errorMessage);
        }

        $files = array_diff(scandir($directory), ['..', '.']);

        foreach ($files as $file) {
            $filePath = $directory.DIRECTORY_SEPARATOR.$file;

            if (is_file($filePath) && is_readable($filePath)) {
                $filePaths[] = $filePath;
            }
        }

        natsort($filePaths);

        return $filePaths;
    }

    protected function setOptions($options = [])
    {
        $opts = [];

        $keymap = [
            'source' => 'source',
            'class-name' => 'className',
            'escape' => 'escape',
            'specific-tests' => 'specificTests',
            'path-to-digital-objects' => 'pathToDigitalObjects',
        ];

        foreach ($keymap as $oldkey => $newkey) {
            if (empty($options[$oldkey])) {
                continue;
            }

            $opts[$newkey] = $options[$oldkey];
        }

        return $opts;
    }
}
{'fraction_non_alphanumeric': 0.1053122878479294, 'fraction_numerical': 0.0005940257976917854, 'mean_word_length': 1.8418133590547383, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5002153', 'n_tokens_mistral': 3009, 'n_tokens_neox': 2918, 'n_words': 817}