<commit_before><?php

namespace Sunlight\Backup;

use Sunlight\Core;
use Sunlight\Database\Database as DB;
use Sunlight\Database\DatabaseLoader;
use Sunlight\Database\SqlReader;
use Sunlight\Util\Filesystem;

class BackupRestorer
{
    /** @var Backup */
    protected $backup;

    /**
     * @param Backup $backup
     */
    public function __construct(Backup $backup)
    {
        $this->backup = $backup;
    }

    /**
     * Validate the backup
     *
     * @param array|null &$errors
     * @return bool
     */
    public function validate(&$errors = null)
    {
        $errors = $this->backup->getMetaDataErrors();

        return empty($errors);
    }

    /**
     * Restore the backup
     *
     * @param bool       $database    restore the database 1/0
     * @param array      $directories directory paths to restore (from backup's metadata), null = all
     * @param array      $files       file paths to restore (from backup's metadata), null = all
     * @param array|null &$errors
     * @return bool
     */
    public function restore($database, array $directories = null, array $files = null, &$errors = null)
    {
        $errors = array();

        $isPatch = $this->backup->getMetaData('is_patch');

        // normalize arguments
        $database = $database && $this->backup->hasDatabaseDump();

        if ($directories === null) {
            $directories = $this->backup->getMetaData('directory_list');
        } else {
            $directories = array_intersect($this->backup->getMetaData('directory_list'), $directories);
        }

        if ($files === null) {
            $files = $this->backup->getMetaData('file_list');
        } else {
            $files = array_intersect($this->backup->getMetaData('file_list'), $files);
        }

        if ($isPatch) {
            $filesToRemove = $this->backup->getMetaData('files_to_remove');
            $directoriesToRemove = $this->backup->getMetaData('directories_to_remove');
            $directoriesToPurge = $this->backup->getMetaData('directories_to_purge');
        } else {
            $filesToRemove = array();
            $directoriesToRemove = array();
            $directoriesToPurge = $directories;
        }

        // verify what we are restoring
        if (!$database && empty($directories) && empty($files)) {
            $errors[] = 'nothing to restore';
        }

        // verify files
        foreach (array_merge($files, $filesToRemove) as $file) {
            $fullPath = _root . $file;

            if (is_file($fullPath) && !is_writable($fullPath)) {
                $errors[] = sprintf('cannot write to "%s", please check privileges (%s)', $fullPath);
            }
        }

        // verify directories
        foreach (array_merge($directoriesToRemove, $directoriesToPurge) as $directory) {
            if (!Filesystem::checkDirectory(_root . $directory, true, $failedPaths)) {
                $failedPathsString = implode(', ', array_slice($failedPaths, 0, 3));
                if (sizeof($failedPaths) > 3) {
                    $failedPathsString .= sprintf(' and %d more', sizeof($failedPaths) - 3);
                }

                $errors[] = sprintf('cannot write to "%s", please check privileges (%s)', $failedPathsString);
            }
        }

        if ($errors) {
            return false;
        }

        // load database
        if ($database) {
            if (!$this->backup->getMetaData('is_patch')) {
                DatabaseLoader::dropTables(DB::getTablesByPrefix());
            }

            DatabaseLoader::load(
                SqlReader::fromStream($this->backup->getDatabaseDump()),
                $this->backup->getMetaData('db_prefix'),
                _dbprefix
            );
        }

        // filesystem cleanup
        foreach ($directoriesToPurge as $directory) {
            Filesystem::purgeDirectory(_root . $directory, array('keep_dir' => true));
        }
        foreach ($directoriesToRemove as $directory) {
            Filesystem::purgeDirectory(_root . $directory);
        }
        foreach ($filesToRemove as $file) {
            unlink($file);
        }

        // extract directories
        if (!empty($directories)) {
            $this->backup->extractDirectories($directories, _root);
        }

        // extract files
        if (!empty($files)) {
            $this->backup->extractFiles($files, _root);
        }

        // clear cache
        Core::$cache->clear();

        // force install check
        Core::updateSetting('install_check', 1);

        return true;
    }
}<commit_msg>Normalize backup restorer file/dir lists<commit_after><?php

namespace Sunlight\Backup;

use Sunlight\Core;
use Sunlight\Database\Database as DB;
use Sunlight\Database\DatabaseLoader;
use Sunlight\Database\SqlReader;
use Sunlight\Util\Filesystem;

class BackupRestorer
{
    /** @var Backup */
    protected $backup;

    /**
     * @param Backup $backup
     */
    public function __construct(Backup $backup)
    {
        $this->backup = $backup;
    }

    /**
     * Validate the backup
     *
     * @param array|null &$errors
     * @return bool
     */
    public function validate(&$errors = null)
    {
        $errors = $this->backup->getMetaDataErrors();

        return empty($errors);
    }

    /**
     * Restore the backup
     *
     * @param bool       $database    restore the database 1/0
     * @param array      $directories directory paths to restore (from backup's metadata), null = all
     * @param array      $files       file paths to restore (from backup's metadata), null = all
     * @param array|null &$errors
     * @return bool
     */
    public function restore($database, array $directories = null, array $files = null, &$errors = null)
    {
        $errors = array();

        $isPatch = $this->backup->getMetaData('is_patch');
        $database = $database && $this->backup->hasDatabaseDump();

        // defaults
        if ($directories === null) {
            $directories = $this->backup->getMetaData('directory_list');
        }
        if ($files === null) {
            $files = $this->backup->getMetaData('file_list');
        }

        // normalize lists
        $files = $this->normalizePathList($files, $this->backup->getMetaData('file_list'));
        $directories = $this->normalizePathList($directories, $this->backup->getMetaData('directory_list'));

        if ($isPatch) {
            $filesToRemove = $this->normalizePathList($this->backup->getMetaData('files_to_remove'), null, true, true);
            $directoriesToRemove = $this->normalizePathList($this->backup->getMetaData('directories_to_remove'), null, true, true);
            $directoriesToPurge = $this->normalizePathList($this->backup->getMetaData('directories_to_purge'), null, true, true);
        } else {
            $filesToRemove = array();
            $directoriesToRemove = array();
            $directoriesToPurge = $this->normalizePathList($directories, null, true, true);
        }

        // verify what we are restoring
        if (!$database && empty($directories) && empty($files)) {
            $errors[] = 'nothing to restore';
        }

        // verify files
        foreach ($files as $file) {
            if (is_file($file) && !is_writable(_root . $file)) {
                $errors[] = sprintf('cannot write to "%s", please check privileges (%s)', $file);
            }
        }
        foreach ($filesToRemove as $file) {
            if (!is_writable($file)) {
                $errors[] = sprintf('cannot write to "%s", please check privileges (%s)', $file);
            }
        }

        // verify directories
        foreach (array_merge($directoriesToRemove, $directoriesToPurge) as $directory) {
            if (!Filesystem::checkDirectory($directory, true, $failedPaths)) {
                $failedPathsString = implode(', ', array_slice($failedPaths, 0, 3));
                if (sizeof($failedPaths) > 3) {
                    $failedPathsString .= sprintf(' and %d more', sizeof($failedPaths) - 3);
                }

                $errors[] = sprintf('cannot write to "%s", please check privileges (%s)', $failedPathsString);
            }
        }

        if ($errors) {
            return false;
        }

        // load database
        if ($database) {
            if (!$this->backup->getMetaData('is_patch')) {
                DatabaseLoader::dropTables(DB::getTablesByPrefix());
            }

            DatabaseLoader::load(
                SqlReader::fromStream($this->backup->getDatabaseDump()),
                $this->backup->getMetaData('db_prefix'),
                _dbprefix
            );
        }

        // filesystem cleanup
        foreach ($directoriesToPurge as $directory) {
            Filesystem::purgeDirectory($directory, array('keep_dir' => true));
        }
        foreach ($directoriesToRemove as $directory) {
            Filesystem::purgeDirectory($directory);
        }
        foreach ($filesToRemove as $file) {
            if (is_file($file)) {
                unlink($file);
            }
        }

        // extract directories
        if (!empty($directories)) {
            $this->backup->extractDirectories($directories, _root);
        }

        // extract files
        if (!empty($files)) {
            $this->backup->extractFiles($files, _root);
        }

        // clear cache
        Core::$cache->clear();

        // force install check
        Core::updateSetting('install_check', 1);

        return true;
    }

    /**
     * @param string[]      $paths
     * @param string[]|null $allowedValues      list of allowed values in $paths
     * @param bool          $addRootPath        prefix normalized paths with _root 1/0
     * @param bool          $excludeNonexistent skip nonexistent paths 1/0
     * @return array
     */
    protected function normalizePathList(array $paths, array $allowedValues = null, $addRootPath = false, $excludeNonexistent = false)
    {
        if ($allowedValues) {
            $paths = array_intersect($allowedValues, $paths);
        }

        $normalizedPaths = array();

        foreach ($paths as $path) {
            if ($excludeNonexistent && !file_exists(_root . $path)) {
                continue;
            }

            $normalizedPath = $path;

            if ($addRootPath) {
                $normalizedPath = _root . $normalizedPath;
            }

            $normalizedPaths[] = $normalizedPath;
        }

        return $normalizedPaths;
    }
}
{'fraction_non_alphanumeric': 0.13389798248953177, 'fraction_numerical': 0.0017129805862200228, 'mean_word_length': 2.040798611111111, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2257506', 'n_tokens_mistral': 2879, 'n_tokens_neox': 2786, 'n_words': 633}