<commit_before><?php

namespace Model\AutoUpdate;

use ZipArchive;
use DirectoryIterator;
use RecursiveIterator;
use RecursiveIteratorIterator;
use RecursiveDirectoryIterator;

// Get all files of a given directory
function get_files_list($directory)
{
    $exclude_list = array(
        '.git',
        'data',
        'scripts',
        'config.php',
    );

    $it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($directory), RecursiveIteratorIterator::SELF_FIRST);
    $files = array();

    while ($it->valid()) {

        if ($it->isFile() && ! is_excluded_path($it->getSubPathname(), $exclude_list)) {
            $files[] = $it->getSubPathname();
        }

        $it->next();
    }

    return $files;
}

// Check if the given path is excluded
function is_excluded_path($path, array $exclude_list)
{
    foreach ($exclude_list as $excluded_path) {

        if (strpos($path, $excluded_path) === 0) {
            return true;
        }
    }

    return false;
}

// Synchronize 2 directories (copy/remove files)
function synchronize($source_directory, $destination_directory)
{
    \Model\Config\debug('[SYNCHRONIZE] '.$source_directory.' to '.$destination_directory);

    $src_files = get_files_list($source_directory);
    $dst_files = get_files_list($destination_directory);

    // Remove files
    $remove_files = array_diff($dst_files, $src_files);

    foreach ($remove_files as $file) {

        $destination_file = $destination_directory.DIRECTORY_SEPARATOR.$file;
        \Model\Config\debug('[REMOVE] '.$destination_file);

        if (! @unlink($destination_file)) {
            return false;
        }
    }

    // Overwrite all files
    foreach ($src_files as $file) {

        $directory = $destination_directory.DIRECTORY_SEPARATOR.dirname($file);

        if (! is_dir($directory)) {

            \Model\Config\debug('[MKDIR] '.$directory);

            if (! @mkdir($directory, 0755, true)) {
                return false;
            }
        }

        $source_file = $source_directory.DIRECTORY_SEPARATOR.$file;
        $destination_file = $destination_directory.DIRECTORY_SEPARATOR.$file;

        \Model\Config\debug('[COPY] '.$source_file.' to '.$destination_file);

        if (! @copy($source_file, $destination_file)) {
            return false;
        }
    }

    return true;<commit_msg>Do not remove .htaccess during auto-update sync<commit_after><?php

namespace Model\AutoUpdate;

use ZipArchive;
use DirectoryIterator;
use RecursiveIterator;
use RecursiveIteratorIterator;
use RecursiveDirectoryIterator;

// Get all files of a given directory
function get_files_list($directory)
{
    $exclude_list = array(
        '.git',
        'data',
        'scripts',
        'config.php',
    );

    $it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($directory), RecursiveIteratorIterator::SELF_FIRST);
    $files = array();

    while ($it->valid()) {

        if ($it->isFile() && ! is_excluded_path($it->getSubPathname(), $exclude_list)) {
            $files[] = $it->getSubPathname();
        }

        $it->next();
    }

    return $files;
}

// Check if the given path is excluded
function is_excluded_path($path, array $exclude_list)
{
    foreach ($exclude_list as $excluded_path) {

        if (strpos($path, $excluded_path) === 0) {
            return true;
        }
    }

    return false;
}

// Synchronize 2 directories (copy/remove files)
function synchronize($source_directory, $destination_directory)
{
    \Model\Config\debug('[SYNCHRONIZE] '.$source_directory.' to '.$destination_directory);

    $src_files = get_files_list($source_directory);
    $dst_files = get_files_list($destination_directory);

    // Remove files
    $remove_files = array_diff($dst_files, $src_files);

    foreach ($remove_files as $file) {

        if ($file !== '.htaccess') {

            $destination_file = $destination_directory.DIRECTORY_SEPARATOR.$file;
            \Model\Config\debug('[REMOVE] '.$destination_file);

            if (! @unlink($destination_file)) {
                return false;
            }
        }
    }

    // Overwrite all files
    foreach ($src_files as $file) {

        $directory = $destination_directory.DIRECTORY_SEPARATOR.dirname($file);

        if (! is_dir($directory)) {

            \Model\Config\debug('[MKDIR] '.$directory);

            if (! @mkdir($directory, 0755, true)) {
                return false;
            }
        }

        $source_file = $source_directory.DIRECTORY_SEPARATOR.$file;
        $destination_file = $destination_directory.DIRECTORY_SEPARATOR.$file;

        \Model\Config\debug('[COPY] '.$source_file.' to '.$destination_file);

        if (! @copy($source_file, $destination_file)) {
            return false;
        }
    }

    return true;
{'fraction_non_alphanumeric': 0.13145833333333334, 'fraction_numerical': 0.0025, 'mean_word_length': 2.777340676632573, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1513146', 'n_tokens_mistral': 1489, 'n_tokens_neox': 1407, 'n_words': 280}