<commit_before><?php
require_once dirname(__FILE__) . '/../videos/configuration.php';
header('Content-Type: application/json');

$files = array();
if(empty($global['disableBulkEncode'])){
    if (!empty($_POST['path'])) {
        $path = $_POST['path'];
        if (substr($path, -1) !== '/') {
            $path .= "/";
        }

        if (file_exists($path)) {
            $filesStr = "{*." . implode(",*.", $global['allowed']) . "}";

            //echo $files;
            $video_array = glob($path . $filesStr, GLOB_BRACE);

            $id = 0;
            foreach ($video_array as $key => $value) {
                $path_parts = pathinfo($value);
                $obj = new stdClass();
                $obj->id = $id++;
                $obj->path = addslashes($value);
                $obj->name = addslashes($path_parts['basename']);
                $files[] = $obj;
            }
        }
    }
}
echo json_encode($files);
<commit_msg>Bulk Encode<commit_after><?php
require_once dirname(__FILE__) . '/../videos/configuration.php';
header('Content-Type: application/json');

$files = array();
if(empty($global['disableBulkEncode'])){
    if (!empty($_POST['path'])) {
        $path = $_POST['path'];
        if (substr($path, -1) !== '/') {
            $path .= "/";
        }

        if (file_exists($path)) {
            $filesStr = "{*." . implode(",*.", $global['allowed']) . "}";

            //echo $files;
            $video_array = glob($path . $filesStr, GLOB_BRACE);

            $id = 0;
            foreach ($video_array as $key => $value) {
                $path_parts = pathinfo($value);
                $obj = new stdClass();
                $obj->id = $id++;
                $obj->path = utf8_encode($value);
                $obj->name = utf8_encode($path_parts['basename']);
                $files[] = $obj;
            }
        }
    }
}
echo json_encode($files);

{'fraction_non_alphanumeric': 0.20042194092827004, 'fraction_numerical': 0.0031645569620253164, 'mean_word_length': 1.814540059347181, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7624701', 'n_tokens_mistral': 590, 'n_tokens_neox': 573, 'n_words': 101}