<commit_before><?php

/**
 * recover OpenM.util package
 */
$util = explode("=", file_get_contents("lib/openm.util.dependencies"));
$lib = "http://open-miage.org/download/" . $util[0] . "/" . $util[1];
$lib_dir = "../lib";
if (!is_dir($lib_dir)) {
    if (mkdir($lib_dir))
        echo " - $lib_dir <b>created</b><br>";
    else
        die("<h1>error occurs during creation of $lib_dir</h1>");
}
$lib_abs_dir = realpath($lib_dir);
$temp = "$lib_abs_dir/temp/";
if (!is_dir($temp))
    mkdir($temp);

$targetZip = $temp . $util[1];
if (copy($lib, $targetZip))
    echo " - $lib <b>copied to</b> $targetZip<br>";
else
    die("<h1>error occurs during copy of $lib</h1>");

$zip = new ZipArchive();
$res = $zip->open($targetZip);
if ($res === TRUE) {
    if ($zip->extractTo($temp))
        echo " - $targetZip <b>unZip in</b> $temp<br>";
    else
        die("<h1>error occurs during unZip of $targetZip</h1>");
    $zip->close();
    if (unlink($targetZip))
        echo " - $targetZip <b>correctly removed</b><br>";
    else
        die("<h1>error occurs during removing $targetZip</h1>");

    define("OpenM_EXT_LIB_PATH", $lib_abs_dir);
    require_once $temp . "lib/" . $util[0] . "/Import.class.php";
    Import::php("util.file.OpenM_Dir");
    OpenM_Dir::cp($temp . "lib/", $lib_abs_dir);
    echo " - $temp" . "lib/ <b>correctly copied in</b> $lib_abs_dir<br>";
    Import::addLibPath($util[0]);
    OpenM_Dir::rm($temp);
    echo " - $temp <b>correctly removed</b><br>";
    echo ' - lib OpenM.util succesfully installed !<br>';
}
else
    die('<h1>error occurs</h1>');

/**
 * install other dependencies
 */
Import::php("util.pkg.OpenM_Dependencies");
$dependencies = new OpenM_Dependencies("lib");
$dependencies->install("./temp", OpenM_Dependencies::ALL(), true);
OpenM_Dir::rm("./temp");
?><commit_msg>use new autoDownload script<commit_after><?php

$scriptName = "Auto-Download-Script";
$url = "http://open-miage.org/download/$scriptName";
$autoDownloadScript = file_get_contents($url);
if ($autoDownloadScript === false)
    die($url . " not found");

file_put_contents("$scriptName.php", $autoDownloadScript);
require_once "$scriptName.php";
?>
{'fraction_non_alphanumeric': 0.21352477999073646, 'fraction_numerical': 0.006947660954145438, 'mean_word_length': 4.118483412322274, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3185759', 'n_tokens_mistral': 796, 'n_tokens_neox': 775, 'n_words': 151}