<commit_before>const fs = require('fs');
const http = require('http');
const shell = require('shelljs');
const decompress = require('decompress');
const decompressTargz = require('decompress-targz');
const env = require('./env.js');
const { exec } = require('./helper.js');

function writeFile(url, destination, resolve, reject) {
    const file = fs.createWriteStream(destination);

    const req = http.get(url, (res) => {
        res.pipe(file);
    });

    req.on('error', (e) => {
        file.unlink(destination);
        reject(e);
    });

    file.on('error', () => {
        file.unlink(destination);
        reject(new Error('File error'));
    });

    file.on('finish', resolve);
}

function download(url, destination) {
    if (fs.existsSync(destination)) {
        shell.echo(`${destination} already exists. Skipping download.`);

        return Promise.resolve();
    }

    const executor = writeFile.bind(null, url, destination);
    return new Promise(executor);
}

function validateFile(fileName) {
    let res;

    if (env.isWindows) {
        const output = exec(`certutil -hashfile ${fileName} sha1`).split('\r\n');

        const arr = output[0].replace(':', '').split(' ');
        const name = arr.find(n => n === fileName);

        const sha1 = output[1];
        res = `${sha1}  ${name}`;
    } else {
        res = exec(`shasum -a 1 ${fileName}`).trim();
    }

    if (res !== env.sha1sum) {
        throw new Error(`Wrong sha1 for ${fileName}! Expected "${env.sha1sum}", got "${res}".`);
    }
}

function clearPath() {
    if (!shell.test('-d', env.sourceFolder)) {<commit_msg>fix: `certutil` HashAlgorithm arg should be PASCAL<commit_after>const fs = require('fs');
const http = require('http');
const shell = require('shelljs');
const decompress = require('decompress');
const decompressTargz = require('decompress-targz');
const env = require('./env.js');
const { exec } = require('./helper.js');

function writeFile(url, destination, resolve, reject) {
    const file = fs.createWriteStream(destination);

    const req = http.get(url, (res) => {
        res.pipe(file);
    });

    req.on('error', (e) => {
        file.unlink(destination);
        reject(e);
    });

    file.on('error', () => {
        file.unlink(destination);
        reject(new Error('File error'));
    });

    file.on('finish', resolve);
}

function download(url, destination) {
    if (fs.existsSync(destination)) {
        shell.echo(`${destination} already exists. Skipping download.`);

        return Promise.resolve();
    }

    const executor = writeFile.bind(null, url, destination);
    return new Promise(executor);
}

function validateFile(fileName) {
    let res;

    if (env.isWindows) {
        const output = exec(`certutil -hashfile ${fileName} SHA1`).split('\r\n');

        const arr = output[0].replace(':', '').split(' ');
        const name = arr.find(n => n === fileName);

        const sha1 = output[1];
        res = `${sha1}  ${name}`;
    } else {
        res = exec(`shasum -a 1 ${fileName}`).trim();
    }

    if (res !== env.sha1sum) {
        throw new Error(`Wrong sha1 for ${fileName}! Expected "${env.sha1sum}", got "${res}".`);
    }
}

function clearPath() {
    if (!shell.test('-d', env.sourceFolder)) {
{'fraction_non_alphanumeric': 0.18429929469487888, 'fraction_numerical': 0.005519779208831647, 'mean_word_length': 3.01230012300123, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '621126', 'n_tokens_mistral': 1022, 'n_tokens_neox': 1021, 'n_words': 238}