<commit_before>'use strict';

let gitPromise = require(`git-promise`);
let del = require(`del`);
let fs = require(`fs`);
let os = require(`os`);
let promisify = require(`es6-promisify`);
let log = require(`loglevel`);
let crypto = require(`crypto`);
let shell = require(`shelljs`);
let stringify = require(`json-stable-stringify`);
let uniq = require(`lodash/array/uniq`);

require('es6-promise').polyfill();

let readFilePromise = promisify(fs.readFile);
let delPromise = promisify(del);
let statPromise = promisify(fs.stat);

const PLATFORM_SPECIFIC_MODULES = {
    'fsevents': 'darwin'
};

module.exports = (cwd, {repo, verbose, crossPlatform}) => {

    let packageJsonSha1;
    let packageJsonVersion;
    let leaveAsIs = false;
    log.setLevel(verbose ? `debug`: `info`);
    return readFilePromise(`${cwd}/package.json`, `utf-8`)
    .then((packageJsonContent) => {
        let packageJson = JSON.parse(packageJsonContent);
        // compute a hash based on the stable-stringified contents of package.json
        // (`packageJsonContent` might differ on different platforms, depending on line endings etc.)
        let stableContent = stringify(packageJson);
        // replace / in hash with _ because git does not allow leading / in tags
        packageJsonSha1 = crypto.createHash(`sha1`).update(stableContent).digest(`base64`).replace(/\//g, "_");
        packageJsonVersion = packageJson.version;
        log.debug(`SHA-1 of package.json (version ${packageJsonVersion}) is ${packageJsonSha1}`);
        return packageJsonSha1;
    })
    .then(() => {
        return statPromise(`${cwd}/node_modules`)
        .then(() => {
            log.debug(`Checking if remote ${repo} exists`);
            process.chdir(`${cwd}/node_modules`);
            return git(`git remote -v`)
            .then((remoteCommandOutput) => {
                if (remoteCommandOutput.indexOf(repo) !== -1) {
                    // repo is in remotes
                    return git(`tag -l --points-at HEAD`)
                    .then((tags) => {
                        if (tags.split('\n').indexOf(packageJsonSha1) >= 0) {
                            // if the current HEAD is at the right commit, don't change anything
                            log.debug(`${repo} is already at tag ${packageJsonSha1}, leaving as is`);
                            leaveAsIs = true;
                        } else {
                            log.debug(`Remote exists, fetching from it`);
                            return git(`git fetch -t ${repo}`);
                        }
                    });
                }
                return cloneRepo();
            });
        })
        .catch(cloneRepo)
    })
    .then((tags) => {
        if (leaveAsIs) {
            return;
        }
        log.debug(`${repo} is in node_modules cwd, checking out ${packageJsonSha1} tag`);
        process.chdir(`${cwd}/node_modules`);
        return git(`checkout tags/${packageJsonSha1}`)
        .then(() => {
            log.debug(`Cleanup checked out commit`);
            return git(`clean -df`);
        })
        .then(() => {
            if (crossPlatform) {
                return rebuildAndIgnorePlatformSpecific();
            }
        })
        .catch(installPackagesTagAndPustToRemote);
    })
    .then(() => {
        process.chdir(`${cwd}`);
        log.info(`Node_modules are in sync with ${repo} ${packageJsonSha1}`);
    })
    .catch((error) => {
        process.chdir(`${cwd}`);
        log.info(`Failed to synchronise node_modules with ${repo}: ${error}`);
        throw error;
    });

    function cloneRepo() {
        log.debug(`Remote ${repo} is not present in ${cwd}/node_modules/.git repo`);<commit_msg>exit with status code 1 when there is an exception<commit_after>'use strict';

let gitPromise = require(`git-promise`);
let del = require(`del`);
let fs = require(`fs`);
let os = require(`os`);
let promisify = require(`es6-promisify`);
let log = require(`loglevel`);
let crypto = require(`crypto`);
let shell = require(`shelljs`);
let stringify = require(`json-stable-stringify`);
let uniq = require(`lodash/array/uniq`);

require('es6-promise').polyfill();

let readFilePromise = promisify(fs.readFile);
let delPromise = promisify(del);
let statPromise = promisify(fs.stat);

const PLATFORM_SPECIFIC_MODULES = {
    'fsevents': 'darwin'
};

module.exports = (cwd, {repo, verbose, crossPlatform}) => {

    let packageJsonSha1;
    let packageJsonVersion;
    let leaveAsIs = false;
    log.setLevel(verbose ? `debug`: `info`);
    return readFilePromise(`${cwd}/package.json`, `utf-8`)
    .then((packageJsonContent) => {
        let packageJson = JSON.parse(packageJsonContent);
        // compute a hash based on the stable-stringified contents of package.json
        // (`packageJsonContent` might differ on different platforms, depending on line endings etc.)
        let stableContent = stringify(packageJson);
        // replace / in hash with _ because git does not allow leading / in tags
        packageJsonSha1 = crypto.createHash(`sha1`).update(stableContent).digest(`base64`).replace(/\//g, "_");
        packageJsonVersion = packageJson.version;
        log.debug(`SHA-1 of package.json (version ${packageJsonVersion}) is ${packageJsonSha1}`);
        return packageJsonSha1;
    })
    .then(() => {
        return statPromise(`${cwd}/node_modules`)
        .then(() => {
            log.debug(`Checking if remote ${repo} exists`);
            process.chdir(`${cwd}/node_modules`);
            return git(`git remote -v`)
            .then((remoteCommandOutput) => {
                if (remoteCommandOutput.indexOf(repo) !== -1) {
                    // repo is in remotes
                    return git(`tag -l --points-at HEAD`)
                    .then((tags) => {
                        if (tags.split('\n').indexOf(packageJsonSha1) >= 0) {
                            // if the current HEAD is at the right commit, don't change anything
                            log.debug(`${repo} is already at tag ${packageJsonSha1}, leaving as is`);
                            leaveAsIs = true;
                        } else {
                            log.debug(`Remote exists, fetching from it`);
                            return git(`git fetch -t ${repo}`);
                        }
                    });
                }
                return cloneRepo();
            });
        })
        .catch(cloneRepo)
    })
    .then((tags) => {
        if (leaveAsIs) {
            return;
        }
        log.debug(`${repo} is in node_modules cwd, checking out ${packageJsonSha1} tag`);
        process.chdir(`${cwd}/node_modules`);
        return git(`checkout tags/${packageJsonSha1}`)
        .then(() => {
            log.debug(`Cleanup checked out commit`);
            return git(`clean -df`);
        })
        .then(() => {
            if (crossPlatform) {
                return rebuildAndIgnorePlatformSpecific();
            }
        })
        .catch(installPackagesTagAndPustToRemote);
    })
    .then(() => {
        process.chdir(`${cwd}`);
        log.info(`Node_modules are in sync with ${repo} ${packageJsonSha1}`);
    })
    .catch((error) => {
        process.chdir(`${cwd}`);
        log.info(`Failed to synchronise node_modules with ${repo}: ${error}`);
        process.exit(1);
    });

    function cloneRepo() {
        log.debug(`Remote ${repo} is not present in ${cwd}/node_modules/.git repo`);
{'fraction_non_alphanumeric': 0.14473507712944333, 'fraction_numerical': 0.005097250167672703, 'mean_word_length': 2.114452798663325, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '494339', 'n_tokens_mistral': 2104, 'n_tokens_neox': 2108, 'n_words': 499}