<commit_before>const async = require('async');
const chalk = require('chalk');
const child_process = require('child_process');
const config = require('./util/config');
const glob = require('glob');
const gspdata = require('./util/gspdata');
const fs = require('fs');
const path = require('path');
const upa = require('upa');

exports.configure = function () {
    let cwd = process.cwd();
    require(cwd + '/gss.conf.js')(config);
    let options = config.get();

    console.log('Configuring...');

    async.waterfall([
        function (callback) {
            if (!fs.existsSync(path.join(cwd, '.gitrepos'))) {
                fs.mkdirSync(path.join(cwd, '.gitrepos'));
            }
            console.log('Starting cloning git repositories...');
            async.eachLimit(options.gitrepos.urls, 5, function (repo, c) {
                if (fs.existsSync(path.join(cwd, '.gitrepos', repo))) {
                    c();
                }
                else {
                    console.log('Cloning ' + repo + '...');
                    child_process.exec('git clone --mirror ' + options.gitrepos.url_prefix + repo, {cwd: path.join(cwd, '.gitrepos')}, c);
                }
            }, callback);
        },
        function (callback) {
            glob('**/*.git', {cwd: path.join(cwd, '.gitrepos')}, function (err, result) {
                if (!err) {
                    let repoLocation = {};
                    result.forEach(function (repo) {
                        repoLocation[repo.slice(0, -4)] = path.join(cwd, '.gitrepos', repo);
                    });
                    gspdata.set('repositories', repoLocation);
                }
                callback(err);
            });
        },
        function (callback) {
            console.log('Starting cloning subversion repository...');
            if (fs.existsSync(path.join(cwd, '.svnrepo'))) {
                callback();
            }
            else {
                let username = upa.list()[0];
                let password = upa.get(username);<commit_msg>do not allow to create deep directories for repository<commit_after>const async = require('async');
const chalk = require('chalk');
const child_process = require('child_process');
const config = require('./util/config');
const gspdata = require('./util/gspdata');
const fs = require('fs');
const path = require('path');
const upa = require('upa');

exports.configure = function () {
    let cwd = process.cwd();
    require(cwd + '/gss.conf.js')(config);
    let options = config.get();

    console.log('Configuring...');

    async.waterfall([
        function (callback) {
            if (!fs.existsSync(path.join(cwd, '.gitrepos'))) {
                fs.mkdirSync(path.join(cwd, '.gitrepos'));
            }
            console.log('Starting cloning git repositories...');
            async.eachLimit(options.gitrepos.urls, 5, function (repo, c) {
                if (fs.existsSync(path.join(cwd, '.gitrepos', path.basename(repo)))) {
                    c();
                }
                else {
                    console.log('Cloning ' + repo + '...');
                    child_process.exec('git clone --mirror ' + options.gitrepos.url_prefix + repo, {cwd: path.join(cwd, '.gitrepos')}, c);
                }
            }, callback);
        },
        function (callback) {
            fs.readdir(path.join(cwd, '.gitrepos'), function (err, result) {
                if (!err) {
                    let repoLocation = {};
                    result.forEach(function (repo) {
                        repoLocation[repo.slice(0, -4)] = path.join(cwd, '.gitrepos', repo);
                    });
                    gspdata.set('repositories', repoLocation);
                }
                callback(err);
            });
        },
        function (callback) {
            console.log('Starting cloning subversion repository...');
            if (fs.existsSync(path.join(cwd, '.svnrepo'))) {
                callback();
            }
            else {
                let username = upa.list()[0];
                let password = upa.get(username);
{'fraction_non_alphanumeric': 0.1573061026015074, 'fraction_numerical': 0.0019450522732798443, 'mean_word_length': 1.7047994740302432, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3989087', 'n_tokens_mistral': 1176, 'n_tokens_neox': 1098, 'n_words': 228}