<commit_before>var Promise = require('bluebird');
var request = require('request');
var progress = require('progress');
var fs = require('fs');
var path = require('path');
var zlib = require('zlib');
var tar = require('tar-fs');
var temp = require('temp');
var ncp = require('graceful-ncp').ncp;
var rimraf = require('rimraf');
var extract = require('extract-zip');

// Automatically track and cleanup files at exit
temp.track();
var isWin = /^win/.test(process.platform);

// one progressbar for all downloads
var bar;

module.exports = {
    checkCache: function(cachepath, files) {
        var missing;

        // if the version is >=0.12.3, then we don't which files we want from the archives, so just check that the folder
        // exists and has at least 3 files in it.
        if(files.length === 1 && files[0] === '*'){
            return fs.existsSync(cachepath) && fs.readdirSync(cachepath).length >= 3;
        }

        files.forEach(function(file) {
            if (missing) {
                return;
            }<commit_msg>osx 0.20.3 normal only has 2 files at root folder (#435)<commit_after>var Promise = require('bluebird');
var request = require('request');
var progress = require('progress');
var fs = require('fs');
var path = require('path');
var zlib = require('zlib');
var tar = require('tar-fs');
var temp = require('temp');
var ncp = require('graceful-ncp').ncp;
var rimraf = require('rimraf');
var extract = require('extract-zip');

// Automatically track and cleanup files at exit
temp.track();
var isWin = /^win/.test(process.platform);

// one progressbar for all downloads
var bar;

module.exports = {
    checkCache: function(cachepath, files) {
        var missing;

        // if the version is >=0.12.3, then we don't which files we want from the archives, so just check that the folder
        // exists and has at least 3 files in it.
        if(files.length === 1 && files[0] === '*'){
            return fs.existsSync(cachepath) && fs.readdirSync(cachepath).length >= 2;
        }

        files.forEach(function(file) {
            if (missing) {
                return;
            }
{'fraction_non_alphanumeric': 0.14513856270549555, 'fraction_numerical': 0.011272898074213245, 'mean_word_length': 2.9813084112149535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6768420', 'n_tokens_mistral': 642, 'n_tokens_neox': 611, 'n_words': 206}