<commit_before>var fs = require('fs');
var path = require('path');
var crypto = require('crypto');

var babel = require('babel-core');
var micromatch = require('micromatch');

function fileLastModifiedHash(path) {
    var mtime = fs.lstatSync(path).mtime.getTime();

    return crypto
            .createHash('md5')
            .update(mtime + '-' + path)
            .digest('hex');
}

module.exports = function(options) {
    options = options || {};

    var srcPath = options.srcPath;
    var cachePath = options.cachePath || 'memory';
    var isMemoryCache = cachePath === 'memory';
    var exclude = options.exclude || [];
    var debug = options.debug || false;

    // filename to last known hash map
    var hashMap = {};

    // hash to transpiled file contents map
    var cacheMap = {};

    if (!isMemoryCache) {
        try {
            fs.mkdirSync(cachePath);
        } catch (e) {}
    }

    var babelOptions = options.babelOptions || { stage: 0 };

    function log() {
        if (debug) {
            console.log.apply(undefined, arguments);
        }
    }

    function pathForHash(hash) {
        return path.resolve(cachePath + '/' + hash + '.js');
    }

    return function(req, res, next) {
        var src = path.resolve(srcPath + '/' + req.path); // XXX Need the correct path
        var hash = fileLastModifiedHash(src);
        var lastKnownHash = hashMap[src];
        var hashPath;

        if (exclude.length) {
            if (micromatch.any(req.path.replace(/^\/+|\/+$/g, ''), exclude)) {
                log('Excluded: %s (%s)', req.path, exclude);
                res.append('X-Babel-Cache', false);
                res.sendFile(src, {}, function(err) {
                    if (err) {
                        res.status(500).send(err).end();
                    }
                });
                return;
            }
        }

        log('Preparing: %s (%s)', src, hash);

        res.append('X-Babel-Cache', true);
        res.append('X-Babel-Cache-Hash', hash);

        if (!isMemoryCache) {
            hashPath = pathForHash(hash);
            try {
                fs.statSync(hashPath);
                hashMap[src] = lastKnownHash = hash;
            } catch(e) {}
        }

        if (lastKnownHash && lastKnownHash === hash) {
            // file unchanged, exit early
            var cacheMiss = false;
            if (!isMemoryCache) {
                try {
                    fs.lstatSync(hashPath);
                } catch(e) {
                    cacheMiss = true;
                }

                // Ensure Cache directory exists
                if (cacheMiss) {
                    try {
                        fs.lstatSync(cachePath);
                    } catch (e) {
                        fs.mkdirSync(cachePath);
                    }
                }
            }

            if (!cacheMiss) {
                res.append('X-Babel-Cache-Hit', true);
                if (isMemoryCache) {
                    log('Serving (cached): %s', src);
                    res.write(cacheMap[hash]);
                    res.end();
                } else {
                    log('Serving (cached): %s', hashPath);
                    res.sendFile(hashPath, {}, function(err) {
                        if (err) {
                            res.status(500).send(err).end();
                        }
                    });
                }
                return;
            }
        }

        res.append('X-Babel-Cache-Hit', false);

        if (isMemoryCache && lastKnownHash && lastKnownHash in cacheMap) {
            delete cacheMap[lastKnownHash];
        } else if (!isMemoryCache && lastKnownHash) {
            try {
                fs.unlinkSync(pathForHash(lastKnownHash));
            } catch(e) {}
        }

        var result;
        try {
            result = babel.transformFileSync(src, babelOptions);
        } catch(e) {
            res.status(500).send(e);
            res.end();
            return;
        }

        var code = result.code;
        hashMap[src] = hash;

        if (isMemoryCache) {
            cacheMap[hash] = code;
        } else {
            fs.writeFile(hashPath, code, function(err) {
                if (err) {
                    // console.error('Error saving ' + hashPath + ': ' + err);
                    delete hashMap[src];
                }
            });
        }
        log('Serving (uncached): %s', src);
        res.write(code);
        res.end();
    };
};<commit_msg>Bail out if requested to parse something other than JavaScript<commit_after>var fs = require('fs');
var path = require('path');
var crypto = require('crypto');

var babel = require('babel-core');
var micromatch = require('micromatch');

function fileLastModifiedHash(path) {
    var mtime = fs.lstatSync(path).mtime.getTime();

    return crypto
            .createHash('md5')
            .update(mtime + '-' + path)
            .digest('hex');
}

module.exports = function(options) {
    options = options || {};

    var srcPath = options.srcPath;
    var cachePath = options.cachePath || 'memory';
    var isMemoryCache = cachePath === 'memory';
    var exclude = options.exclude || [];
    var debug = options.debug || false;

    // filename to last known hash map
    var hashMap = {};

    // hash to transpiled file contents map
    var cacheMap = {};

    if (!isMemoryCache) {
        try {
            fs.mkdirSync(cachePath);
        } catch (e) {}
    }

    var babelOptions = options.babelOptions || { stage: 0 };

    function log() {
        if (debug) {
            console.log.apply(undefined, arguments);
        }
    }

    function pathForHash(hash) {
        return path.resolve(cachePath + '/' + hash + '.js');
    }

    return function(req, res, next) {
        // Bail out if we're asked to load something other than JavaScript
        // TODO: Make this configurable.
        if (!micromatch.any(req.path, ['**/*.js', '**/*.jsx'])) {
            next();
            return;
        }

        var src = path.resolve(srcPath + '/' + req.path); // XXX Need the correct path
        var hash = fileLastModifiedHash(src);
        var lastKnownHash = hashMap[src];
        var hashPath;

        if (exclude.length) {
            if (micromatch.any(req.path.replace(/^\/+|\/+$/g, ''), exclude)) {
                log('Excluded: %s (%s)', req.path, exclude);
                res.append('X-Babel-Cache', false);
                res.sendFile(src, {}, function(err) {
                    if (err) {
                        res.status(500).send(err).end();
                    }
                });
                return;
            }
        }

        log('Preparing: %s (%s)', src, hash);

        res.append('X-Babel-Cache', true);
        res.append('X-Babel-Cache-Hash', hash);

        if (!isMemoryCache) {
            hashPath = pathForHash(hash);
            try {
                fs.statSync(hashPath);
                hashMap[src] = lastKnownHash = hash;
            } catch(e) {}
        }

        if (lastKnownHash && lastKnownHash === hash) {
            // file unchanged, exit early
            var cacheMiss = false;
            if (!isMemoryCache) {
                try {
                    fs.lstatSync(hashPath);
                } catch(e) {
                    cacheMiss = true;
                }

                // Ensure Cache directory exists
                if (cacheMiss) {
                    try {
                        fs.lstatSync(cachePath);
                    } catch (e) {
                        fs.mkdirSync(cachePath);
                    }
                }
            }

            if (!cacheMiss) {
                res.append('X-Babel-Cache-Hit', true);
                if (isMemoryCache) {
                    log('Serving (cached): %s', src);
                    res.write(cacheMap[hash]);
                    res.end();
                } else {
                    log('Serving (cached): %s', hashPath);
                    res.sendFile(hashPath, {}, function(err) {
                        if (err) {
                            res.status(500).send(err).end();
                        }
                    });
                }
                return;
            }
        }

        res.append('X-Babel-Cache-Hit', false);

        if (isMemoryCache && lastKnownHash && lastKnownHash in cacheMap) {
            delete cacheMap[lastKnownHash];
        } else if (!isMemoryCache && lastKnownHash) {
            try {
                fs.unlinkSync(pathForHash(lastKnownHash));
            } catch(e) {}
        }

        var result;
        try {
            result = babel.transformFileSync(src, babelOptions);
        } catch(e) {
            res.status(500).send(e);sfdfdfdf
            res.end();
            return;
        }ddfdf

        var code = result.code;
        hashMap[src] = hash;

        if (isMemoryCache) {
            cacheMap[hash] = code;
        } else {
            fs.writeFile(hashPath, code, function(err) {
                if (err) {
                    // console.error('Error saving ' + hashPath + ': ' + err);
                    delete hashMap[src];
                }
            });
        }
        log('Serving (uncached): %s', src);
        res.write(code);
        res.end();
    };
};
{'fraction_non_alphanumeric': 0.13261687264911337, 'fraction_numerical': 0.0023643202579258465, 'mean_word_length': 1.316654219566841, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2152967', 'n_tokens_mistral': 2671, 'n_tokens_neox': 2483, 'n_words': 533}