<commit_before>var fs = require('fs');
var path = require('path');
var url = require('url');
var crypto = require('crypto');
var EventEmitter = require('events').EventEmitter;

// Third party modules
var _ = require('underscore');
var srs = require('srs');
var get = require('get');
var zipfile = require('zipfile');
var Step = require('step');

// Known SRS values
var SRS = {
    'WGS84': '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs',
    '900913': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over'
};

var downloads = {};
var pool = require('generic-pool').Pool({
    create: function(callback) { callback(null, {}); },
    destroy: function(obj) { delete obj; },
    max: 5
});

function download(url, filepath, callback) {
    var dl = filepath + '.download'
    if (downloads[url]) return downloads[url].once('done', callback);

    downloads[url] = new EventEmitter();
    pool.acquire(function(obj) {
        (new get(url)).toDisk(dl, function(err, file) {
            pool.release(obj);
            if (err) {
                downloads[url].emit('done', err);
                delete downloads[url];
                return callback(err);
            }
            fs.rename(dl, filepath, function(err) {
                downloads[url].emit('done', err, filepath);
                delete downloads[url];
                return callback(err, filepath);
            });
        });
    });
};

// https://gist.github.com/707661
function mkdirP(p, mode, f) {
    var cb = f || function() {};
    if (p.charAt(0) != '/') {
        cb(new Error('Relative path: ' + p));
        return;
    }

    var ps = path.normalize(p).split('/');
    path.exists(p, function(exists) {
        if (exists) cb(null);
        else mkdirP(ps.slice(0, -1).join('/'), mode, function(err) {
            if (err && err.code !== 'EEXIST') cb(err);
            else {
                fs.mkdir(p, mode, cb);
            }
        });
    });
};

// Recursive rm.
function rm(filepath, callback) {
    var killswitch = false;
    fs.stat(filepath, function(err, stat) {
        if (err) return callback(err);
        if (stat.isFile()) return fs.unlink(filepath, callback);
        if (!stat.isDirectory()) return callback(new Error('Unrecognized file.'));
        Step(function() {
            fs.readdir(filepath, this);
        },
        function(err, files) {
            if (err) throw err;
            if (files.length === 0) return this(null, []);
            var group = this.group();
            _(files).each(function(file) {
                rm(path.join(filepath, file), group());
            });
        },
        function(err) {
            if (err) return callback(err);
            fs.rmdir(filepath, callback);
        });
    });
};

// Generate the cache path for a given URL.
function cachepath(location) {
    var uri = url.parse(location);
    if (!uri.protocol) throw new Error('Invalid URL: ' + location);

    var hash = crypto.createHash('md5')
        .update(location)
        .digest('hex')
        .substr(0,8)
        + '-' + path.basename(uri.pathname, path.extname(uri.pathname));
    var extname = path.extname(uri.pathname);
    return _(['.shp', '.zip']).include(extname.toLowerCase())
        ? path.join(hash, hash + extname)
        : path.join(hash + extname);
};

// Unzip function, geared specifically toward unpacking a shapefile.
function unzip(file, callback) {
    try {
        var zf = new zipfile.ZipFile(file);
    } catch (err) {
        return callback(err);
    }

    var remaining = zf.names.length;
    var shp = _(zf.names).chain()
        .map(function(name) {
            if (path.extname(name).toLowerCase() !== '.shp') return;
            return path.join(
                path.dirname(file),
                path.basename(file, path.extname(file)) +
                path.extname(name).toLowerCase()
            );
        })
        .compact()
        .first()
        .value();
    if (!shp) return callback(new Error('Shapefile not found in zip ' + file));

    zf.names.forEach(function(name) {
        // Skip directories, hiddens.
        if (!path.extname(name) || name[0] === '.') {
            remaining--;
            if (!remaining) callback(null, shp);
        }
        // We're brutal in our expectations -- don't support nested
        // directories, and rename any file from `arbitraryName.SHP`
        // to `[hash].shp`.
        var dest = path.join(
            path.dirname(file),
            path.basename(file, path.extname(file)) +
            path.extname(name).toLowerCase()
        );
        zf.readFile(name, function(err, buff) {
            if (err) return callback(err);
            fs.open(dest, 'w', 0755, function(err, fd) {
                if (err) return callback(err);
                fs.write(fd, buff, 0, buff.length, null, function(err) {
                    if (err) return callback(err);
                    fs.close(fd, function(err) {
                        if (err) return callback(err);
                        remaining--;
                        if (!remaining) callback(null, shp);<commit_msg>No +x on unzipped files.<commit_after>var fs = require('fs');
var path = require('path');
var url = require('url');
var crypto = require('crypto');
var EventEmitter = require('events').EventEmitter;

// Third party modules
var _ = require('underscore');
var srs = require('srs');
var get = require('get');
var zipfile = require('zipfile');
var Step = require('step');

// Known SRS values
var SRS = {
    'WGS84': '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs',
    '900913': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over'
};

var downloads = {};
var pool = require('generic-pool').Pool({
    create: function(callback) { callback(null, {}); },
    destroy: function(obj) { delete obj; },
    max: 5
});

function download(url, filepath, callback) {
    var dl = filepath + '.download'
    if (downloads[url]) return downloads[url].once('done', callback);

    downloads[url] = new EventEmitter();
    pool.acquire(function(obj) {
        (new get(url)).toDisk(dl, function(err, file) {
            pool.release(obj);
            if (err) {
                downloads[url].emit('done', err);
                delete downloads[url];
                return callback(err);
            }
            fs.rename(dl, filepath, function(err) {
                downloads[url].emit('done', err, filepath);
                delete downloads[url];
                return callback(err, filepath);
            });
        });
    });
};

// https://gist.github.com/707661
function mkdirP(p, mode, f) {
    var cb = f || function() {};
    if (p.charAt(0) != '/') {
        cb(new Error('Relative path: ' + p));
        return;
    }

    var ps = path.normalize(p).split('/');
    path.exists(p, function(exists) {
        if (exists) cb(null);
        else mkdirP(ps.slice(0, -1).join('/'), mode, function(err) {
            if (err && err.code !== 'EEXIST') cb(err);
            else {
                fs.mkdir(p, mode, cb);
            }
        });
    });
};

// Recursive rm.
function rm(filepath, callback) {
    var killswitch = false;
    fs.stat(filepath, function(err, stat) {
        if (err) return callback(err);
        if (stat.isFile()) return fs.unlink(filepath, callback);
        if (!stat.isDirectory()) return callback(new Error('Unrecognized file.'));
        Step(function() {
            fs.readdir(filepath, this);
        },
        function(err, files) {
            if (err) throw err;
            if (files.length === 0) return this(null, []);
            var group = this.group();
            _(files).each(function(file) {
                rm(path.join(filepath, file), group());
            });
        },
        function(err) {
            if (err) return callback(err);
            fs.rmdir(filepath, callback);
        });
    });
};

// Generate the cache path for a given URL.
function cachepath(location) {
    var uri = url.parse(location);
    if (!uri.protocol) throw new Error('Invalid URL: ' + location);

    var hash = crypto.createHash('md5')
        .update(location)
        .digest('hex')
        .substr(0,8)
        + '-' + path.basename(uri.pathname, path.extname(uri.pathname));
    var extname = path.extname(uri.pathname);
    return _(['.shp', '.zip']).include(extname.toLowerCase())
        ? path.join(hash, hash + extname)
        : path.join(hash + extname);
};

// Unzip function, geared specifically toward unpacking a shapefile.
function unzip(file, callback) {
    try {
        var zf = new zipfile.ZipFile(file);
    } catch (err) {
        return callback(err);
    }

    var remaining = zf.names.length;
    var shp = _(zf.names).chain()
        .map(function(name) {
            if (path.extname(name).toLowerCase() !== '.shp') return;
            return path.join(
                path.dirname(file),
                path.basename(file, path.extname(file)) +
                path.extname(name).toLowerCase()
            );
        })
        .compact()
        .first()
        .value();
    if (!shp) return callback(new Error('Shapefile not found in zip ' + file));

    zf.names.forEach(function(name) {
        // Skip directories, hiddens.
        if (!path.extname(name) || name[0] === '.') {
            remaining--;
            if (!remaining) callback(null, shp);
        }
        // We're brutal in our expectations -- don't support nested
        // directories, and rename any file from `arbitraryName.SHP`
        // to `[hash].shp`.
        var dest = path.join(
            path.dirname(file),
            path.basename(file, path.extname(file)) +
            path.extname(name).toLowerCase()
        );
        zf.readFile(name, function(err, buff) {
            if (err) return callback(err);
            fs.open(dest, 'w', 0644, function(err, fd) {
                if (err) return callback(err);
                fs.write(fd, buff, 0, buff.length, null, function(err) {
                    if (err) return callback(err);
                    fs.close(fd, function(err) {
                        if (err) return callback(err);
                        remaining--;
                        if (!remaining) callback(null, shp);
{'fraction_non_alphanumeric': 0.1601158860453887, 'fraction_numerical': 0.011395461129888942, 'mean_word_length': 2.0566706021251475, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6463381', 'n_tokens_mistral': 3294, 'n_tokens_neox': 3171, 'n_words': 733}