<commit_before>/**
 * This module contains utility methods used to load and prepare TileMill
 * projects as a source of tiles
 */
var _ = require('underscore');
var Path = require('path')
var Url = require('url');
var FS = require('fs');
var Crypto = require('crypto');
var Http = require('http');

var Q = require('q');
var CartoJsonCss = require('./carto-json-css');
var Carto = require('carto');
var AdmZip = require('adm-zip');

function TileSourceProvider() {
    if (_.isFunction(this.initialize)) {
        this.initialize.apply(this, arguments);
    }
}
_.extend(TileSourceProvider.prototype, {
    initialize : function(options) {
        this.options = options;
    },
    loadTileSource : function(params) {
    },
    clearTileSource : function(params) {
    }
});

TileSourceProvider.TileMillProvider = TileMillSourceProvider;
function TileMillSourceProvider() {
    TileSourceProvider.apply(this, arguments);
}
_.extend(TileMillSourceProvider.prototype, TileSourceProvider.prototype);
_.extend(TileMillSourceProvider.prototype, {

    clearTileSource : function(params) {
        var that = this;
        return Q().then(function() {
            var sourceKey = params.source;
            var xmlTilepinFile = that._getTilepinProjectFile(sourceKey);
            if (!FS.existsSync(xmlTilepinFile))
                return true;
            return Q.nfcall(FS.unlink, xmlTilepinFile).then(function() {
                return true;
            });
        })
    },

    /**
     * This method loads and returns an object contining tilesource XML and a
     * base directory for this tilesource.
     */
    loadTileSource : function(params) {
        var that = this;
        var baseDir;
        var sourceFile;
        return that._prepareTileSourceFile(params).then(function(file) {
            sourceFile = file;
            baseDir = Path.dirname(sourceFile);
            return {
                base : baseDir,
                path : sourceFile
            };
        });
    },

    _getTileSourceFile : function(sourceKey, fileName) {
        var dir = this.options.dir || this.options.styleDir || __dirname;
        dir = Path.join(dir, sourceKey);
        dir = Path.resolve(dir);
        var fullPath = Path.join(dir, fileName);
        return fullPath;
    },

    _findDataIndex : function(dir, url) {
        return Q.nfcall(FS.readdir, dir).then(function(list) {
            var result = null;
            _.each(list, function(file) {
                if (file.lastIndexOf('.shp') > 0) {
                    result = Path.join(dir, file);
                }
            })
            return result;
        })
    },

    _getSha1 : function(str) {
        var shasum = Crypto.createHash('sha1');
        shasum.update(str);
        var hash = shasum.digest('hex');
        return hash;
    },

    _downloadAndUnzip : function(url, dir) {
        var that = this;
        var obj = Url.parse(url);
        var ext = Path.extname(obj.pathname);
        var path = Path.basename(obj.pathname, ext) || '.tmp-'
                + that._getSha1(url);
        var dataDir = Path.join(dir, path);
        var dataFileName = Path.join(dataDir, Path.basename(obj.pathname));
        var promise = Q();
        if (!FS.existsSync(dataDir)) {
            promise = promise.then(function() {
                var segments = dataDir.split('/');
                var p = '';
                _.each(segments, function(segment) {
                    p = (p == '' && segment == '') ? '/' : Path
                            .join(p, segment);
                    if (!FS.existsSync(p)) {
                        FS.mkdirSync(p);
                    }
                });
            })
        }
        promise = promise.then(function() {
            if (!FS.existsSync(dataFileName)) {
                return that._download(dataFileName, url)
            }
        }).then(function() {
            return that._unzip(dataFileName, dataDir);
        }).then(function() {
            return dataDir;
        });
    },

    _getMaxRedirects : function() {
        return 10;
    },

    _download : function(dataFileName, url) {
        var that = this;
        function httpGet(options, redirectCount) {
            var diferred = Q.defer();
            try {
                if (redirectCount >= that._getMaxRedirects()) {
                    throw new Error('Too many redirections. (' + redirectCount
                            + ').');
                }<commit_msg>bug: no returned value <commit_after>/**
 * This module contains utility methods used to load and prepare TileMill
 * projects as a source of tiles
 */
var _ = require('underscore');
var Path = require('path')
var Url = require('url');
var FS = require('fs');
var Crypto = require('crypto');
var Http = require('http');

var Q = require('q');
var CartoJsonCss = require('./carto-json-css');
var Carto = require('carto');
var AdmZip = require('adm-zip');

function TileSourceProvider() {
    if (_.isFunction(this.initialize)) {
        this.initialize.apply(this, arguments);
    }
}
_.extend(TileSourceProvider.prototype, {
    initialize : function(options) {
        this.options = options;
    },
    loadTileSource : function(params) {
    },
    clearTileSource : function(params) {
    }
});

TileSourceProvider.TileMillProvider = TileMillSourceProvider;
function TileMillSourceProvider() {
    TileSourceProvider.apply(this, arguments);
}
_.extend(TileMillSourceProvider.prototype, TileSourceProvider.prototype);
_.extend(TileMillSourceProvider.prototype, {

    clearTileSource : function(params) {
        var that = this;
        return Q().then(function() {
            var sourceKey = params.source;
            var xmlTilepinFile = that._getTilepinProjectFile(sourceKey);
            if (!FS.existsSync(xmlTilepinFile))
                return true;
            return Q.nfcall(FS.unlink, xmlTilepinFile).then(function() {
                return true;
            });
        })
    },

    /**
     * This method loads and returns an object contining tilesource XML and a
     * base directory for this tilesource.
     */
    loadTileSource : function(params) {
        var that = this;
        var baseDir;
        var sourceFile;
        return that._prepareTileSourceFile(params).then(function(file) {
            sourceFile = file;
            baseDir = Path.dirname(sourceFile);
            return {
                base : baseDir,
                path : sourceFile
            };
        });
    },

    _getTileSourceFile : function(sourceKey, fileName) {
        var dir = this.options.dir || this.options.styleDir || __dirname;
        dir = Path.join(dir, sourceKey);
        dir = Path.resolve(dir);
        var fullPath = Path.join(dir, fileName);
        return fullPath;
    },

    _findDataIndex : function(dir, url) {
        return Q.nfcall(FS.readdir, dir).then(function(list) {
            var result = null;
            _.each(list, function(file) {
                if (file.lastIndexOf('.shp') > 0) {
                    result = Path.join(dir, file);
                }
            })
            return result;
        })
    },

    _getSha1 : function(str) {
        var shasum = Crypto.createHash('sha1');
        shasum.update(str);
        var hash = shasum.digest('hex');
        return hash;
    },

    _downloadAndUnzip : function(url, dir) {
        var that = this;
        var obj = Url.parse(url);
        var ext = Path.extname(obj.pathname);
        var path = Path.basename(obj.pathname, ext) || '.tmp-'
                + that._getSha1(url);
        var dataDir = Path.join(dir, path);
        var dataFileName = Path.join(dataDir, Path.basename(obj.pathname));
        var promise = Q();
        if (!FS.existsSync(dataDir)) {
            promise = promise.then(function() {
                var segments = dataDir.split('/');
                var p = '';
                _.each(segments, function(segment) {
                    p = (p == '' && segment == '') ? '/' : Path
                            .join(p, segment);
                    if (!FS.existsSync(p)) {
                        FS.mkdirSync(p);
                    }
                });
            })
        }
        return promise.then(function() {
            if (!FS.existsSync(dataFileName)) {
                return that._download(dataFileName, url)
            }
        }).then(function() {
            return that._unzip(dataFileName, dataDir);
        }).then(function() {
            return dataDir;
        });
    },

    _getMaxRedirects : function() {
        return 10;
    },

    _download : function(dataFileName, url) {
        var that = this;
        function httpGet(options, redirectCount) {
            var diferred = Q.defer();
            try {
                if (redirectCount >= that._getMaxRedirects()) {
                    throw new Error('Too many redirections. (' + redirectCount
                            + ').');
                }
{'fraction_non_alphanumeric': 0.1238424634608948, 'fraction_numerical': 0.0013388374428204841, 'mean_word_length': 1.8475222363405337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2315785', 'n_tokens_mistral': 2508, 'n_tokens_neox': 2431, 'n_words': 540}