<commit_before>var _ = require('underscore');
var carto = require('carto');
var crypto = require('crypto');
var fs = require('fs');
var path = require('path');
var util = require('util');
var url = require('url');
var mkdirp = require('mkdirp');
var sm = new (require('sphericalmercator'));
var yaml = require('js-yaml');
var tm = require('./tm');
var MBTiles = require('mbtiles');
var Bridge = require('tilelive-bridge');
var xray = require('tilelive-vector').xray;
var TileJSON = require('tilejson');
var tilelive = require('tilelive');
var style;
var CachingTileJSON = require('./cache');
var CachingBridge = require('./cache');
var mapnik = require('mapnik');
var mapnikref = require('mapnik-reference').version.latest;
var upload = require('mapbox-upload');
var progress = require('progress-stream');
var task = require('./task');
var https = require('https');
var zlib = require('zlib');

var defaults = {
    name:'',
    description:'',
    attribution:'',
    minzoom:0,
    maxzoom:6,
    center:[0,0,3],
    Layer:[],
    _prefs: {
        saveCenter: true,
        disabled: [],
        inspector: false,
        mapid: '',
        rev: ''
    }
};
var deflayer = {
    id:'',
    srs:'',
    description:'',
    fields: {},
    Datasource: {},
    properties: {
        'buffer-size': 8
    }
};

var cache = {};

module.exports = source;
tilelive.protocols['mapbox:'] = source;
tilelive.protocols['tmsource:'] = source;
tilelive.protocols['http:'] = source;
tilelive.protocols['https:'] = source;

source.defaults = defaults;
source.deflayer = deflayer;

var protocolIsValid = function (protocol) {
    return _(['tmsource:', 'mapbox:', 'http:', 'https:']).contains(protocol);
};

function source(arg, callback) {
    if ('string' !== typeof arg) {
        var id = url.format(arg);
        var uri = arg;
    } else {
        var id = arg;
        var uri = tm.parse(arg);
    }

    if (!protocolIsValid(uri.protocol))
        return callback(new Error('Invalid source protocol'));

    if (cache[id]) return callback(null, cache[id]);

    source.info(id, function(err, data) {
        if (err) return callback(err);
        try {
            data = source.normalize(data);
        } catch(err) {
            return callback(err);
        }
        source.toXML(data, function(err, xml) {
            if (err) return callback(err);
            source.refresh(data, xml, callback);
        });
    });
}

// Load or refresh the relevant source using specified data + xml.
source.refresh = function(data, xml, callback) {
    var id = data.id;
    var uri = tm.parse(data.id);
    var done = function(err, p) {
        if (err) return callback(err);
        cache[id] = cache[id] || p;
        cache[id].data = data;

        xray({
            source: cache[id],
            minzoom: data.minzoom,
            maxzoom: data.maxzoom,
            vector_layers: (data.vector_layers||[]).filter(function(l) {
                return data._prefs.disabled.indexOf(l.id) === -1
            })
        }, function(err, xraystyle) {
            if (err) return callback(err);
            cache[id].style = xraystyle;
            source.invalidate(id, function(err) {
                if (err) return callback(err);
                return callback(null, cache[id]);
            });
        });
    };
    if (xml) {
        var opts = {};
        opts.xml = xml;
        opts.base = !source.tmpid(id) && uri.dirname;
        var cb = CachingBridge(Bridge, tm.config().cache);
        return cache[id] ? cache[id].update(opts, done) : new cb(opts, done);
    } else {
        var ctj = CachingTileJSON(TileJSON, tm.config().cache);
        return cache[id] ? done(null, cache[id]) : new ctj({data:data}, done);
    }
};

// Writing.
source.save = function(data, callback) {
    var id = data.id;
    var uri = tm.parse(data.id);
    var perm = !source.tmpid(id);
    var remote = ['mapbox:', 'http:', 'https:'].indexOf(uri.protocol) >= 0;

    // "Soft" write for remote sources.
    if (remote) return source.info(id, function(err, info) {
        if (err) return callback(err);
        info._prefs = data._prefs || info._prefs;
        try {
            info = source.normalize(info);
        } catch(err) {
            return callback(err);
        }
        source.refresh(info, null, callback);
    });

    data = _(data).defaults(defaults);
    data._tmp = source.tmpid(id);
    try {
        data = source.normalize(data);
    } catch(err) {
        return callback(err);
    }

    // validate key info keys.
    var err = tilelive.verify(data, [
        'name',
        'description',
        'attribution',
        'center',
        'minzoom',
        'maxzoom'
    ]);
    if (err) return callback(err);

    source.toXML(data, function(err, xml) {
        if (err) return callback(err);
        if (!perm) return source.refresh(data, xml, callback);

        var files = [];
        var filtered = tm.filterkeys(data, defaults);
        filtered.Layer = filtered.Layer.map(function(l) { return tm.filterkeys(l, deflayer) });
        files.push({
            basename: 'data.yml',
            data: yaml.dump(tm.sortkeys(filtered), null, 2)
        });
        files.push({ basename: 'data.xml', data: xml });

        tm.writefiles(uri.dirname, files, function(err) {
            if (err) return callback(err);
            source.refresh(data, xml, function(err, p) {
                if (err) return callback(err);
                source.thumbSave(id, path.join(uri.dirname,'.thumb.png'));
                callback(null, p);
            });
        });
    });
};

// Helper for tm.tmpid().
source.tmpid = function(id, md5) { return tm.tmpid('tmsource:', id, md5) };

source.toXML = function(data, callback) {<commit_msg>Exception for 0-length vector layer check.<commit_after>var _ = require('underscore');
var carto = require('carto');
var crypto = require('crypto');
var fs = require('fs');
var path = require('path');
var util = require('util');
var url = require('url');
var mkdirp = require('mkdirp');
var sm = new (require('sphericalmercator'));
var yaml = require('js-yaml');
var tm = require('./tm');
var MBTiles = require('mbtiles');
var Bridge = require('tilelive-bridge');
var xray = require('tilelive-vector').xray;
var TileJSON = require('tilejson');
var tilelive = require('tilelive');
var style;
var CachingTileJSON = require('./cache');
var CachingBridge = require('./cache');
var mapnik = require('mapnik');
var mapnikref = require('mapnik-reference').version.latest;
var upload = require('mapbox-upload');
var progress = require('progress-stream');
var task = require('./task');
var https = require('https');
var zlib = require('zlib');

var defaults = {
    name:'',
    description:'',
    attribution:'',
    minzoom:0,
    maxzoom:6,
    center:[0,0,3],
    Layer:[],
    _prefs: {
        saveCenter: true,
        disabled: [],
        inspector: false,
        mapid: '',
        rev: ''
    }
};
var deflayer = {
    id:'',
    srs:'',
    description:'',
    fields: {},
    Datasource: {},
    properties: {
        'buffer-size': 8
    }
};

var cache = {};

module.exports = source;
tilelive.protocols['mapbox:'] = source;
tilelive.protocols['tmsource:'] = source;
tilelive.protocols['http:'] = source;
tilelive.protocols['https:'] = source;

source.defaults = defaults;
source.deflayer = deflayer;

var protocolIsValid = function (protocol) {
    return _(['tmsource:', 'mapbox:', 'http:', 'https:']).contains(protocol);
};

function source(arg, callback) {
    if ('string' !== typeof arg) {
        var id = url.format(arg);
        var uri = arg;
    } else {
        var id = arg;
        var uri = tm.parse(arg);
    }

    if (!protocolIsValid(uri.protocol))
        return callback(new Error('Invalid source protocol'));

    if (cache[id]) return callback(null, cache[id]);

    source.info(id, function(err, data) {
        if (err) return callback(err);
        try {
            data = source.normalize(data);
        } catch(err) {
            return callback(err);
        }
        source.toXML(data, function(err, xml) {
            if (err) return callback(err);
            source.refresh(data, xml, callback);
        });
    });
}

// Load or refresh the relevant source using specified data + xml.
source.refresh = function(data, xml, callback) {
    var id = data.id;
    var uri = tm.parse(data.id);
    var done = function(err, p) {
        if (err) return callback(err);
        cache[id] = cache[id] || p;
        cache[id].data = data;

        xray({
            source: cache[id],
            minzoom: data.minzoom,
            maxzoom: data.maxzoom,
            vector_layers: (data.vector_layers||[]).filter(function(l) {
                return data._prefs.disabled.indexOf(l.id) === -1
            })
        }, function(err, xraystyle) {
            if (err) return callback(err);
            cache[id].style = xraystyle;
            source.invalidate(id, function(err) {
                if (err) return callback(err);
                return callback(null, cache[id]);
            });
        });
    };
    if (xml) {
        var opts = {};
        opts.xml = xml;
        opts.base = !source.tmpid(id) && uri.dirname;
        var cb = CachingBridge(Bridge, tm.config().cache);
        return cache[id] ? cache[id].update(opts, done) : new cb(opts, done);
    } else {
        var ctj = CachingTileJSON(TileJSON, tm.config().cache);
        return cache[id] ? done(null, cache[id]) : new ctj({data:data}, done);
    }
};

// Writing.
source.save = function(data, callback) {
    var id = data.id;
    var uri = tm.parse(data.id);
    var perm = !source.tmpid(id);
    var remote = ['mapbox:', 'http:', 'https:'].indexOf(uri.protocol) >= 0;

    // "Soft" write for remote sources.
    if (remote) return source.info(id, function(err, info) {
        if (err) return callback(err);
        info._prefs = data._prefs || info._prefs;
        try {
            info = source.normalize(info);
        } catch(err) {
            return callback(err);
        }
        source.refresh(info, null, callback);
    });

    data = _(data).defaults(defaults);
    data._tmp = source.tmpid(id);
    try {
        data = source.normalize(data);
    } catch(err) {
        return callback(err);
    }

    // validate key info keys.
    var err = tilelive.verify(data, [
        'name',
        'description',
        'attribution',
        'center',
        'minzoom',
        'maxzoom'
    ]);
    // Exception for vector_layers length check. While sources are being
    // edited a vector_layers array of length 0 is ok.
    if (err && err.message !== 'vector_layers must be an array of layer objects') {
        return callback(err);
    }

    source.toXML(data, function(err, xml) {
        if (err) return callback(err);
        if (!perm) return source.refresh(data, xml, callback);

        var files = [];
        var filtered = tm.filterkeys(data, defaults);
        filtered.Layer = filtered.Layer.map(function(l) { return tm.filterkeys(l, deflayer) });
        files.push({
            basename: 'data.yml',
            data: yaml.dump(tm.sortkeys(filtered), null, 2)
        });
        files.push({ basename: 'data.xml', data: xml });

        tm.writefiles(uri.dirname, files, function(err) {
            if (err) return callback(err);
            source.refresh(data, xml, function(err, p) {
                if (err) return callback(err);
                source.thumbSave(id, path.join(uri.dirname,'.thumb.png'));
                callback(null, p);
            });
        });
    });
};

// Helper for tm.tmpid().
source.tmpid = function(id, md5) { return tm.tmpid('tmsource:', id, md5) };

source.toXML = function(data, callback) {
{'fraction_non_alphanumeric': 0.16163167776071002, 'fraction_numerical': 0.002048131080389145, 'mean_word_length': 2.4226051401869158, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1144882', 'n_tokens_mistral': 3654, 'n_tokens_neox': 3603, 'n_words': 870}