<commit_before>'use strict';

var util = require('util');
var pathLib = require('path');
var _ = require('underscore');
var BBPromise = require('bluebird');
var yaml = require('js-yaml');

var fs = require("fs");
BBPromise.promisifyAll(fs);

// constant
var sourceByRefProtocol = 'sourceref:';

var core = require('./core');
var Err = core.Err;

function Sources(app, tilelive, npmResolver, serviceRootDir) {
    this.log = app.logger.log.bind(app.logger);
    this.tilelive = tilelive;
    this.npmResolver = npmResolver;
    this.serviceRootDir = serviceRootDir;
    this.sources = {};
    this.variables = {};

    // Set up a ref protocol resolver - this way its enough to specify the source
    // by sourceref:///?ref=sourceID URL, instead of a full source URL.
    var self = this;
    tilelive.protocols[sourceByRefProtocol] = function (uri, callback) {
        BBPromise.try(function () {
            uri = core.normalizeUri(uri);
            if (!uri.query.ref) {
                throw new Err('ref uri parameter is not set');
            }
            return self.getSourceById(uri.query.ref).handler;
        }).nodeify(callback);
    };
}

/**
 * Source ID must begin with a letter, and may contain letters, digits, and underscores
 */
Sources.isValidSourceId = function (sourceId) {
    return typeof sourceId === 'string' && sourceId.length > 0 && /^[A-Za-z]\w*$/.test(sourceId);
};

Sources.prototype.loadAsync = function(conf) {
    var self = this;
    // Load variables (e.g. passwords, etc)
    return self._localOrExternalDataAsync(conf.variables, self.serviceRootDir, 'variables')
        .then(function (variables) {
            self.variables = _.extend(self.variables, variables);
            // Load sources
            return self._localOrExternalDataAsync(conf.sources, self.serviceRootDir, 'sources');
        }).then(function (sources) {
            self.sources = _.extend(self.sources, sources);

            // Parse sources in parallel (loading on the other hand has to done in order)
            return core.mapSequentialAsync(self.sources, function (src, key) {
                if (!Sources.isValidSourceId(key))
                    throw new Err('sources.%s key must only contain letters and digits', key);
                if (typeof src !== 'object')
                    throw new Err('sources.%s must be an object', key);

                core.checkType(src, 'uri', 'string', true, 1);
                src.uri = core.normalizeUri(src.uri);

                // These params are stored, but not acted on within core
                // Kartotherian service uses them when handling user's requests
                // If public is not set or false, the rest of the values are unused
                core.checkType(src, 'public', 'boolean');
                core.checkType(src, 'minzoom', 'zoom');
                core.checkType(src, 'maxzoom', 'zoom');
                core.checkType(src, 'defaultHeaders', 'object');
                core.checkType(src, 'headers', 'object');

                // Add URI query values, e.g.  ?password=...
                if (core.checkType(src, 'params', 'object')) {
                    _.each(src.params, function (v, k) {
                        src.uri.query[k] = self._resolveValue(v, k);
                    });
                }
                // Set URI's path, e.g. /srv/data/mydir
                if (core.checkType(src, 'pathname', 'object')) {
                    src.uri.pathname = self._resolveValue(src.pathname, 'pathname');
                }
                if (src.xml) {
                    return self._loadXmlAsync(src);
                }
            });
        }).then(function () {
            return core.mapSequentialAsync(self.sources, function (src) {
                return self.tilelive
                    .loadAsync(src.uri)
                    .then(function (handler) {
                        src.handler = BBPromise.promisifyAll(handler);
                        return true;
                    });<commit_msg>added formats and maxscale source handling<commit_after>'use strict';

var util = require('util');
var pathLib = require('path');
var _ = require('underscore');
var BBPromise = require('bluebird');
var yaml = require('js-yaml');

var fs = require("fs");
BBPromise.promisifyAll(fs);

// constant
var sourceByRefProtocol = 'sourceref:';

var core = require('./core');
var Err = core.Err;

function Sources(app, tilelive, npmResolver, serviceRootDir) {
    this.log = app.logger.log.bind(app.logger);
    this.tilelive = tilelive;
    this.npmResolver = npmResolver;
    this.serviceRootDir = serviceRootDir;
    this.sources = {};
    this.variables = {};

    // Set up a ref protocol resolver - this way its enough to specify the source
    // by sourceref:///?ref=sourceID URL, instead of a full source URL.
    var self = this;
    tilelive.protocols[sourceByRefProtocol] = function (uri, callback) {
        BBPromise.try(function () {
            uri = core.normalizeUri(uri);
            if (!uri.query.ref) {
                throw new Err('ref uri parameter is not set');
            }
            return self.getSourceById(uri.query.ref).handler;
        }).nodeify(callback);
    };
}

/**
 * Source ID must begin with a letter, and may contain letters, digits, and underscores
 */
Sources.isValidSourceId = function (sourceId) {
    return typeof sourceId === 'string' && sourceId.length > 0 && /^[A-Za-z]\w*$/.test(sourceId);
};

Sources.prototype.loadAsync = function(conf) {
    var self = this;
    // Load variables (e.g. passwords, etc)
    return self._localOrExternalDataAsync(conf.variables, self.serviceRootDir, 'variables')
        .then(function (variables) {
            self.variables = _.extend(self.variables, variables);
            // Load sources
            return self._localOrExternalDataAsync(conf.sources, self.serviceRootDir, 'sources');
        }).then(function (sources) {
            self.sources = _.extend(self.sources, sources);

            // Parse sources in parallel (loading on the other hand has to done in order)
            return core.mapSequentialAsync(self.sources, function (src, key) {
                if (!Sources.isValidSourceId(key))
                    throw new Err('sources.%s key must only contain letters and digits', key);
                if (typeof src !== 'object')
                    throw new Err('sources.%s must be an object', key);

                core.checkType(src, 'uri', 'string', true, 1);
                src.uri = core.normalizeUri(src.uri);

                // These params are stored, but not acted on within core
                // Kartotherian service uses them when handling user's requests
                // If public is not set or false, the rest of the values are unused
                core.checkType(src, 'public', 'boolean');
                core.checkType(src, 'minzoom', 'zoom');
                core.checkType(src, 'maxzoom', 'zoom');
                core.checkType(src, 'defaultHeaders', 'object');
                core.checkType(src, 'headers', 'object');
                core.checkType(src, 'formats', 'string-array');
                core.checkType(src, 'maxscale', 'integer', false, 2, 9);

                // Add URI query values, e.g.  ?password=...
                if (core.checkType(src, 'params', 'object')) {
                    _.each(src.params, function (v, k) {
                        src.uri.query[k] = self._resolveValue(v, k);
                    });
                }
                // Set URI's path, e.g. /srv/data/mydir
                if (core.checkType(src, 'pathname', 'object')) {
                    src.uri.pathname = self._resolveValue(src.pathname, 'pathname');
                }
                if (src.xml) {
                    return self._loadXmlAsync(src);
                }
            });
        }).then(function () {
            return core.mapSequentialAsync(self.sources, function (src) {
                return self.tilelive
                    .loadAsync(src.uri)
                    .then(function (handler) {
                        src.handler = BBPromise.promisifyAll(handler);
                        return true;
                    });
{'fraction_non_alphanumeric': 0.12664714494875548, 'fraction_numerical': 0.0007320644216691069, 'mean_word_length': 1.9677769732078205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6366478', 'n_tokens_mistral': 2295, 'n_tokens_neox': 2178, 'n_words': 594}