<commit_before>'use strict';

require('./crypt');

var fs = require('fs');
var gulp = require('gulp');
var $ = require('gulp-load-plugins')();
var mkdirp = require('mkdirp');
var path = require('path');
var Mustache = require('mustache');
var _ = require('lodash');

var Config = require('./config');

gulp.task('templates:create', ['config:load'], function(done){
    var expand = require('glob-expand');

    //Fetch reports
    var reports  = expand(Config.paths.reports);
    var reportSources = [];
    reports.forEach(function(report){
        reportSources.push(fs.readFileSync(report, 'utf-8'));
    });

    //Fetch elastic search mappings
    var mappingsFile  = expand(Config.paths.elasticSearchMapping)[0];
    var elasticSearchMapping = '()';
    if(fs.existsSync(mappingsFile)) {
        elasticSearchMapping = fs.readFileSync(mappingsFile, 'utf-8');
    }

    //Indicates which datasource is for query and which one is for search
    var template = '{ name: "<%= name %>", category: "<%= category %>", db: "<%= credentials.db %>" }';
    var queryDatasource  = _.chain(Config.credentials['28'].datasources)
            .filter(function(source){ return source.types.indexOf('query') !== -1; })
            .map(function(source){ return _.template(template)(source); }).value()[0] || '()';
    var searchDatasource = _.chain(Config.credentials['28'].datasources)
            .filter(function(source){ return source.types.indexOf('search') !== -1; })
            .map(function(source){ return _.template(template)(source); }).value()[0] || '()';

    //API documentation
    var queries = JSON.parse(fs.readFileSync('swagger/queries.json', 'utf-8'));
    queries.title = 'Cell Store Query API';
    var templates = [
        {
            src: 'tasks/templates/config.js.mustache',
            data: {
                cellstore: Config.credentials.cellstore,
                staging: {
                    isCI: Config.isOnTravis,
                    e2eReportsDir: Config.paths.e2eReportsDir,
                    configId: Config.configId
                },
                http: Config.credentials.http
            },
            dest: 'tests/e2e/config/config.js'
        },
        {
            src: 'tasks/templates/config.jq.mustache',
            data: {
                cellstore: Config.credentials.cellstore,
                sendmail: Config.credentials.sendmail,
                frontend: {
                    url: Config.bucketName
                },
                elasticSearchMapping: elasticSearchMapping,
                queryDatasource: queryDatasource,
                searchDatasource: searchDatasource
            },
            dest: Config.paths.queries + '/modules/io/28/apps/config.jq'
        },
        {
            src: 'tasks/templates/UpdateReportSchema.jq.mustache',
            data: {
                reports: reportSources.join(',')
            },
            dest: Config.paths.queries + '/private/UpdateReportSchema.jq'
        },
        {
            src: 'tasks/templates/api.md.mustache',
            data: {
                api: queries
            },
            dest: 'documentation/api/queries.md'
        },
        {
            src: 'tasks/templates/constants.js.mustache',
            data: {
                APPNAME: Config.projectName,
                API_URL: Config.credentials.cellstore.protocol + '://' + Config.projectName + '.' + Config.credentials['28'].portalDomain + '/v1',
                DEBUG: false,
                ACCOUNT_URL: '/account/info',
                REGISTRATION_URL: '/auth',
                PROFILE: Config.credentials.cellstore.profile,
                EXCEL_RULE_TEMPLATE: fs.readFileSync(__dirname + '/templates/excel-rule.template', 'utf-8').toString().replace(/[\r\n]+/g,'\\n')
            },
            dest: Config.paths.app + '/constants.js'
        }
    ];

    templates.forEach(function(tpl){
        var src = fs.readFileSync(tpl.src, 'utf-8');
        var result = Mustache.render(src, tpl.data);
        mkdirp.sync(path.dirname(tpl.dest));
        fs.writeFileSync(tpl.dest, result, 'utf-8');
        $.util.log('created template: ' + tpl.dest);
    });
    done();
});
<commit_msg>:bug: fix #982<commit_after>'use strict';

require('./crypt');

var fs = require('fs');
var gulp = require('gulp');
var $ = require('gulp-load-plugins')();
var mkdirp = require('mkdirp');
var path = require('path');
var Mustache = require('mustache');
var _ = require('lodash');

var Config = require('./config');

gulp.task('templates:create', ['config:load'], function(done){
    var expand = require('glob-expand');

    //Fetch reports
    var reports  = expand(Config.paths.reports);
    var reportSources = [];
    reports.forEach(function(report){
        reportSources.push(fs.readFileSync(report, 'utf-8'));
    });

    //Fetch elastic search mappings
    var mappingsFile  = expand(Config.paths.elasticSearchMapping)[0];
    var elasticSearchMapping = '()';
    if(fs.existsSync(mappingsFile)) {
        elasticSearchMapping = fs.readFileSync(mappingsFile, 'utf-8');
    }

    //Indicates which datasource is for query and which one is for search
    var template = '{ name: "<%= name %>", category: "<%= category %>", db: "<%= credentials.db %>" }';
    var queryDatasource  = _.chain(Config.credentials['28'].datasources)
            .filter(function(source){ return source.types.indexOf('query') !== -1; })
            .map(function(source){ return _.template(template)(source); }).value()[0] || '()';
    var searchDatasource = _.chain(Config.credentials['28'].datasources)
            .filter(function(source){ return source.types.indexOf('search') !== -1; })
            .map(function(source){ return _.template(template)(source); }).value()[0] || '()';

    //API documentation
    var queries = JSON.parse(fs.readFileSync('swagger/queries.json', 'utf-8'));
    queries.title = 'Cell Store Query API';
    var templates = [
        {
            src: 'tasks/templates/config.js.mustache',
            data: {
                cellstore: Config.credentials.cellstore,
                staging: {
                    isCI: Config.isOnTravis,
                    e2eReportsDir: Config.paths.e2eReportsDir,
                    configId: Config.configId
                },
                http: Config.credentials.http
            },
            dest: 'tests/e2e/config/config.js'
        },
        {
            src: 'tasks/templates/config.jq.mustache',
            data: {
                cellstore: Config.credentials.cellstore,
                sendmail: Config.credentials.sendmail,
                frontend: {
                    url: Config.bucketName
                },
                elasticSearchMapping: elasticSearchMapping,
                queryDatasource: queryDatasource,
                searchDatasource: searchDatasource
            },
            dest: Config.paths.queries + '/modules/io/28/apps/config.jq'
        },
        {
            src: 'tasks/templates/UpdateReportSchema.jq.mustache',
            data: {
                reports: reportSources.join(',')
            },
            dest: Config.paths.queries + '/private/UpdateReportSchema.jq'
        },
        {
            src: 'tasks/templates/api.md.mustache',
            data: {
                api: queries
            },
            dest: 'documentation/api/queries.md'
        },
        {
            src: 'tasks/templates/constants.js.mustache',
            data: {
                APPNAME: Config.projectName,
                API_URL: Config.credentials.cellstore.protocol + '://' + Config.projectName + '.' + Config.credentials['28'].portalDomain + '/v1',
                DEBUG: false,
                ACCOUNT_URL: '/account/info',
                REGISTRATION_URL: Config.isOnProduction ? 'https://app.secxbrl.info/auth/sign-up' : '/auth',
                PROFILE: Config.credentials.cellstore.profile,
                EXCEL_RULE_TEMPLATE: fs.readFileSync(__dirname + '/templates/excel-rule.template', 'utf-8').toString().replace(/[\r\n]+/g,'\\n')
            },
            dest: Config.paths.app + '/constants.js'
        }
    ];

    templates.forEach(function(tpl){
        var src = fs.readFileSync(tpl.src, 'utf-8');
        var result = Mustache.render(src, tpl.data);
        mkdirp.sync(path.dirname(tpl.dest));
        fs.writeFileSync(tpl.dest, result, 'utf-8');
        $.util.log('created template: ' + tpl.dest);
    });
    done();
});

{'fraction_non_alphanumeric': 0.1500059150597421, 'fraction_numerical': 0.005796758547261327, 'mean_word_length': 2.0619340818544005, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1831094', 'n_tokens_mistral': 2483, 'n_tokens_neox': 2382, 'n_words': 406}