<commit_before>var SourceType 		= require('mongoose').model('SourceType'),
    Source   		= require('mongoose').model('Source'),
    Project   		= require('mongoose').model('Project'),
    Link 	        = require('mongoose').model('Link'),
    async           = require('async'),
    _               = require("underscore"),
    request         = require('request'),
    encrypt 		= require('../utilities/encryption');
exports.getSourceTypes = function(req, res) {
    var limit = Number(req.params.limit),
        skip = Number(req.params.skip);

    var project_id = [];
    async.waterfall([
        sourceTypeCount,
        getProjects,
        getLinkProjects,
        getSourceTypeSet
    ], function (err, result) {
        if (err) {
            res.send(err);
        } else{
            if (req.query && req.query.callback) {
                return res.jsonp("" + req.query.callback + "(" + JSON.stringify(result) + ");");
            } else {
                return res.send(result);
            }
        }
    });

    function sourceTypeCount(callback) {
        SourceType.find({}).count().exec(function(err, source_type_count) {
            if(source_type_count) {
                callback(null, source_type_count);
            } else {
                callback(err);
            }
        });
    }
    function getProjects(source_type_count, callback) {
        var sources = [];
        Project.find({})
            .select("_id proj_country.country proj_country.source proj_established_source")
            .populate('proj_established_source')
            .exec(function(err, projects) {
                if(projects.length>0) {
                    _.map(_.groupBy(projects,function(doc){
                        if(doc.proj_established_source!=undefined && doc.proj_established_source!=null) {
                            return doc.proj_established_source.source_type_id;
                        }
                    }),function(grouped){
                        if(grouped[0].proj_established_source!=undefined && grouped[0].proj_established_source!=null && grouped[0].proj_established_source.source_type_id!=undefined && grouped[0].proj_established_source.source_type_id!=null) {
                            sources.push({
                                '_id': grouped[0].proj_established_source.source_type_id,
                                'source_id': grouped[0].proj_established_source._id,
                                'count': grouped.length

                            })
                        }
                        return sources;
                    });
                    callback(null, source_type_count, sources);
                } else {
                    callback(null, source_type_count, []);
                }
            });
    }
    function getLinkProjects(source_type_count,sources, callback) {
        var link =[];
        Link.find({'entities':'project'})
            .select("source project.proj_country.country project.proj_country.source project.proj_established_source")
            .populate('proj_established_source source project company concession site contract')
            .exec(function(err, links) {
                if(links.length>0) {
                    link = _.map(_.groupBy(links,function(doc){
                        if(doc.project!=undefined && doc.project!= null
                            && doc.project.proj_established_source!=undefined
                            && doc.project.proj_established_source!=null) {
                            return doc.project.proj_established_source.source_type_id;
                        }
                    }),function(grouped){
                        if(grouped[0].project!=undefined && grouped[0].project!= null
                            && grouped[0].project.proj_established_source!=undefined
                            && grouped[0].project.proj_established_source!=null
                            && grouped[0].project.proj_established_source.source_type_id!=undefined
                            && grouped[0].project.proj_established_source.source_type_id!=null) {
                            sources.push({
                                '_id': grouped[0].project.proj_established_source.source_type_id,
                                'source_id': grouped[0].project.proj_established_source._id,
                                'count': grouped.length
                            })
                        }
                        return sources;
                    });
                    callback(null, source_type_count, sources);
                } else {
                    callback(null, source_type_count, sources);
                }
            });
    }
    function getSourceTypeSet(source_type_count,sources, callback) {
        SourceType.find({})
            .sort({
                source_type_name: 'asc'
            })
            .skip(skip)
            .limit(limit)
            .lean()
            .exec(function(err, source_types) {
                if(source_types.length>0) {
                    source_types = _.map(source_types, function (source_type) {
                        source_type.project_count = 0
                        _.each(sources, function (source) {
                            if (source._id.toString() == source_type._id.toString()) {
                                source_type.project_count = source_type.project_count + source.count;
                            }
                        })
                        return source_type;<commit_msg>populate source type list page with unique project counts and unique country coverage #153<commit_after>var SourceType 		= require('mongoose').model('SourceType'),
    Source   		= require('mongoose').model('Source'),
    Project   		= require('mongoose').model('Project'),
    Link 	        = require('mongoose').model('Link'),
    async           = require('async'),
    _               = require("underscore"),
    request         = require('request'),
    encrypt 		= require('../utilities/encryption');
exports.getSourceTypes = function(req, res) {
    var limit = Number(req.params.limit),
        skip = Number(req.params.skip);

    async.waterfall([
        sourceTypeCount,
        getProjects,
        getLinkProjects,
        getSourceTypeSet
    ], function (err, result) {
        if (err) {
            res.send(err);
        } else{
            if (req.query && req.query.callback) {
                return res.jsonp("" + req.query.callback + "(" + JSON.stringify(result) + ");");
            } else {
                return res.send(result);
            }
        }
    });

    function sourceTypeCount(callback) {
        SourceType.find({}).count().exec(function(err, source_type_count) {
            if(source_type_count) {
                callback(null, source_type_count);
            } else {
                callback(err);
            }
        });
    }
    function getProjects(source_type_count, callback) {
        var sources = [];
        Project.find({})
            .select("_id proj_country.country proj_country.source proj_established_source")
            .populate('proj_established_source proj_country.source')
            .exec(function(err, projects) {
                if(projects.length>0) {
                    _.map(_.groupBy(projects,function(doc){
                        if(doc.proj_established_source!=undefined && doc.proj_established_source!=null) {
                            return doc.proj_established_source.source_type_id;
                        }
                    }),function(grouped, id){
                        var len = grouped.length;
                        var count = [];
                        if(grouped[0].proj_established_source!=undefined
                            && grouped[0].proj_established_source!=null
                            && grouped[0].proj_established_source.source_type_id!=undefined
                            && grouped[0].proj_established_source.source_type_id!=null) {
                            _.each(grouped, function (group, key) {
                                if (group.proj_country[0] != undefined
                                    && group.proj_country[0] != null
                                    && group.proj_country[0].country != undefined
                                    && group.proj_country[0].country != null) {
                                    count.push(group.proj_country[0].country.toString())
                                    if (len - 1 == key) {
                                        var country_count = _.uniq(count);
                                        sources.push({
                                            '_id': id,
                                            'source_id': grouped[0].proj_country[0].source._id,
                                            'count': len,
                                            'country_count': country_count.length
                                        })
                                    }
                                }
                            })
                        }
                        return sources;
                    });
                    callback(null, source_type_count, sources);
                } else {
                    callback(null, source_type_count, []);
                }
            });
    }
    function getLinkProjects(source_type_count,sources, callback) {
        var link =[];
        Link.find({'entities':'project'})
            .select("source project.proj_country.country project.proj_country.source project.proj_established_source")
            .populate('source project company concession site contract project.proj_established_source project.proj_country.source')
            .exec(function(err, links) {
                if(links.length>0) {
                    link = _.map(_.groupBy(links,function(doc){
                        if(doc.project!=undefined && doc.project!= null
                            && doc.project.proj_established_source!=undefined
                            && doc.project.proj_established_source!=null) {
                            return doc.project.proj_established_source.source_type_id;
                        }
                    }),function(grouped,id){
                        var len = grouped.length;
                        var count = [];
                        if(grouped[0].project!=undefined && grouped[0].project!= null
                            && grouped[0].project.proj_established_source!=undefined
                            && grouped[0].project.proj_established_source!=null
                            && grouped[0].project.proj_established_source.source_type_id!=undefined
                            && grouped[0].project.proj_established_source.source_type_id!=null) {
                            _.each(grouped, function (group,key) {
                                if(group.proj_country[0]!=undefined
                                    && group.project.proj_country[0]!=null
                                    && group.project.proj_country[0].country!=undefined
                                    && group.project.proj_country[0].country!=null) {
                                    count.push(group.project.proj_country[0].country.toString())
                                    if (len - 1 == key) {
                                        var country_count = _.uniq(count);
                                        sources.push({
                                            '_id': id,
                                            'source_id': grouped[0].project.proj_country[0].source._id,
                                            'count': len,
                                            'country_count': country_count.length
                                        })
                                    }
                                }
                            })
                            sources.push({
                                '_id': grouped[0].project.proj_established_source.source_type_id,
                                'source_id': grouped[0].project.proj_established_source._id,
                                'count': grouped.length,
                                'country_count': 0
                            })
                        }
                        return sources;
                    });
                    callback(null, source_type_count, sources);
                } else {
                    callback(null, source_type_count, sources);
                }
            });
    }
    function getSourceTypeSet(source_type_count,sources, callback) {
        SourceType.find({})
            .sort({
                source_type_name: 'asc'
            })
            .skip(skip)
            .limit(limit)
            .lean()
            .exec(function(err, source_types) {
                if(source_types.length>0) {
                    source_types = _.map(source_types, function (source_type) {
                        source_type.project_count = 0;
                        source_type.country_count = 0;
                        _.each(sources, function (source) {
                            if (source._id.toString() == source_type._id.toString()) {
                                source_type.project_count = source_type.project_count + source.count;
                                source_type.country_count = source_type.country_count + source.country_count;
                            }
                        })
                        return source_type;
{'fraction_non_alphanumeric': 0.10264534669515879, 'fraction_numerical': 0.004052759560828237, 'mean_word_length': 1.2459043521429753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 39, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4990979', 'n_tokens_mistral': 3373, 'n_tokens_neox': 3080, 'n_words': 435}