<commit_before>var Concession 		= require('mongoose').model('Concession'),
    Transfer 	    = require('mongoose').model('Transfer'),
    Production 	    = require('mongoose').model('Production'),
    Link            = require('mongoose').model('Link'),
    async           = require('async'),
    _               = require("underscore");

exports.getConcessions = function(req, res) {
    var concession_len, link_len, concession_counter, link_counter,
        limit = Number(req.params.limit),
        skip = Number(req.params.skip);

    async.waterfall([
        concessionCount,
        getConcessionSet,
        getConcessionLinks,
        getTransfersCount,
        getProductionCount
    ], function (err, result) {
        if (err) {
            res.send(err);
        }
    });
    function concessionCount(callback) {
        Concession.find({}).count().exec(function(err, concession_count) {
            if(concession_count) {
                callback(null, concession_count);
            } else {
                callback(err);
            }
        });
    }
    function getConcessionSet(concession_count, callback) {
        Concession.find(req.query)
            .sort({
                concession_name: 'asc'
            })
            .skip(skip)
            .limit(limit)
            .populate('concession_country.country', '_id iso2 name')
            .populate('concession_commodity.commodity', ' _id commodity_name commodity_type commodity_id')
            .populate('concession_established_source')
            .lean()
            .exec(function(err, concessions) {
                if(concessions) {
                    //TODO clean up returned data if we see performance lags
                    callback(null, concession_count, concessions);
                } else {
                    res.send({data: concessions, count: concession_count});
                }
            });
    }
    function getConcessionLinks(concession_count, concessions, callback) {
        concession_len = concessions.length;
        concession_counter = 0;
        if(concession_len>0) {
            concessions.forEach(function (concession) {
                concession.transfers_query = [concession._id];
                concession.source_type = {p: false, c: false};
                Link.find({concession: concession._id})
                    .populate('commodity', '_id commodity_name commodity_type commodity_id')
                    .populate('company site project')
                    .deepPopulate('source.source_type_id site.site_commodity.commodity project.proj_commodity.commodity')
                    .exec(function (err, links) {
                        ++concession_counter;
                        link_len = links.length;
                        link_counter = 0;
                        if(link_len>0) {
                            concession.project_count = 0;
                            concession.company_count = 0;
                            concession.site_count = 0;
                            concession.field_count  = 0;
                            concession.contract_count = 0;
                            links.forEach(function (link) {
                                ++link_counter;
                                var entity = _.without(link.entities, 'concession')[0];
                                if (!concession.source_type.p || !concession.source_type.c) {
                                    if (link.source.source_type_id.source_type_authority === 'authoritative') {
                                        concession.source_type.c = true;
                                    } else if (link.source.source_type_id.source_type_authority === 'non-authoritative') {
                                        concession.source_type.c = true;
                                    } else if (link.source.source_type_id.source_type_authority === 'disclosure') {
                                        concession.source_type.p = true;
                                    }
                                }
                                switch (entity) {
                                    case 'company':
                                        concession.company_count += 1;
                                        break;
                                    case 'contract':
                                        concession.contract_count += 1;
                                        break;
                                    case 'project':
                                        if (link.project.proj_commodity.length>0) {
                                            if (_.where(concession.concession_commodity, {_id:_.last(link.project.proj_commodity)._id}).length<1) {
                                                concession.concession_commodity.push({
                                                    _id: _.last(link.project.proj_commodity).commodity._id,
                                                    commodity_name: _.last(link.project.proj_commodity).commodity.commodity_name,
                                                    commodity_type: _.last(link.project.proj_commodity).commodity.commodity_type,
                                                    commodity_id: _.last(link.project.proj_commodity).commodity.commodity_id
                                                });
                                            }
                                        }
                                        if (!_.contains(concession.transfers_query, link.project)) {
                                            concession.transfers_query.push(link.project);
                                        }
                                        concession.project_count += 1;
                                        break;
                                    case 'site':
                                        if (link.site.site_commodity.length>0) {
                                            if (_.where(concession.concession_commodity, {_id:_.last(link.site.site_commodity)._id}).length<1) {
                                                concession.concession_commodity.push({
                                                    _id: _.last(link.site.site_commodity)._id,
                                                    commodity_name: _.last(link.site.site_commodity).commodity.commodity_name,
                                                    commodity_type: _.last(link.site.site_commodity).commodity.commodity_type,
                                                    commodity_id: _.last(link.site.site_commodity).commodity.commodity_id
                                                });
                                            }
                                        }
                                        if (!_.contains(concession.transfers_query, link.site._id)) {
                                            concession.transfers_query.push(link.site._id);
                                        }
                                        if (link.site.field) {
                                            concession.site_count += 1;
                                        } else if (!link.site.field) {
                                            concession.field_count += 1;
                                        }
                                        break;
                                    default:
                                        console.log(entity, 'skipped...');
                                }
                            });
                            if (concession_counter == concession_len && link_counter == link_len) {
                                callback(null, concession_count, concessions);
                            }
// <<<<<<< 08f7f24303a4d45507c21cee9c0a9c3bf4bc5853
                        } else {
                            if (concession_counter == concession_len && link_counter == link_len) {
                                callback(null, concession_count, concessions);
// =======
//                             switch (entity) {
//                                 case 'company':
//                                     concession.company_count += 1;
//                                     break;
//                                 case 'contract':
//                                     concession.contract_count += 1;
//                                     break;
//                                 case 'project':
//                                     concession.projects.push(link.project._id);
//                                     concession.project_count += 1;
//                                     break;
//                                 case 'site':
//                                     if (concession.site!=undefined) {
//                                         if (concession.site.field) {
//                                             concession.sites.push(link.site._id);
//                                             concession.site_count += 1;
//                                         } else if (!concession.site.field) {
//                                             concession.fields.push(link.site._id);
//                                             concession.field_count += 1;
//                                         }
//                                     }
//                                     break;
//                                 default:
//                                     console.log(entity, 'skipped...');
// >>>>>>> SAVEPOINT
                            }
                        }

                    });
            });
        } else{
            callback(null, concession_count, concessions);<commit_msg>SAVEPOINT<commit_after>var Concession 		= require('mongoose').model('Concession'),
    Transfer 	    = require('mongoose').model('Transfer'),
    Production 	    = require('mongoose').model('Production'),
    Link            = require('mongoose').model('Link'),
    async           = require('async'),
    _               = require("underscore");

exports.getConcessions = function(req, res) {
    var concession_len, link_len, concession_counter, link_counter,
        limit = Number(req.params.limit),
        skip = Number(req.params.skip);

    async.waterfall([
        concessionCount,
        getConcessionSet,
        getConcessionLinks,
        getTransfersCount,
        getProductionCount
    ], function (err, result) {
        if (err) {
            res.send(err);
        }
    });
    function concessionCount(callback) {
        Concession.find({}).count().exec(function(err, concession_count) {
            if(concession_count) {
                callback(null, concession_count);
            } else {
                callback(err);
            }
        });
    }
    function getConcessionSet(concession_count, callback) {
        Concession.find(req.query)
            .sort({
                concession_name: 'asc'
            })
            .skip(skip)
            .limit(limit)
            .populate('concession_country.country', '_id iso2 name')
            .populate('concession_commodity.commodity', ' _id commodity_name commodity_type commodity_id')
            .populate('concession_established_source')
            .lean()
            .exec(function(err, concessions) {
                if(concessions) {
                    //TODO clean up returned data if we see performance lags
                    callback(null, concession_count, concessions);
                } else {
                    res.send({data: concessions, count: concession_count});
                }
            });
    }
    function getConcessionLinks(concession_count, concessions, callback) {
        concession_len = concessions.length;
        concession_counter = 0;
        if(concession_len>0) {
            concessions.forEach(function (concession) {
                concession.transfers_query = [concession._id];
                concession.source_type = {p: false, c: false};
                Link.find({concession: concession._id})
                    .populate('commodity', '_id commodity_name commodity_type commodity_id')
                    .populate('company site project')
                    .deepPopulate('source.source_type_id site.site_commodity.commodity project.proj_commodity.commodity')
                    .exec(function (err, links) {
                        ++concession_counter;
                        link_len = links.length;
                        link_counter = 0;
                        if(link_len>0) {
                            concession.project_count = 0;
                            concession.company_count = 0;
                            concession.site_count = 0;
                            concession.field_count  = 0;
                            concession.contract_count = 0;
                            links.forEach(function (link) {
                                ++link_counter;
                                var entity = _.without(link.entities, 'concession')[0];
                                if (!concession.source_type.p || !concession.source_type.c) {
                                    if (link.source.source_type_id.source_type_authority === 'authoritative') {
                                        concession.source_type.c = true;
                                    } else if (link.source.source_type_id.source_type_authority === 'non-authoritative') {
                                        concession.source_type.c = true;
                                    } else if (link.source.source_type_id.source_type_authority === 'disclosure') {
                                        concession.source_type.p = true;
                                    }
                                }
                                switch (entity) {
                                    case 'company':
                                        concession.company_count += 1;
                                        break;
                                    case 'contract':
                                        concession.contract_count += 1;
                                        break;
                                    case 'project':
                                        if (link.project.proj_commodity.length>0) {
                                            if (_.where(concession.concession_commodity, {_id:_.last(link.project.proj_commodity)._id}).length<1) {
                                                concession.concession_commodity.push({
                                                    _id: _.last(link.project.proj_commodity).commodity._id,
                                                    commodity_name: _.last(link.project.proj_commodity).commodity.commodity_name,
                                                    commodity_type: _.last(link.project.proj_commodity).commodity.commodity_type,
                                                    commodity_id: _.last(link.project.proj_commodity).commodity.commodity_id
                                                });
                                            }
                                        }
                                        if (!_.contains(concession.transfers_query, link.project)) {
                                            concession.transfers_query.push(link.project);
                                        }
                                        concession.project_count += 1;
                                        break;
                                    case 'site':
                                        if (link.site.site_commodity.length>0) {
                                            if (_.where(concession.concession_commodity, {_id:_.last(link.site.site_commodity)._id}).length<1) {
                                                concession.concession_commodity.push({
                                                    _id: _.last(link.site.site_commodity)._id,
                                                    commodity_name: _.last(link.site.site_commodity).commodity.commodity_name,
                                                    commodity_type: _.last(link.site.site_commodity).commodity.commodity_type,
                                                    commodity_id: _.last(link.site.site_commodity).commodity.commodity_id
                                                });
                                            }
                                        }
                                        if (!_.contains(concession.transfers_query, link.site._id)) {
                                            concession.transfers_query.push(link.site._id);
                                        }
                                        if (link.site.field) {
                                            concession.site_count += 1;
                                        } else if (!link.site.field) {
                                            concession.field_count += 1;
                                        }
                                        break;
                                    default:
                                        console.log(entity, 'skipped...');
                                }
                            });
                            if (concession_counter == concession_len && link_counter == link_len) {
                                callback(null, concession_count, concessions);
                            }
                        } else {
                            if (concession_counter == concession_len && link_counter == link_len) {
                                callback(null, concession_count, concessions);
                            }
                        }

                    });
            });
        } else{
            callback(null, concession_count, concessions);
{'fraction_non_alphanumeric': 0.08567274354099094, 'fraction_numerical': 0.003914550945084442, 'mean_word_length': 0.9187768240343348, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 199, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4207903', 'n_tokens_mistral': 4037, 'n_tokens_neox': 3622, 'n_words': 542}