<commit_before>"use strict"
const   mongoose = require('mongoose'),
    Article = require('../../models/Article'),
    Concept = require('../../models/Concept');


module.exports = (callback) => {


  let data = {}

  Concept.find()
    .where('size').gt(10)
    .sort('-size')
    .limit(40)
    .exec((err, collection)=>{
       if (err) throw err



      let bunchOfCallBacks = [];

      collection.forEach((concept)=>{

        data[concept.label] = {}
        data[concept.label]['articles'] = []

        concept.articles.forEach((articleId)=>{

          let newPromise = new Promise((resolve, reject) => {

            Article.findOne({_id: articleId})
            .exec((err, article)=>{
              if (err) {reject(); throw err}

              if (article) {
                  data[concept.label]._id = concept._id
                  data[concept.label].articles.push({
                  url: article.url,
                  title: article.title,
                  id: article._id
                })
              }
              resolve()
            })
          }) //end promise
          bunchOfCallBacks.push(newPromise)
        }) //end for each
      }) //end for each

    Promise.all(bunchOfCallBacks).then(()=>{
      callback(data)
    });
    }) // end find
} //end exports<commit_msg>add size property to dataset returned by popular api endpoint<commit_after>"use strict"
const   mongoose = require('mongoose'),
    Article = require('../../models/Article'),
    Concept = require('../../models/Concept');


module.exports = (callback) => {


  let data = {}

  Concept.find()
    .where('size').gt(10)
    .sort('-size')
    .limit(40)
    .exec((err, collection)=>{
       if (err) throw err



      let bunchOfCallBacks = [];

      collection.forEach((concept)=>{

        data[concept.label].size = concept.size
        data[concept.label]._id = concept._id
        data[concept.label] = {}
        data[concept.label]['articles'] = []

        concept.articles.forEach((articleId)=>{

          let newPromise = new Promise((resolve, reject) => {

            Article.findOne({_id: articleId})
            .exec((err, article)=>{
              if (err) {reject(); throw err}

              if (article) {
                  data[concept.label].articles.push({
                  url: article.url,
                  title: article.title,
                  id: article._id
                })
              }
              resolve()
            })
          }) //end promise
          bunchOfCallBacks.push(newPromise)
        }) //end for each
      }) //end for each

    Promise.all(bunchOfCallBacks).then(()=>{
      callback(data)
    });
    }) // end find
} //end exports
{'fraction_non_alphanumeric': 0.15668883961566887, 'fraction_numerical': 0.0029563932002956393, 'mean_word_length': 1.685515873015873, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '922454', 'n_tokens_mistral': 805, 'n_tokens_neox': 740, 'n_words': 150}