<commit_before>
var _ = require('lodash');

var urlConfig = require('./urlConfig-test.json');
var siteMgr = require('./site.manager.js');
var loadWebPage = require('./loadWebPage.js'); // Promise


_.forEach(urlConfig, (urls, siteID)=>{
  var queryFn = siteMgr.queryMap[siteID];
  if (queryFn == undefined) {
    console.log(key + ' not define query function');
    return;
  }
  _.forEach(urls, (url, categoryID)=>{
    console.log("----> start fetch:" + siteID + '  ' + categoryID +'  ' + url);
    loadWebPage(url)
    .then(html=>siteMgr.parseTitleAndURL(queryFn, html))
    .then(hotList=>{
      console.log('Get hot list of '+siteID+' '+categoryID+ ' length: ' + hotList.items.length);
      let cloneItem = Object.assign({}, hotList);
      cloneItem.siteID = siteID;
      cloneItem.categoryID = categoryID;
      //console.log(cloneItem);
    });
  });
});<commit_msg>use Promise.all to collect all result<commit_after>
var _ = require('lodash');

var urlConfig = require('./urlConfig-test.json');
var siteMgr = require('./site.manager.js');
var loadWebPage = require('./loadWebPage.js'); // Promise


function start() {
  var startTime = Date.now();

  _.forEach(urlConfig, (urls, siteID)=>{
    var queryFn = siteMgr.queryMap[siteID];
    if (queryFn == undefined) {
      console.log(key + ' not define query function');
      return;
    }

    // wrap url-fetch and html-parse for Promise.all
    var transformWrappers = _.map(urls, (url, categoryID)=>{

      console.log("----> start fetch:" + siteID + '  ' + categoryID +'  ' + url);

      return loadWebPage(url)
        .then(html=>siteMgr.parseTitleAndURL(queryFn, html))
        .then(hotList=>{

          console.log('Get hot list of '+siteID+' '+categoryID+ ' length: ' + hotList.items.length);

          let cloneItem = Object.assign({}, hotList);
          cloneItem.siteID = siteID;
          cloneItem.categoryID = categoryID;
          cloneItem.time = Date.now() - startTime;
          return hotList;
        });
    });

    Promise.all(transformWrappers).then((results)=>{
      console.log(JSON.stringify(results));
    });
  });
}

function ParseWebPageList(outputPath) {
  this.outputPath = outputPath;
}

module.exports = ParseWebPageList;
{'fraction_non_alphanumeric': 0.16599371915657246, 'fraction_numerical': 0.0, 'mean_word_length': 3.1449814126394053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6448032', 'n_tokens_mistral': 714, 'n_tokens_neox': 707, 'n_words': 147}