<commit_before>var request = require('request'),
    cheerio = require('cheerio'),
    async = require('async'),
    format = require('util').format;

var start = 'http://www.foodmaster.com/prodBrowse?code=296',
    concurrency = 2;

expand(start, 'a', function(element) {
  var link = element.attr('href');

  console.log(element.text() + '==>');

  if (link.indexOf('companies') != -1) {
    expand(link, 'ul.DIRECTORIES-CHILDPRODUCTS > li > a', function(element) {
      console.log('  ' + element.text());
    });
  }

  console.log('\n\n');

});

function expand(url, select, cb) {
  request(url, function (err, response, body) {
    if (err) {
      console.err('.');
      return
    }

    var $ = cheerio.load(body);

    async.eachLimit($(select), concurrency, function (element, next) {
      cb($(element));
      next();
    });
  });
}<commit_msg>finishes when all requests come back<commit_after>var request = require('request'),
    cheerio = require('cheerio'),
    async = require('async'),
    format = require('util').format;

var start = 'http://www.foodmaster.com/prodBrowse?code=296',
    concurrency = 2,
    counter = 0,
    map = {};

function done() {
  console.log(map);
}

function expand(url, select, cb) {
  counter++;

  request(url, function (err, response, body) {
    if (err) {
      console.err('X');
      return
    }

    var $ = cheerio.load(body);

    async.eachLimit($(select), concurrency, function (element, next) {
      cb($(element));
      next();
    });

    counter--;

    /* finished spidering */
    if (counter === 0) {
      done();
    }
  });
}

process.stdout.write('Spidering');

expand(start, 'a', function(element) {
  process.stdout.write('.');
  var link = element.attr('href');
  map[element.text()] = [];

  if (link.indexOf('companies') != -1) {
    expand(link, 'ul.DIRECTORIES-CHILDPRODUCTS > li > a', function(el) {
      process.stdout.write('.');
      map[element.text()].push(el.text());
    });
  }
});
{'fraction_non_alphanumeric': 0.19909044972208187, 'fraction_numerical': 0.00606366851945427, 'mean_word_length': 3.082474226804124, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '687601', 'n_tokens_mistral': 682, 'n_tokens_neox': 665, 'n_words': 128}