<commit_before>var Promise = require('mpromise');
var cheerio = require('cheerio');
var request = require('request');

module.exports = function () {
  var promise = new Promise;

  request('http://hyperpolyglot.org/version-control', function (error, response, body) {
    if (!error && response.statusCode === 200) {
      var features = [];

      var $ = cheerio.load(body);
      var technologies = [];

      $('.wiki-content-table').first().find('tr').each(function (index) {
        if (index === 1) {
          $(this).find('th').each(function (index) {
            if (index > 0) {
              var technology = $(this).text().replace(/\s*\(\d{4}\)/, '');
              technologies.push(technology);
            }
          });
        } else if (index > 1) {
          var feature = { examples: [] };
          $(this).find('td').each(function (index) {
            if (index === 0) {
              feature.name = $(this).text();
            } else if ($(this).text()) {
              feature.examples.push({
                technology: technologies[index - 1],
                snippets: $(this).text()
              });
            }
          });
          features.push(feature);
        }
      });

      promise.fulfill(features);
    }
  });<commit_msg>Add comments to the scraper<commit_after>var Promise = require('mpromise');
var cheerio = require('cheerio');
var request = require('request');

module.exports = function () {
  var promise = new Promise;

  request('http://hyperpolyglot.org/version-control', function (error, response, body) {
    if (!error) {
      var features = [];
      var $ = cheerio.load(body);
      var technologies = [];

      // Find the first table.
      $('.wiki-content-table').first().find('tr').each(function (index) {
        if (index === 1) {
          $(this).find('th').each(function (index) {
            if (index > 0) {
              // Get the names of the technologies, removing the years.
              var technology = $(this).text().replace(/\s*\(\d{4}\)/, '');
              technologies.push(technology);
            }
          });
        } else if (index > 1) {
          // Create a feature.
          var feature = { examples: [] };
          $(this).find('td').each(function (index) {
            // Get feature data.
            if (index === 0) {
              // Get the name of the feature.
              feature.name = $(this).text();
            } else if ($(this).text()) {
              // Collect the cell data as one example.
              feature.examples.push({
                technology: technologies[index - 1],
                snippets: $(this).text()
              });
            }
          });
          features.push(feature);
        }
      });

      // Fulfill the promise.
      promise.fulfill(features);
    }
  });
{'fraction_non_alphanumeric': 0.1706798866855524, 'fraction_numerical': 0.005311614730878187, 'mean_word_length': 1.7059386973180077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6360219', 'n_tokens_mistral': 787, 'n_tokens_neox': 760, 'n_words': 175}