<commit_before>var http = require('http');
var https = require('https');
var cheerio = require('cheerio');
var url = require('url');

module.exports = {
  fetchPage : function (http, URL, pageCallback) {
    http.get(URL, function(r) {
      var body = '';

      r.on('data', function (chunk) {
        body += chunk;
      });

      r.on('end', function () {
        pageCallback(body);
      });
    }).on('error', function(e) {
      console.error(URL, 'connection error:', e);
    });
  },

  getAlexa : function (domain, statCallback) {
    this.fetchPage(http, 'http://www.alexa.com/siteinfo/' + encodeURIComponent(domain), function(body) {
      var $ = cheerio.load(body);
      statCallback({
        'source': 'alexa.com',
        'domain': domain,
        'rank':
        {
          'global': parseInt($('[data-cat=globalRank] .metrics-data').text().replace(/,/g,'')),
          'country': parseInt($('[data-cat=countryRank] .metrics-data').text())
        },
        'bounceRate': parseFloat($('[data-cat=bounce_percent] .metrics-data').text())/100,
        'dailyPageviewsPerVisitor': parseFloat($('[data-cat=pageviews_per_visitor] .metrics-data').text()),
        'dailyTimeOnSite': $('[data-cat=time_on_site] .metrics-data').text(),
        'searchVisits': parseFloat($('[data-cat=search_percent] .metrics-data').text())/100
      });
    });
  },

  getGooglePlus : function (URL, statCallback) {
    this.fetchPage(https, 'https://plusone.google.com/_/+1/fastbutton?url=' + encodeURIComponent(URL), function(body) {
      var $ = cheerio.load(body);
      var aggregate = $('#aggregateCount').text();

      if (aggregate.match(/B$/)) // kinda doubt they actually do that one, but meh.
      {
         aggregate = parseFloat(aggregate) * 1e9;
      }
      else if (aggregate.match(/M$/))
      {
         aggregate = parseFloat(aggregate) * 1e6;
      }
      else if (aggregate.match(/B$/))
      {
         aggregate = parseFloat(aggregate) * 1e3;
      }
      else
      {
         aggregate = parseFloat(aggregate);
      }

      statCallback({
        'source': 'google.com/+',
        'url': URL,
        'aggregateCount': aggregate
      });
    });
  },

  getTwitter : function (URL, statCallback) {
    this.fetchPage(http, 'http://urls.api.twitter.com/1/urls/count.json?url=' + encodeURIComponent(URL), function(body) {
      try {
        var tweets = JSON.parse(body);
        statCallback({
          'source': 'twitter.com',
          'url': URL,
          'tweets': tweets.count ? tweets.count : 0,
          'raw': tweets
        });
      } catch (e) {
        console.error(e);
      }
    });
  },

  getFacebook : function (URL, statCallback) {
    this.fetchPage(http, 'http://graph.facebook.com/?id=' + encodeURIComponent(URL), function(body) {
      try {
        var facebookInteractions = JSON.parse(body);
        statCallback({
          'source': 'facebook.com',
          'url': URL,
          'shares': facebookInteractions.shares ? facebookInteractions.shares : 0,
          'comments': facebookInteractions.comments ? facebookInteractions.comments : 0,
          'raw': facebookInteractions
        });
      } catch (e) {
        console.error(e);
      }
    });
  },

  getLinkedin : function (URL, statCallback) {
    this.fetchPage(http, 'http://www.linkedin.com/countserv/count/share?format=json&url=' + encodeURIComponent(URL), function(body) {
      try {
        var interactions = JSON.parse(body);
        statCallback({
          'source': 'linkedin.com',
          'url': URL,
          'count': interactions.count ? interactions.count : 0,
          'raw': interactions
        });
      } catch (e) {
        console.error(e);
      }<commit_msg>fix linkedin stats errors - yay, they're now using https :)<commit_after>var http = require('http');
var https = require('https');
var cheerio = require('cheerio');
var url = require('url');

module.exports = {
  fetchPage : function (http, URL, pageCallback) {
    http.get(URL, function(r) {
      var body = '';

      r.on('data', function (chunk) {
        body += chunk;
      });

      r.on('end', function () {
        pageCallback(body);
      });
    }).on('error', function(e) {
      console.error(URL, 'connection error:', e);
    });
  },

  getAlexa : function (domain, statCallback) {
    this.fetchPage(http, 'http://www.alexa.com/siteinfo/' + encodeURIComponent(domain), function(body) {
      var $ = cheerio.load(body);
      statCallback({
        'source': 'alexa.com',
        'domain': domain,
        'rank':
        {
          'global': parseInt($('[data-cat=globalRank] .metrics-data').text().replace(/,/g,'')),
          'country': parseInt($('[data-cat=countryRank] .metrics-data').text())
        },
        'bounceRate': parseFloat($('[data-cat=bounce_percent] .metrics-data').text())/100,
        'dailyPageviewsPerVisitor': parseFloat($('[data-cat=pageviews_per_visitor] .metrics-data').text()),
        'dailyTimeOnSite': $('[data-cat=time_on_site] .metrics-data').text(),
        'searchVisits': parseFloat($('[data-cat=search_percent] .metrics-data').text())/100
      });
    });
  },

  getGooglePlus : function (URL, statCallback) {
    this.fetchPage(https, 'https://plusone.google.com/_/+1/fastbutton?url=' + encodeURIComponent(URL), function(body) {
      var $ = cheerio.load(body);
      var aggregate = $('#aggregateCount').text();

      if (aggregate.match(/B$/)) // kinda doubt they actually do that one, but meh.
      {
         aggregate = parseFloat(aggregate) * 1e9;
      }
      else if (aggregate.match(/M$/))
      {
         aggregate = parseFloat(aggregate) * 1e6;
      }
      else if (aggregate.match(/B$/))
      {
         aggregate = parseFloat(aggregate) * 1e3;
      }
      else
      {
         aggregate = parseFloat(aggregate);
      }

      statCallback({
        'source': 'google.com/+',
        'url': URL,
        'aggregateCount': aggregate
      });
    });
  },

  getTwitter : function (URL, statCallback) {
    this.fetchPage(http, 'http://urls.api.twitter.com/1/urls/count.json?url=' + encodeURIComponent(URL), function(body) {
      try {
        var tweets = JSON.parse(body);
        statCallback({
          'source': 'twitter.com',
          'url': URL,
          'tweets': tweets.count ? tweets.count : 0,
          'raw': tweets
        });
      } catch (e) {
        console.error(e);
      }
    });
  },

  getFacebook : function (URL, statCallback) {
    this.fetchPage(http, 'http://graph.facebook.com/?id=' + encodeURIComponent(URL), function(body) {
      try {
        var facebookInteractions = JSON.parse(body);
        statCallback({
          'source': 'facebook.com',
          'url': URL,
          'shares': facebookInteractions.shares ? facebookInteractions.shares : 0,
          'comments': facebookInteractions.comments ? facebookInteractions.comments : 0,
          'raw': facebookInteractions
        });
      } catch (e) {
        console.error(e);
      }
    });
  },

  getLinkedin : function (URL, statCallback) {
    this.fetchPage(https, 'https://www.linkedin.com/countserv/count/share?format=json&url=' + encodeURIComponent(URL), function(body) {
      try {
        var interactions = JSON.parse(body);
        statCallback({
          'source': 'linkedin.com',
          'url': URL,
          'count': interactions.count ? interactions.count : 0,
          'raw': interactions
        });
      } catch (e) {
        console.error(e);
      }
{'fraction_non_alphanumeric': 0.17036344200962053, 'fraction_numerical': 0.004810261892036344, 'mean_word_length': 2.427197802197802, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6387870', 'n_tokens_mistral': 2378, 'n_tokens_neox': 2307, 'n_words': 411}