<commit_before>/*
 * * Module dependencies
 * */

var request = require('request');
var cheerio = require('cheerio');
var express = require('express')

    , nib = require('nib')


var app = express() // sets up the server


// Start Ycomb stuff

function YComb(callback){
  request('https://news.ycombinator.com', function(error, response, html){
  		if(!error && response.statusCode === 200){
        var metadataArray = [ ];
  			var $ = cheerio.load(html);
  			$('span.comhead').each(function(i, element){
  			var a=$(this).prev(); //selects previous data
  			var rank=a.parent().parent().text(); //gets ranks by parsing text two elements higher
  			var title=a.text(); // parses link title
  			var url=a.attr('href'); // parses href attribute from "a" element
  			var subtext = a.parent().parent().next().children('.subtext').children(); // gets the subtext from the children
  			var points = $(subtext).eq(0).text();
  			var username = $(subtext).eq(1).text();
        var YCOMB_COMMENT_URL = "https://news.ycombinator.com/"
  			var comments = $(subtext).eq(2).text();
  			var comments_link = YCOMB_COMMENT_URL + $(subtext).eq(2).attr('href');

  			var metadata = { // creates a new object
  				rank: parseInt(rank),
          site: "HN",
  				title:title,
  				url:url,
  				points: parseInt(points),
  				username: username,
  				comments: parseInt(comments),
                                comments_link:comments_link
  			};
  			metadataArray.push(metadata); // pushes the object
  			});
        callback(metadataArray);
  		}
  });
}

app.get('/ycomb', function(req,res) {
  YComb(function(data){
    res.send(data)
  });
})


// end ycomb

metadataArray = [ ];

// start Lobster stuff

function Lobster(callback){
  request('https://lobste.rs', function(error, response, html){
      if(!error && response.statusCode === 200){
        var metadataArray = [ ];

        var $ = cheerio.load(html);
        $('span.link').each(function(i, element){
        var a=$(this); //selects previous data
        var url=a.children().attr('href'); // parses href attribute from "a" element
        var title=a.text(); // parses link title

        // Note: There currently (2014-07-12) is no comments tag to be parsed
        //       available by scraping the Loste.rs webpage.
        //       This means 0 comments will have no link to the comment page.
        var comments_label = a.parent().children('.byline').children('span.comments_label');
        var comments_link = comments_label.children('a').attr("href");
        var commentsMatch = comments_label.text().match("[0-9]+");
        var comments = commentsMatch !== null ? commentsMatch[0] : 0;

        var metadata = { // creates a new object
          site: "lobste.rs",
          title:title,
          url:url,
          comments:comments,
          comments_link:comments_link
        };
        metadataArray.push(metadata); // pushes the object
        });
        callback(metadataArray);
      }
  });
}

app.get('/lobster', function(req,res) {
  Lobster(function(data){
    res.send(data)
  });
})

// end Lobster stuff

metadataArray = [ ];
<commit_msg>Added scraped votes from Lobste.rs<commit_after>/*
 * * Module dependencies
 * */

var request = require('request');
var cheerio = require('cheerio');
var express = require('express')

    , nib = require('nib')


var app = express() // sets up the server


// Start Ycomb stuff

function YComb(callback){
  request('https://news.ycombinator.com', function(error, response, html){
  		if(!error && response.statusCode === 200){
        var metadataArray = [ ];
  			var $ = cheerio.load(html);
  			$('span.comhead').each(function(i, element){
  			var a=$(this).prev(); //selects previous data
  			var rank=a.parent().parent().text(); //gets ranks by parsing text two elements higher
  			var title=a.text(); // parses link title
  			var url=a.attr('href'); // parses href attribute from "a" element
  			var subtext = a.parent().parent().next().children('.subtext').children(); // gets the subtext from the children
  			var points = $(subtext).eq(0).text();
  			var username = $(subtext).eq(1).text();
        var YCOMB_COMMENT_URL = "https://news.ycombinator.com/"
  			var comments = $(subtext).eq(2).text();
  			var comments_link = YCOMB_COMMENT_URL + $(subtext).eq(2).attr('href');

  			var metadata = { // creates a new object
  				rank: parseInt(rank),
          site: "HN",
  				title:title,
  				url:url,
  				points: parseInt(points),
  				username: username,
  				comments: parseInt(comments),
                                comments_link:comments_link
  			};
  			metadataArray.push(metadata); // pushes the object
  			});
        callback(metadataArray);
  		}
  });
}

app.get('/ycomb', function(req,res) {
  YComb(function(data){
    res.send(data)
  });
})


// end ycomb

metadataArray = [ ];

// start Lobster stuff

function Lobster(callback){
  request('https://lobste.rs', function(error, response, html){
      if(!error && response.statusCode === 200){
        var metadataArray = [ ];

        var $ = cheerio.load(html);
        $('span.link').each(function(i, element){
        var a=$(this); //selects previous data
        var url=a.children().attr('href'); // parses href attribute from "a" element
        var title=a.text(); // parses link title

        // Note: There currently (2014-07-12) is no comments tag to be parsed
        //       available by scraping the Loste.rs webpage.
        //       This means 0 comments will have no link to the comment page.
        var comments_label = a.parent().children('.byline').children('span.comments_label');
        var comments_link = comments_label.children('a').attr("href");
        var commentsMatch = comments_label.text().match("[0-9]+");
        var comments = commentsMatch !== null ? commentsMatch[0] : 0;

        var metadata = { // creates a new object
          site: "lobste.rs",
          title:title,
          url:url,
          comments:comments,
          comments_link:comments_link
        };
        metadataArray.push(metadata); // pushes the object
        });

        // Iterate through every link's score
        $('div.score').each(function (i, element) {
          // push the score value to it's respective link object
          metadataArray[i].votes = $(this).text();
        });
        
        callback(metadataArray);
      }
  });
}

app.get('/lobster', function(req,res) {
  Lobster(function(data){
    res.send(data)
  });
})

// end Lobster stuff

metadataArray = [ ];

{'fraction_non_alphanumeric': 0.15423340961098397, 'fraction_numerical': 0.007017543859649123, 'mean_word_length': 2.5747001090512542, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '686905', 'n_tokens_mistral': 2103, 'n_tokens_neox': 1991, 'n_words': 479}