<commit_before>var request = require('request');
var cheerio = require('cheerio');

exports.htop = function(req,res){
  request('https://news.ycombinator.com', function(error, response, html){
      if(!error && response.statusCode === 200){
        var metadataArray = [ ];
        var $ = cheerio.load(html);
        $('span.comhead').each(function(){
        var a=$(this).prev(); //selects previous data
        var rank=a.parent().parent().text(); //gets ranks by parsing text two elements higher
        var title=a.text(); // parses link title
        var url=a.attr('href'); // parses href attribute from "a" element
        var subtext = a.parent().parent().next().children('.subtext').children(); // gets the subtext from the children
        var points = $(subtext).eq(0).text();
        var username = $(subtext).eq(1).text();
        var YCOMB_COMMENT_URL = "https://news.ycombinator.com/";
        var comments = $(subtext).eq(2).text();
        var comments_link = YCOMB_COMMENT_URL + $(subtext).eq(2).attr('href');

        var metadata = { // creates a new object
          rank: parseInt(rank),
          site: "HN",
          title:title,
          url:url,
          points: parseInt(points),
          username: username,
          comments: parseInt(comments),
                                comments_link:comments_link
        };
        metadataArray.push(metadata); // pushes the object
        });
        res.send(metadataArray);
      }
  });

};

exports.hnew = function(req,res){
  request('https://news.ycombinator.com/newest', function(error, response, html){
      if(!error && response.statusCode === 200){
        var metadataArray = [ ];
        var $ = cheerio.load(html);
        $('span.comhead').each(function(){
        var a=$(this).prev(); //selects previous data
        var rank=a.parent().parent().text(); //gets ranks by parsing text two elements higher
        var title=a.text(); // parses link title
        var url=a.attr('href'); // parses href attribute from "a" element
        var subtext = a.parent().parent().next().children('.subtext').children(); // gets the subtext from the children
        var points = $(subtext).eq(0).text();
        var username = $(subtext).eq(1).text();

        var YCOMB_COMMENT_URL = "https://news.ycombinator.com/";
        var comments = $(subtext).eq(2).text();
        var comments_link = YCOMB_COMMENT_URL + $(subtext).eq(2).attr('href');

        var metadata = { // creates a new object
          rank: parseInt(rank),
          site: "HN",
          title:title,
          url:url,
          points: parseInt(points),
          username: username,
          comments: parseInt(comments),
          comments_link:comments_link
        };
        metadataArray.push(metadata); // pushes the object
        });
        res.send(metadataArray);
      }
  });

};
<commit_msg>Cleans up routes/hackernews.js<commit_after>var request = require('request');
var cheerio = require('cheerio');

function parse(html){
  var metadataArray = [ ];
  var $ = cheerio.load(html);
  $('span.comhead').each(function(){
    var a=$(this).prev(); //selects previous data
    var rank=a.parent().parent().text(); //gets ranks by parsing text two elements higher
    var title=a.text(); // parses link title
    var url=a.attr('href'); // parses href attribute from "a" element
    var subtext = a.parent().parent().next().children('.subtext').children(); // gets the subtext from the children
    var points = $(subtext).eq(0).text();
    var username = $(subtext).eq(1).text();
    var YCOMB_COMMENT_URL = "https://news.ycombinator.com/";
    var comments = $(subtext).eq(2).text();
    var comments_link = YCOMB_COMMENT_URL + $(subtext).eq(2).attr('href');

    var metadata = { // creates a new object
      rank: parseInt(rank),
      site: "HN",
      title:title,
      url:url,
      points: parseInt(points),
      username: username,
      comments: parseInt(comments),
      comments_link:comments_link
    };
    metadataArray.push(metadata); // pushes the object
  });
  return(metadataArray);
  // res.send(metadataArray);
}

exports.htop = function(req,res){
  request('https://news.ycombinator.com', function(error, response, html){
      if(!error && response.statusCode === 200){
        res.send(parse(html));
      }
  });

};

exports.hnew = function(req,res){
  request('https://news.ycombinator.com/newest', function(error, response, html){
      if(!error && response.statusCode === 200){
        res.send(parse(html));
      }
  });

};

{'fraction_non_alphanumeric': 0.1649393605292172, 'fraction_numerical': 0.005292171995589856, 'mean_word_length': 2.6758508914100485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6233523', 'n_tokens_mistral': 1350, 'n_tokens_neox': 1325, 'n_words': 301}