<commit_before>var express = require('express');
var phantom = require('node-phantom');
var app = express();

app.use(express.static(__dirname + '/public'));

app.get('/url', function (req, res) {
  var url = req.query.q;
  
  phantom.create(function (err, ph) {
    if (err) return error(res, err, ph);

    return ph.createPage(function (err, page) {
      if (err) return error(res, err, ph);
      
      return page.open(url, function (err, status) {
        page.includeJs('http://localhost:3000/jquery-2.0.3.min.js', function (err) {
          if (err) return error(res, err, ph);
          
          return page.evaluate(function () {
            var hrefs = [];

            $('a').each(function (i, a) {
              var href = a.href;
              var hash = href.indexOf('#');
              if (hash !== -1) href = href.substring(0, hash);

              if ($.inArray(href, hrefs) === -1) {
                hrefs.push(href);
              }
            });<commit_msg>only grab links that start with http<commit_after>var express = require('express');
var phantom = require('node-phantom');
var app = express();

app.use(express.static(__dirname + '/public'));

app.get('/url', function (req, res) {
  var url = req.query.q;
  
  phantom.create(function (err, ph) {
    if (err) return error(res, err, ph);

    return ph.createPage(function (err, page) {
      if (err) return error(res, err, ph);
      
      return page.open(url, function (err, status) {
        page.includeJs('http://localhost:3000/jquery-2.0.3.min.js', function (err) {
          if (err) return error(res, err, ph);
          
          return page.evaluate(function () {
            var hrefs = [];

            $('a').each(function (i, a) {
              var href = a.href;
              var hash = href.indexOf('#');
              if (hash !== -1) href = href.substring(0, hash);

              if ($.inArray(href, hrefs) === -1 && href.startsWith('http')) {
                hrefs.push(href);
              }
            });
{'fraction_non_alphanumeric': 0.16798810703666997, 'fraction_numerical': 0.009910802775024777, 'mean_word_length': 2.0498489425981874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '439047', 'n_tokens_mistral': 640, 'n_tokens_neox': 614, 'n_words': 157}