<commit_before>var PORT = require('system').env.PORT || 8080;

var server = require('webserver').create();
server.listen(PORT, function(request, response) {
  var page = new WebPage();
  page.open('http://benjaminbenben.com', function (status) {

    var links = page.evaluate(function () {
      return [].reduce.call(document.querySelectorAll('a'), function(memo, a){
        var bb = a.getBoundingClientRect(),
          area = bb.width * bb.height,
          href = a.href

        if(area)
          memo[href] = (memo[href] || 0) + area;
       
        return memo;
      },{});
    });

    response.setHeader('Content-Type', 'application/json; charset=utf-8');
    response.setHeader('Access-Control-Allow-Origin','*');
    response.statusCode = 200;
    response.write(JSON.stringify(links));
    response.close();
    page.close();
  });
});<commit_msg>let example look at any site in THE ENTIRE WORLD<commit_after>var PORT = require('system').env.PORT || 8080;

var server = require('webserver').create();
server.listen(PORT, function(request, response) {

  var url = request.url;
  var idx = url.indexOf('?url=');
  if(idx != -1){
    var stripped = unescape(url.substr(idx + 5));
    console.log(stripped);

    var page = new WebPage();
    page.open(stripped, function (status) {

      var links = page.evaluate(function () {
        return [].reduce.call(document.querySelectorAll('a'), function(memo, a){
          var bb = a.getBoundingClientRect(),
            area = bb.width * bb.height,
            href = a.href;

          if(area)
            memo[href] = (memo[href] || 0) + area;
         
          return memo;
        },{});
      });

      response.setHeader('Content-Type', 'application/json; charset=utf-8');
      response.setHeader('Access-Control-Allow-Origin','*');
      response.statusCode = 200;
      response.write(JSON.stringify(links));
      response.close();
      page.close();
    });

  } else {
    response.statusCode = 200;
    response.write('nope');
    response.close();
  }

});
{'fraction_non_alphanumeric': 0.1692836113837095, 'fraction_numerical': 0.011285574092247301, 'mean_word_length': 2.761992619926199, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '317348', 'n_tokens_mistral': 658, 'n_tokens_neox': 620, 'n_words': 123}