<commit_before>var express = require('express')
  , cheerio = require('cheerio')
  , request = require('superagent')
  , _ = require('lodash')
  , app = module.exports = express();

app.configure('development', function(){
    app.use(express.errorHandler({ dumpExceptions: true, showStack: true }));
});

app.configure('production', function(){
    app.use(express.errorHandler());
});


function parse(html, url) {
  var $ = cheerio.load(html)
    , data = {};

  $('meta[property^=og]').each(function(i, el) {
    data[el.attribs.property] = el.attribs.content;
  });

  data.title = data['og:title'] || $('title').text();
  data.image = data['og:image'] || $('img').eq(0).attr('src');
  if(data.image)
    data.image = require('url').resolve(url, data.image);
  return data;
}

function retrieve(url, cb) {
  request
    .get(url)
    .end(function(err, res) {
      if(err) return cb(err, null);
      cb(null, res);
    });
}

function normalize(url, protocol) {
  protocol = protocol || 'http';
  if(! /^https?\:\/\/.*/.test(url))
    url = protocol + '://' + url;
  return url;
}

app.get('/', function(req, res) {
  var url = normalize(req.param('url'));
  retrieve(url, function(err, response) {
    if(err) return res.send(err);

    var parts = response.type.split('/')
      , data = {title: url, url: url, content: parts[0]};

    switch(parts[0]) {
    case 'text':
    if(parts[1] === 'html')
      _.extend(data, parse(response.text, url));
    break;
    case 'image':
      data.image = url;
    break;
    case 'video':
      data.image = ''; // Default video icon?
    break;
    case 'audio':
      data.image = ''; // Default audio icon?
      break;
    }

    res.json(data);
  });
});

app.listen(process.env.PORT || 5000);<commit_msg>add port log<commit_after>var express = require('express')
  , cheerio = require('cheerio')
  , request = require('superagent')
  , _ = require('lodash')
  , app = module.exports = express();

app.configure('development', function(){
    app.use(express.errorHandler({ dumpExceptions: true, showStack: true }));
});

app.configure('production', function(){
    app.use(express.errorHandler());
});


function parse(html, url) {
  var $ = cheerio.load(html)
    , data = {};

  $('meta[property^=og]').each(function(i, el) {
    data[el.attribs.property] = el.attribs.content;
  });

  data.title = data['og:title'] || $('title').text();
  data.image = data['og:image'] || $('img').eq(0).attr('src');
  if(data.image)
    data.image = require('url').resolve(url, data.image);
  return data;
}

function retrieve(url, cb) {
  request
    .get(url)
    .end(function(err, res) {
      if(err) return cb(err, null);
      cb(null, res);
    });
}

function normalize(url, protocol) {
  protocol = protocol || 'http';
  if(! /^https?\:\/\/.*/.test(url))
    url = protocol + '://' + url;
  return url;
}

app.get('/', function(req, res) {
  var url = normalize(req.param('url'));
  retrieve(url, function(err, response) {
    if(err) return res.send(err);

    var parts = response.type.split('/')
      , data = {title: url, url: url, content: parts[0]};

    switch(parts[0]) {
    case 'text':
    if(parts[1] === 'html')
      _.extend(data, parse(response.text, url));
    break;
    case 'image':
      data.image = url;
    break;
    case 'video':
      data.image = ''; // Default video icon?
    break;
    case 'audio':
      data.image = ''; // Default audio icon?
      break;
    }

    res.json(data);
  });
});

var port = process.env.PORT || 5000;
app.listen(port);
console.log('listening on port', port);
{'fraction_non_alphanumeric': 0.20709893795416434, 'fraction_numerical': 0.004471771939631079, 'mean_word_length': 3.190866510538642, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3757867', 'n_tokens_mistral': 1255, 'n_tokens_neox': 1226, 'n_words': 252}