<commit_before>var _ = require('underscore');
var http = require('http');
var now = require('now');
var nodestatic = require('node-static');
var uri = require('uri-js');

var config = require('./config');
var utils = require('./utils');


// Web server
files = new (nodestatic.Server)('./static');

httpServer = http.createServer(function(request, response) {
  request.addListener('end', function() {
    files.serve(request, response);
  });
}).listen(config.web.port);


// Now.js connection
var everyone = now.initialize(httpServer);
screenIds = [];
var currentScreen = 0;

everyone.disconnected(function() {
  console.log('screen disconnected ' + JSON.stringify(this));
  var index = screenIds.indexOf(this.user.clientId);
  if (index >= 0) {
    screenIds.splice(index, 1);
  }
  // Make sure that `currentScreen` is still valid.
  currentScreen = (currentScreen + 1) % screenIds.length;
});

everyone.connected(function() {
  console.log('screen connected');
  screenIds.push(this.user.clientId);
  // Make the next url go to this screen.
  currentScreen = screenIds.length - 1;
});

// Pick a screen to show a URL on, and kick of the process.
exports.setUrl = function setUrl(url, callback) {
  var id = screenIds[currentScreen];
  if (id === undefined) {
    utils.async(callback, "No screens connected.");
    return;
  }
  currentScreen = (currentScreen + 1) % screenIds.length;

  now.getClient(id, function() {
    var screen = this;
    _processUrl(url, function(opts) {
      if (opts.message) {
        utils.async(callback, opts.message);
      }
      if (opts.url) {
        if (opts.type === 'image') {
          screen.now.setImage(opts.url);
        } else {
          screen.now.setUrl(opts.url);
        }
      }
    });
  });
};

function _processUrl(url, callback) {
  var components = uri.parse(url);
  if (components.host === undefined && components.protocol === undefined) {
    components = uri.parse('http://' + url);
  }

  if (components.errors.length) {
    utils.async(callback, {
      message: "I couldn't parse a url from that."
    });
    return;
  }

  if (components.host.indexOf('noodletalk.org') >= 0) {
    utils.async(callback, {
      message:'NOPE',
      type: 'image',
      url: '/img/nope.gif'
    });
    return;
  }

  var path = components.path;
  if (components.query) {
    path += '?' + components.query;
  }
  if (components.fragment) {
    path += '#' + components.fragment;
  }

  var options = {
    method: 'HEAD',
    host: components.host,
    port: components.port || 80,
    path: path
  };

  var req = http.request(options, function(res) {
    var headers = {};
    _.each(res.headers, function(value, key) {
      headers[key.toLowerCase()] = value;
    });
    console.log(JSON.stringify(headers));

    if (res.statusCode === 301) {
      // redirect, handle it.
      console.log('redirect ' + headers.location);
      return _processUrl(headers.location, callback);
    }

    if (res.statusCode === 404) {
      utils.async(callback, {
        message: 'There was a problem with the url (' + res.statusCode + ')'
      });
      return;
    }

    if (headers['content-type'].indexOf('image/') === 0) {<commit_msg>https support<commit_after>var _ = require('underscore');
var http = require('http');
var https = require('https');
var now = require('now');
var nodestatic = require('node-static');
var uri = require('uri-js');

var config = require('./config');
var utils = require('./utils');


// Web server
files = new (nodestatic.Server)('./static');

httpServer = http.createServer(function(request, response) {
  request.addListener('end', function() {
    files.serve(request, response);
  });
}).listen(config.web.port);


// Now.js connection
var everyone = now.initialize(httpServer);
screenIds = [];
var currentScreen = 0;

everyone.disconnected(function() {
  console.log('screen disconnected ' + JSON.stringify(this));
  var index = screenIds.indexOf(this.user.clientId);
  if (index >= 0) {
    screenIds.splice(index, 1);
  }
  // Make sure that `currentScreen` is still valid.
  currentScreen = (currentScreen + 1) % screenIds.length;
});

everyone.connected(function() {
  console.log('screen connected');
  screenIds.push(this.user.clientId);
  // Make the next url go to this screen.
  currentScreen = screenIds.length - 1;
});

// Pick a screen to show a URL on, and kick off the process.
exports.setUrl = function setUrl(url, callback) {
  var id = screenIds[currentScreen];
  if (id === undefined) {
    utils.async(callback, "No screens connected.");
    return;
  }
  currentScreen = (currentScreen + 1) % screenIds.length;

  now.getClient(id, function() {
    var screen = this;
    _processUrl(url, function(opts) {
      if (opts.message) {
        utils.async(callback, opts.message);
      }
      if (opts.url) {
        if (opts.type === 'image') {
          screen.now.setImage(opts.url);
        } else {
          screen.now.setUrl(opts.url);
        }
      }
    });
  });
};

function _processUrl(url, callback) {
  var components = uri.parse(url);
  if (components.errors.length) {
    utils.async(callback, {
      message: "I couldn't parse a url from that."
    });
    return;
  }

  if (components.host.indexOf('noodletalk.org') >= 0) {
    utils.async(callback, {
      message:'NOPE',
      type: 'image',
      url: '/img/nope.gif'
    });
    return;
  }

  var path = components.path;
  if (components.query) {
    path += '?' + components.query;
  }
  if (components.fragment) {
    path += '#' + components.fragment;
  }

  var options = {
    method: 'HEAD',
    host: components.host,
    port: components.port || 80,
    path: path
  };

  var proto = http;
  if (components.scheme === 'https') {
    proto = https;
  }

  var req = proto.request(options, function(res) {
    var headers = {};
    _.each(res.headers, function(value, key) {
      headers[key.toLowerCase()] = value;
    });
    console.log(JSON.stringify(headers));

    if (res.statusCode === 301) {
      // redirect, handle it.
      console.log('redirect ' + headers.location);
      return _processUrl(headers.location, callback);
    }

    if (res.statusCode === 404) {
      utils.async(callback, {
        message: 'There was a problem with the url (' + res.statusCode + ')'
      });
      return;
    }

    if (headers['content-type'].indexOf('image/') === 0) {
{'fraction_non_alphanumeric': 0.16250588789448894, 'fraction_numerical': 0.005024336630554247, 'mean_word_length': 2.9713216957605986, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5634996', 'n_tokens_mistral': 2042, 'n_tokens_neox': 1989, 'n_words': 462}