<commit_before>var util = require('util');
var request = require('request');
var pg = require('pg');

pg.defaults.ssl = true;

function withDb(cb) {
  var dbString = process.env['DATABASE_URL'];
  if(dbString == null) {
    throw new Error('You must set a DATABASE_URL enviroment variable containing the database connection string');
  }
  pg.connect(process.env.DATABASE_URL, function(err, client) {
    if(err) {
      console.log(err);
      throw err;
    }
    cb(client);
  });
}

function sanatizeNames(names) {
  return names.map(function(n) {
    if(n.contains(' ')) {
      throw new Error('Usernames cannot contain a space (' + n + ')');
    }
    return n.toLowerCase().Trim();
  });
}

function getNextStatuses(names, cb) {
  var safeNames = sanatizeNames(names);
  var apiUrl = 'https://api.twitch.tv/kraken/streams?channel=';
  var nameCsv = names.join();
  var requestUrl = apiUrl + nameCsv;
  request(requestUrl, function(error, response, body) {
    if(error || response.statusCode !== 200) {
      console.log('Request error (' + response.statusCode + '): ' + error);
    }
    cb(body);
  });
}

function getCurrentStatuses(names, cb) {
  withDb(function(db) {
    var nameCsv = names.join();
    db.query('SELECT ss.* FROM streamer_statuses ss INNER JOIN streamer s ON s.id = ss.streamer_id WHERE s.name IN (' + nameCsv + ') HAVING MAX(ss.timestamp) GROUP BY s.id')
      .on('row', function(row, result) { result.addRow(row); })
      .on('end', function(result) { cb(result.rows); });
  });
}

function getDifferingStatuses(statuses, nextStatuses) {

}

function saveStatuses(statuses) {

}

function getNames(cb) {
  withDb(function(db) {
    db.query('select name FROM streamer')
      .on('row', function(row, result) { result.addRow(row); })
      .on('end', function(result) { cb(result.rows); });
  });
}

var interval = process.env['TWITCH_STATUS_RECORDER_INTERVAL'] === null ? 60000 : process.env.TWITCH_STATUS_RECORDER_INTERVAL;
setInterval(function() {
  getNames(function(names) {
    if(names.length > 0) {
      getCurrentStatuses(names, function(currentStatuses) {
        getNextStatuses(names, function(nextStatuses) {
          var differentStatuses = getDifferingStatuses(currentStatuses, nextStatuses);
          saveStatuses(differentStatuses);
        });
      });
    }
  });
}, interval);
<commit_msg>Adding some default stuff in while I get things working<commit_after>var util = require('util');
var request = require('request');
var pg = require('pg');

pg.defaults.ssl = true;

function withDb(cb) {
  var dbString = process.env['DATABASE_URL'];
  if(dbString == null) {
    throw new Error('You must set a DATABASE_URL enviroment variable containing the database connection string');
  }
  pg.connect(process.env.DATABASE_URL, function(err, client) {
    if(err) {
      console.log(err);
      throw err;
    }
    cb(client);
  });
}

function sanatizeNames(names) {
  return names.map(function(n) {
    if(n.contains(' ')) {
      throw new Error('Usernames cannot contain a space (' + n + ')');
    }
    return n.toLowerCase().Trim();
  });
}

function getNextStatuses(names, cb) {
  var safeNames = sanatizeNames(names);
  var apiUrl = 'https://api.twitch.tv/kraken/streams?channel=';
  var nameCsv = names.join();
  var requestUrl = apiUrl + nameCsv;
  request(requestUrl, function(error, response, body) {
    if(error || response.statusCode !== 200) {
      console.log('Request error (' + response.statusCode + '): ' + error);
    }
    cb(body);
  });
}

function getCurrentStatuses(names, cb) {
  withDb(function(db) {
    var nameCsv = names.join();
    db.query('SELECT ss.* FROM streamer_statuses ss INNER JOIN streamer s ON s.id = ss.streamer_id WHERE s.name IN (' + nameCsv + ') HAVING MAX(ss.timestamp) GROUP BY s.id')
      .on('row', function(row, result) { result.addRow(row); })
      .on('end', function(result) { cb(result.rows); });
  });
}

function getDifferingStatuses(statuses, nextStatuses) {
  return nextStatuses;
}

function saveStatuses(statuses) {
  console.log(statuses);
}

function getNames(cb) {
  withDb(function(db) {
    db.query('select name FROM streamer')
      .on('row', function(row, result) { result.addRow(row); })
      .on('end', function(result) { cb(result.rows); });
  });
}

var interval = process.env['TWITCH_STATUS_RECORDER_INTERVAL'] === null ? 60000 : process.env.TWITCH_STATUS_RECORDER_INTERVAL;
setInterval(function() {
  getNames(function(names) {
    if(names.length > 0) {
      getCurrentStatuses(names, function(currentStatuses) {
        getNextStatuses(names, function(nextStatuses) {
          var differentStatuses = getDifferingStatuses(currentStatuses, nextStatuses);
          saveStatuses(differentStatuses);
        });
      });
    }
  });
}, interval);

{'fraction_non_alphanumeric': 0.1515468227424749, 'fraction_numerical': 0.003762541806020067, 'mean_word_length': 3.8042168674698793, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5644264', 'n_tokens_mistral': 1605, 'n_tokens_neox': 1577, 'n_words': 328}