<commit_before>var Twitter = require('twitter')
var mongojs = require('mongojs')
// var consumer_key = process.env.CONSUMER_KEY
// var consumer_secret = process.env.CONSUMER_SECRET
// var access_token_key = process.env.ACCESS_TOKEN_KEY
// var access_token_secret = process.env.ACCESS_TOKEN_SECRET
// var dbuser = process.env.DBUSER
// var dbpass = process.env.DBPASSWORD
// var db = mongojs('mongodb://'+dbuser+':'+dbpass+'<EMAIL>:59908/livedata')
var consumer_key = "i1k8pKNzfJLMi8d9F6ISbs7SV"
var consumer_secret = "<KEY>"
var access_token_key = "<KEY>"
var access_token_secret = "<KEY>"
var db = mongojs('mydb')
var HOUR = 3600000
var DAY = 86400000
// var counter = 0;

// log the tweets
module.exports = initialize;

function initialize() {
  var master_controller = new MasterController;
  master_controller.connect()
  master_controller.stream()
}

function MasterController() {
  this.API = null;
  this.globe_controller = new GlobeController;
  this.database_controller = new DatabaseController;
}

MasterController.prototype = {
  connect: function() {
    this.API = new Twitter({
      consumer_key: consumer_key,
      consumer_secret: consumer_secret,
      access_token_key: access_token_key,
      access_token_secret: access_token_secret
    });
  },
  stream: function() {
    this.API.stream('filter', {'locations': '-180,-90,180,90'}, function(stream) {
      stream.on('data', function(data) {
        this.globe_controller.extractCoordinates(data);
        this.database_controller.extractHashtags(data);
        // dbController.parseRawTweet(null, data)
      }.bind(this));
    }.bind(this));
  }
}

function GlobeController() {
  // this.view = new GlobeView;
}

GlobeController.prototype = {
  extractCoordinates: function(data) {
    if (data.coordinates) {
      return data.coordinates.coordinates;
    }
  }
}

function DatabaseController() {}
<commit_msg>Added removeDeprecated method to database_controller and called it within the master_controller stream method.<commit_after>var Twitter = require('twitter')
var mongojs = require('mongojs')
// var consumer_key = process.env.CONSUMER_KEY
// var consumer_secret = process.env.CONSUMER_SECRET
// var access_token_key = process.env.ACCESS_TOKEN_KEY
// var access_token_secret = process.env.ACCESS_TOKEN_SECRET
// var dbuser = process.env.DBUSER
// var dbpass = process.env.DBPASSWORD
// var db = mongojs('mongodb://'+dbuser+':'+dbpass+'@ds059908.mongolab.com:59908/livedata')
var consumer_key = "i1k8pKNzfJLMi8d9F6ISbs7SV"
var consumer_secret = "<KEY>"
var access_token_key = "<KEY>"
var access_token_secret = "<KEY>"
var db = mongojs('mydb')
var HOUR = 3600000
var DAY = 86400000
// var counter = 0;

// log the tweets
module.exports = initialize;

function initialize() {
  var master_controller = new MasterController;
  master_controller.connect()
  master_controller.stream()
}

function MasterController() {
  this.API = null;
  this.globe_controller = new GlobeController;
  this.database_controller = new DatabaseController;
}

MasterController.prototype = {
  connect: function() {
    this.API = new Twitter({
      consumer_key: consumer_key,
      consumer_secret: consumer_secret,
      access_token_key: access_token_key,
      access_token_secret: access_token_secret
    });
  },
  stream: function() {
    this.API.stream('filter', {'locations': '-180,-90,180,90'}, function(stream) {
      this.database_controller.removeDeprecated();
      stream.on('data', function(data) {
        this.globe_controller.extractCoordinates(data);
        this.database_controller.extractHashtags(data);
      }.bind(this));
    }.bind(this));
  }
}

function GlobeController() {
  // this.view = new GlobeView;
}

GlobeController.prototype = {
  extractCoordinates: function(data) {
    if (data.coordinates) {
      return data.coordinates.coordinates;
    }
  }
}

function DatabaseController() {}

{'fraction_non_alphanumeric': 0.12483912483912483, 'fraction_numerical': 0.02059202059202059, 'mean_word_length': 4.316005471956224, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '64104', 'n_tokens_mistral': 1299, 'n_tokens_neox': 1231, 'n_words': 243}