<commit_before>(function() {

	var sys = require('sys');
	var events = require('events');
	var ntwitter = require('ntwitter');

	function TDTwitterStream(options) {
		if (!(this instanceof TDTwitterStream)) { return new TDTwitterStream(options); }

		if (typeof options != "undefined")
		{
			this.set_options(options);
		}
	}

	TDTwitterStream.prototype = new events.EventEmitter();

	TDTwitterStream.prototype.set_options = function(options)
	{
		if (typeof options != "object")
		{
			this.emit("error", "Options must be an object");
		}

		this.options = options;
	}

	TDTwitterStream.prototype.merge_options = function()
	{
		var defaults = {
			"auth": {
				"consumer_key": null,
				"consumer_secret": null,
				"access_token_key": null,
				"access_token_secret": null,
			},
			"follow": [],
			"with": "followings"	
		};

		this.options = defaults.extend(this.options);

		this.check_options();
	}

	TDTwitterStream.prototype.check_options = function()
	{
		// Starting with OAuth tokens (They're all required)
		for (i in this.options.auth)
		{
			if (this.options.auth[i] == null)
			{
				this.emit("error", "Option "+i+" is required");
			}
		}

		// Then a list of people to follow
		if (this.options.follow.length < 1)
		{
			this.emit("error", "You need to follow at least one person's stream");
		}

	}

	
	TDTwitterStream.prototype.run = function()
	{
		var self = this; // Aliasing for later
		self.merge_options(); // Make sure we use defaults if we need

		// Fire up the ntwitter stream
		ntwit = new ntwitter(self.options.auth);
		ntwit.stream('site', {"follow": self.options.follow, "with": self.options.with}, function(stream) {

			stream.on("error", function (error) {
				console.log(sys.inspect(error));

				// Exit process so Forever can restart it
				stream.destroy();
				process.exit(1);
			});

			stream.on("end", function (end) {
				console.log(sys.inspect(end));

				// Exit process so Forever can restart it
				stream.destroy();
				process.exit(1);
			});

			stream.on('data', function (data) {  
				// Aimed at the system
				self._control(data);

				// Aimed at a user
				// We call all of these functions, but most won't do anything
				if (typeof data.for_user != "undefined")
				{
					self._connect(data.for_user, data.message);
					self._tweet(data.for_user, data.message);
					self._favourite(data.for_user, data.message);
				}

				//console.log(sys.inspect(data));
				
			});
		});

	}

	// "Private" methods of taking an envelope and checking if we need to do anything with it

	TDTwitterStream.prototype._control = function(data)
	{
		if (typeof data.control != "undefined")
		{
			this.emit("control", data.control.control_uri);
		}
	}

	TDTwitterStream.prototype._connect = function(recipient, data)
	{
		if (typeof data.friends != "undefined")
		{
			this.emit("connect", data.friends);
		}
	}


	TDTwitterStream.prototype._tweet = function(recipient, data)
	{
		if (typeof data.text != "undefined")
		{
			// If it was a retweet
			if ( typeof data.retweeted_status != "undefined" )
			{
				this.emit("tweet", data.retweeted_status, data.retweeted_status.user);
				this.emit("retweet", data.retweeted_status, data);
				this.emit("user", data.retweeted_status.user);
			}
			else
			{
				this.emit("tweet", data, data.user);
			}

			// Always save the person that RT'd
			this.emit("user", data.user);

		}

		// Deleting tweets
		if (typeof data.delete != "undefined")
		{
			console.log();
			this.emit("tweet_deleted", data.delete.status.user_id_str, data.delete.status.id_str);
		}
	}

	TDTwitterStream.prototype._favourite = function(recipient, data)
	{
		if (typeof data.friends != "undefined")
		{
			this.emit("connect", data.friends);
		}
	}

<commit_msg>Now emitting events for retweets<commit_after>(function() {

	var sys = require('sys');
	var events = require('events');
	var ntwitter = require('ntwitter');

	function TDTwitterStream(options) {
		if (!(this instanceof TDTwitterStream)) { return new TDTwitterStream(options); }

		if (typeof options != "undefined")
		{
			this.set_options(options);
		}
	}

	TDTwitterStream.prototype = new events.EventEmitter();

	TDTwitterStream.prototype.set_options = function(options)
	{
		if (typeof options != "object")
		{
			this.emit("error", "Options must be an object");
		}

		this.options = options;
	}

	TDTwitterStream.prototype.merge_options = function()
	{
		var defaults = {
			"auth": {
				"consumer_key": null,
				"consumer_secret": null,
				"access_token_key": null,
				"access_token_secret": null,
			},
			"follow": [],
			"with": "followings"	
		};

		this.options = defaults.extend(this.options);

		this.check_options();
	}

	TDTwitterStream.prototype.check_options = function()
	{
		// Starting with OAuth tokens (They're all required)
		for (i in this.options.auth)
		{
			if (this.options.auth[i] == null)
			{
				this.emit("error", "Option "+i+" is required");
			}
		}

		// Then a list of people to follow
		if (this.options.follow.length < 1)
		{
			this.emit("error", "You need to follow at least one person's stream");
		}

	}

	
	TDTwitterStream.prototype.run = function()
	{
		var self = this; // Aliasing for later
		self.merge_options(); // Make sure we use defaults if we need

		// Fire up the ntwitter stream
		ntwit = new ntwitter(self.options.auth);
		ntwit.stream('site', {"follow": self.options.follow, "with": self.options.with}, function(stream) {

			stream.on("error", function (error) {
				console.log(sys.inspect(error));

				// Exit process so Forever can restart it
				stream.destroy();
				process.exit(1);
			});

			stream.on("end", function (end) {
				console.log(sys.inspect(end));

				// Exit process so Forever can restart it
				stream.destroy();
				process.exit(1);
			});

			stream.on('data', function (data) {  
				// Aimed at the system
				self._control(data);

				// Aimed at a user
				// We call all of these functions, but most won't do anything
				if (typeof data.for_user != "undefined")
				{
					self._connect(data.for_user, data.message);
					self._tweet(data.for_user, data.message);
					self._favourite(data.for_user, data.message);
				}

				//console.log(sys.inspect(data));
				
			});
		});

	}

	// "Private" methods of taking an envelope and checking if we need to do anything with it

	TDTwitterStream.prototype._control = function(data)
	{
		if (typeof data.control != "undefined")
		{
			this.emit("control", data.control.control_uri);
		}
	}

	TDTwitterStream.prototype._connect = function(recipient, data)
	{
		if (typeof data.friends != "undefined")
		{
			this.emit("connect", recipient, data.friends);
		}
	}


	TDTwitterStream.prototype._tweet = function(recipient, data)
	{
		if (typeof data.text != "undefined")
		{

			// If it was a RT
			if ( typeof data.retweeted_status != "undefined" )
			{
				this.emit("tweet", recipient, data.retweeted_status, data.retweeted_status.user);
				
				this.emit("user", recipient, data.retweeted_status.user);

				if (data.retweeted_status.in_reply_to_user_id_str == recipient){
					self.emit("mention", recipient, data.retweeted_status);
				}

				// Retweets
	            if (data.user.id_str == recipient){
					self.emit("retweet_by_me", recipient, data.retweeted_status, data);
		        }

				if (data.retweeted_status.user.id_str == recipient){
					self.emit("retweet_of_me", recipient, data.retweeted_status, data);
	        	}

	        	this.emit("retweet_by_following", recipient, data.retweeted_status, data);

			}

			// It's not a RT
			else
			{
				this.emit("tweet", recipient, data, data.user);
				if (data.in_reply_to_user_id_str == recipient){
					self.emit("mention", recipient, data);
				}
			}

			// Always, if it's a RT or not
			if (data.user.id_str == recipient){
              self.emit("own_tweet", recipient, data);
            }

			this.emit("user", recipient, data.user);

            

		}

		// Deleting tweets
		if (typeof data.delete != "undefined")
		{
			console.log();
			this.emit("tweet_deleted", data.delete.status.user_id_str, data.delete.status.id_str);
		}
	}

	TDTwitterStream.prototype._favourite = function(recipient, data)
	{
		if (typeof data.friends != "undefined")
		{
			this.emit("connect", data.friends);
		}
	}


{'fraction_non_alphanumeric': 0.1461789996366719, 'fraction_numerical': 0.0007266561705219813, 'mean_word_length': 3.1770359129994943, 'pattern_counts': {'":': 18, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '709513', 'n_tokens_mistral': 3293, 'n_tokens_neox': 2697, 'n_words': 608}