<commit_before>var tweetsdb = require('store').tweets;
var classifier = require('class');

var batchSize = 1000;


classifier.setup(function (classification) {
	tweetsdb.update(
		{ id_str : classification.id_str },
		{ $set : classification }
		);
});

while(true) {
	var tweets = tweetsdb.find({classification:{$exists:false}}).limit(batchSize);

	
}

var db = require("mongojs").connect(nconf.get('database'), [nconf.get('collection')]);

var nTwitter = require("ntwitter");
var twitter = new nTwitter(nconf.get('twitter_auth'));

var i=0;
twitter.stream(
	'statuses/filter',
	{ track: nconf.get('keywords') },
	function(stream) {
		stream.on('data', function(tweet) {
			if(i > nconf.get('capture:frequency') && tweet.user.lang == 'en')
			{
				i = 0;
				tweet.random = Math.random();
				tweet.created_at = new Date(tweet.created_at);
				db.twits.save(tweet);
				console.log('tweet: '+ tweet.created_at);
			}
			i++;
		});
	}
);
<commit_msg>basic classifier2<commit_after>var tweetsdb = require('store').tweets;
var classs = require('class');

var batchSize = 1000;


var classifier = classs.setup(function (classification) {
	tweetsdb.update(
		{ id_str : classification.id_str },
		{ $set : classification }
		);
});

while(true) {
	var tweets = tweetsdb.find({classification:{$exists:false}}).limit(batchSize);

	// Stream tweets from database
	var tstream = tweets.cursor.stream();
	tstream.on('data', function (tweet) {
	  classifier.classify(tweet);
	});
	tstream.on('close', function () {
	});
}

classifier.end();
{'fraction_non_alphanumeric': 0.18027433050293926, 'fraction_numerical': 0.007184846505551927, 'mean_word_length': 4.695167286245353, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2841174', 'n_tokens_mistral': 596, 'n_tokens_neox': 528, 'n_words': 92}