<commit_before>var http = require("http");
var https = require("https");
var crypto = require('crypto');
var journal = require("./journal");

// read config file, containing twitter oauth data and various other preferences
var config = require("./config").read("app");
if (JSON.stringify(config.twitter.auth).indexOf("...") > 0) {
	console.log("Please add your Twitter API authorization tokens to `app.config`");
	process.exit(1);
}

exports.fetch = function(callback) {
	// NOTE: take care when changing the search_interval variable
	// the rate limit in Search API v1.1 is 180 requests / 15 minutes, meaning 1 request every 5 seconds

	setInterval(function() {
		fetchTweets(callback);
	}, config.twitter.search_interval * 1000);

	fetchTweets(callback);
};

exports.stream = function(callback) {
	// https://dev.twitter.com/docs/api/1.1/post/statuses/filter

	var queryParams = {
		"track": config.twitter.search_terms.join(",") // filter for common english words
	};

	var httpOptions = {
		hostname: 'stream.twitter.com',
		path: "/1.1/statuses/filter.json?" + serializeEncodeObject(queryParams).join("&"),
		headers: {
			'user-agent': 'Longest Poem In The World (v1, ' + process.pid + ')',
			'authorization': generateAuthorizationHeader("get", "https://stream.twitter.com/1.1/statuses/filter.json", queryParams)
		}
	};

	var request = https.request(httpOptions, function(response) {
		response.setEncoding('utf8');
		response.on('data', function(data) {
			try {
				var tweet = JSON.parse(data);
				callback(tweet);
			} catch (e) {
				// TODO: parseexception handling
			}
		});

		response.on('end', function() {
			console.log("Twitter stream connection interrupted.");
		});
	});

	request.end();
};

function fetchTweets(callback) {
	// https://dev.twitter.com/docs/api/1.1/get/search/tweets
	journal.info("fetching tweets");

	var queryParams = {
		"lang": "en",
		"result_type": "recent",
		"count": "100",
		"q": config.twitter.search_terms.join(" OR ") // filter for common english words
	};<commit_msg>Small typo<commit_after>var http = require("http");
var https = require("https");
var crypto = require('crypto');
var journal = require("./journal");

// read config file, containing twitter oauth data and various other preferences
var config = require("./config").read("app");
if (JSON.stringify(config.twitter.auth).indexOf("...") > 0) {
	console.log("Please add your Twitter API authorization tokens to `app.config`");
	process.exit(1);
}

exports.fetch = function(callback) {
	// NOTE: take care when changing the search_interval variable
	// the rate limit in Search API v1.1 is 180 requests / 15 minutes, meaning 1 request every 5 seconds

	setInterval(function() {
		fetchTweets(callback);
	}, config.twitter.search_interval * 1000);

	fetchTweets(callback);
};

exports.stream = function(callback) {
	// https://dev.twitter.com/docs/api/1.1/post/statuses/filter

	var queryParams = {
		"track": config.twitter.search_terms.join(",") // filter for common english words
	};

	var httpOptions = {
		hostname: 'stream.twitter.com',
		path: "/1.1/statuses/filter.json?" + serializeEncodeObject(queryParams).join("&"),
		headers: {
			'user-agent': 'Longest Poem In The World (v1, ' + process.pid + ')',
			'authorization': generateAuthorizationHeader("get", "https://stream.twitter.com/1.1/statuses/filter.json", queryParams)
		}
	};

	var request = https.request(httpOptions, function(response) {
		response.setEncoding('utf8');
		response.on('data', function(data) {
			try {
				var tweet = JSON.parse(data);
				callback(tweet);
			} catch (e) {
				// TODO: parse exception handling
			}
		});

		response.on('end', function() {
			console.log("Twitter stream connection interrupted.");
		});
	});

	request.end();
};

function fetchTweets(callback) {
	// https://dev.twitter.com/docs/api/1.1/get/search/tweets
	journal.info("fetching tweets");

	var queryParams = {
		"lang": "en",
		"result_type": "recent",
		"count": "100",
		"q": config.twitter.search_terms.join(" OR ") // filter for common english words
	};
{'fraction_non_alphanumeric': 0.166584280771132, 'fraction_numerical': 0.01384083044982699, 'mean_word_length': 4.195121951219512, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5677224', 'n_tokens_mistral': 1391, 'n_tokens_neox': 1260, 'n_words': 300}