<commit_before>'use strict';

var twit = require('twit'),
    _ = require('lodash'),
    moment = require('moment'),
    async = require('async');

var twitter = new twit({
    consumer_key: 'wVASXWImPduD7FaWNNK79iDio',
    consumer_secret: '<KEY>',
    access_token: '<KEY>',
    access_token_secret: '<KEY>'
});

var getTweetsByHashtag = function(hashtag, maxTweetsPerHashtag, fromTime, lang, callback) {
    var options = {
        q: '#' + hashtag,
        count: 100
    };
    if (lang) {
        //options.lang = lang;
    }
    twitter.get('search/tweets', options, function(err, data, response) {
        if (err) {
            callback(err, null);
        } else {
            var tweets = data.statuses;
            console.log(hashtag);
            console.log(tweets.length);
            if (fromTime) {
                tweets = _.filter(tweets, function(tweet) {
                    return parseInt(moment(new Date(tweet.created_at)).format('X')) > fromTime;
                });
            }
            //Filter retweets and links and replies
            tweets = _.filter(tweets, function(tweet) {
                return tweet.text.indexOf('http') === -1 && tweet.retweeted === false && tweet.text.indexOf('@') !== 0 && tweet.text.indexOf('RT') !== 0;
            });
            console.log(tweets.length);
            if (maxTweetsPerHashtag < 100) {
                var i = 0;
                tweets = _.filter(tweets, function(tweet) {
                    i = i + 1;
                    if (i <= maxTweetsPerHashtag) {
                        return true;
                    } else {
                        return false;
                    }
                });
            }
            console.log(tweets.length);
            tweets = _.map(tweets, function(tweet) {
                return {
                    timestamp: parseInt(moment(new Date(tweet.created_at)).format('X')),
                    text: tweet.text,
                    //tags: tweet.entities.hashtags,
                    //tag: media.tags[0]
                };
            });
            callback(null, tweets);
        }
    });
<commit_msg>Remove unnecessary logging<commit_after>'use strict';

var twit = require('twit'),
    _ = require('lodash'),
    moment = require('moment'),
    async = require('async');

var twitter = new twit({
    consumer_key: 'wVASXWImPduD7FaWNNK79iDio',
    consumer_secret: '<KEY>',
    access_token: '<KEY>',
    access_token_secret: '<KEY>'
});

var getTweetsByHashtag = function(hashtag, maxTweetsPerHashtag, fromTime, lang, callback) {
    var options = {
        q: '#' + hashtag,
        count: 100
    };
    if (lang) {
        //options.lang = lang;
    }
    twitter.get('search/tweets', options, function(err, data, response) {
        if (err) {
            callback(err, null);
        } else {
            var tweets = data.statuses;
            if (fromTime) {
                tweets = _.filter(tweets, function(tweet) {
                    return parseInt(moment(new Date(tweet.created_at)).format('X')) > fromTime;
                });
            }
            //Filter retweets and links and replies
            tweets = _.filter(tweets, function(tweet) {
                return tweet.text.indexOf('http') === -1 && tweet.retweeted === false && tweet.text.indexOf('@') !== 0 && tweet.text.indexOf('RT') !== 0;
            });
            if (maxTweetsPerHashtag < 100) {
                var i = 0;
                tweets = _.filter(tweets, function(tweet) {
                    i = i + 1;
                    if (i <= maxTweetsPerHashtag) {
                        return true;
                    } else {
                        return false;
                    }
                });
            }
            tweets = _.map(tweets, function(tweet) {
                return {
                    timestamp: parseInt(moment(new Date(tweet.created_at)).format('X')),
                    text: tweet.text,
                    //tags: tweet.entities.hashtags,
                    //tag: media.tags[0]
                };
            });
            callback(null, tweets);
        }
    });

{'fraction_non_alphanumeric': 0.1345735684948055, 'fraction_numerical': 0.00724812756704518, 'mean_word_length': 1.467222884386174, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1841460', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1196, 'n_words': 234}