<commit_before>var mongoose = require('mongoose')
    , Account = mongoose.model('Account')
    , ntwitter = require('ntwitter')
    , keys = require('../../../config/twitterKeys.json')
    , params    
    , maxTweets = 200
    , foundTweets = 0
    , corpus
    , twit
    , acct
    , res
    , publicSources = ['earthquake, usgs, ieeevis']


/* Checks cache for screenname, count and date.
 *
 * This should use maxid if date is stale so that the 
 * full record doesn't need to be fetched...
 * 
 * */

exports.checkCache = function (acct, args) {
    //no way to access cache
    if (!acct.streams) return null
    if (!args) return null
   
    var sn = ct = null 
    if (args[0]) sn = args[0]
    if (args[1]) ct = args[1]

    var dt = new Date();
    dt.setMinutes(dt.getMinutes() - 15); 
    
    for (var index in acct.streams) {
        var at = acct.streams[index]
        if (at.screen_name==sn && at.count>=ct && at.dateRequested>=dt) {
            return at
        }
    }
    return null
}

exports.init = function (account, args, resp) {
    
    acct    = account
    res     = resp
    corpus  = "["
    foundTweets=0

    var key = keys
    var isPublic = true
    //use public feeds...

    if (acct) {
        if (acct.tokens) {
            key.access_token_key = acct.tokens.token
            key.access_token_secret = acct.tokens.tokenSecret
            isPublic = false
        }
    }

    if (isPublic) {
        if (args[1] in publicSources) {
            //set params and call getTweets...
        }
        //screen name must be one of public sources... 
    }

    //use account keys rather than global keys...
    twit = new ntwitter(key)

    if (args) {
        if (args[1]) maxTweets=args[1]
        //is it streaming or user feed
        if (args[0]) {
            params = {
                screen_name: args[0],
                count: 200,
                include_rts:true
            }
            res.writeHead(200, {'Content-Type':'application/json'})
            res.write("[")
            getTweets(null, "") 
        } else {
            res.send("streaming is not yet supported")
        }
    
        //getTweets(null, "")
        //instead of null pass maxid from cache, 
        //then concat with cache
        //in this case... subtract cache count from maxTweet count?
    }
}


function getTweets(maxid, tweets) {
     
    if (maxid) params.max_id = maxid
    
    twit.getUserTimeline(params, function (err,data) {
        if (err) {
            res.json(err) 
            return err
        }
        if ( data.length > 1 && maxTweets > foundTweets ) {
            //foundTweets=foundTweets+200; //200 tweets per request, less accurate, but faster
            
            var tweetID = countRT = countT = 0
            data.forEach(function(tweet) {
                if (tweet.retweeted_status) {
                    retweet = tweet.retweeted_status
                    tweets = tweets.concat(<commit_msg>modified to limit only public sources when querying without a twitter account<commit_after>var mongoose = require('mongoose')
    , Account = mongoose.model('Account')
    , ntwitter = require('ntwitter')
    , keys = require('../../../config/twitterKeys.json')
    , params    
    , maxTweets = 200
    , foundTweets = 0
    , corpus
    , twit
    , acct
    , res
    , publicSources = {'earthquake':1, 'usgs':1, 'ieeevis':1}


/* Checks cache for screenname, count and date.
 *
 * This should use maxid if date is stale so that the 
 * full record doesn't need to be fetched...
 * 
 * */

exports.checkCache = function (acct, args) {
    //no way to access cache
    if (!acct.streams) return null
    if (!args) return null
   
    var sn = ct = null 
    if (args[0]) sn = args[0]
    if (args[1]) ct = args[1]

    var dt = new Date();
    dt.setMinutes(dt.getMinutes() - 15); 
    
    for (var index in acct.streams) {
        var at = acct.streams[index]
        if (at.screen_name==sn && at.count>=ct && at.dateRequested>=dt) {
            return at
        }
    }
    return null
}

exports.init = function (account, args, resp) {
    
    acct    = account
    res     = resp
    corpus  = "["
    foundTweets=0

    var key = keys
    var isPublic = true
    //use public feeds...

    if (acct) {
        if (acct.tokens) {
            key.access_token_key = acct.tokens.token
            key.access_token_secret = acct.tokens.tokenSecret
            isPublic = false
        }
    }
    if (isPublic) {
        if (args[0] in publicSources) {
            //set params and call getTweets...
        } else {
            var srcs = ""
            for (i in publicSources) {
                srcs+=i+" "
            }
            return res.send("Please select a public source: " + srcs)
        }
        //screen name must be one of public sources... 
    }

    //use account keys rather than global keys...
    twit = new ntwitter(key)

    if (args) {
        if (args[1]) maxTweets=args[1]
        //is it streaming or user feed
        if (args[0]) {
            params = {
                screen_name: args[0],
                count: 200,
                include_rts:true
            }
            res.writeHead(200, {'Content-Type':'application/json'})
            res.write("[")
            getTweets(null, "") 
        } else {
            res.send("streaming is not yet supported")
        }
    
        //getTweets(null, "")
        //instead of null pass maxid from cache, 
        //then concat with cache
        //in this case... subtract cache count from maxTweet count?
    }
}


function getTweets(maxid, tweets) {
     
    if (maxid) params.max_id = maxid
   
        //getFriendsId
        //getFollowersIds

    twit.getUserTimeline(params, function (err,data) {
        if (err) {
            res.json(err) 
            return err
        }
        if ( data.length > 1 && maxTweets > foundTweets ) {
            //foundTweets=foundTweets+200; //200 tweets per request, less accurate, but faster
            
            var tweetID = countRT = countT = 0
            data.forEach(function(tweet) {
                if (tweet.retweeted_status) {
                    retweet = tweet.retweeted_status
                    tweets = tweets.concat(
{'fraction_non_alphanumeric': 0.11463336535382646, 'fraction_numerical': 0.010086455331412104, 'mean_word_length': 1.6403212172442942, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5551227', 'n_tokens_mistral': 2020, 'n_tokens_neox': 1918, 'n_words': 523}