<commit_before>
var mongo = require('mongodb');
var config = require('../conf');

    var mongoClient = mongo.MongoClient;

    
    var tweetsWithCoordinates = {};
    
    var getCoordinateViaTimeZone = false;
    
    var TDataObj = function (date,type,sentiment,tweet){
                        this.date = date;
                        this.type = type;
                        this.sentiment = sentiment;
                        this.tweet = tweet;
                    };

    var Coordinates = function (lat,lng){
        this.lat = lat
        this.lng = lng;
    }
    var MiniTData = function (lat, lng){
        this.coordinates = new Coordinates(lat,lng);
        this.tweetCount = 1;
        this.averageSentiment = 0;
    }
    
    var startReadingStream = function (){



        // Connect to the db //strat mango db before trying to connect.
        mongoClient.connect("mongodb://"+config.ip+":27017/wardolph", function(err, db) {
          if(!err) {
            console.log("tread: We are connected to mongo db");
            var collection = db.collection('feminism');//try feminismCoordinate for smaller set
            var timezoneCol = db.collection('timezone');
            var readyToDraw = db.collection('feminism_readyToDraw');
            readyToDraw.drop();
            var timezone = null;
            var dataCoordinateMappedSentiment = {};


            //start debugging section;
            var totalTweetCount = 1;
            collection.count(function(err, countRet) {
              totalTweetCount = countRet;
                console.log('total collections elements: '+totalTweetCount);
            });
            var currentStreamCount = 0;
            var printCompletionPercentageIntervalId = 0;
            var printCompletionPercentage = function(){
              var percentage = (currentStreamCount/totalTweetCount*100);
              console.log('completed percentage % : '+ percentage);
              if(percentage >= 100){
                clearInterval(printCompletionPercentageIntervalId);
              }
            }
            printCompletionPercentageIntervalId = setInterval(printCompletionPercentage, 30000);//print percentage every 30 sec
            //end debugging section;

            var getTimeZone = function(tweet){
                var timezoneFound = null;
                for(var i=0; i<timezone.length; i++){
                    if(typeof(tweet.user.time_zone) != 'undefined' && tweet.user.time_zone!=null && tweet.user.time_zone == timezone[i].name){
                        timezoneFound = timezone[i];
                        break;
                    }
                }
                return timezoneFound;
            }

            var initStream = function(){
                var stream = collection.find().stream();

                var firststream = true;
                stream.on("data", function(item) {

                    for (key in item) {
                        
                        if (key!='_id' && item.hasOwnProperty(key)) {
                            var sentiment = item[key].sentiment;
                            var tweet = item[key].tweet;
                            if(tweet!=null && typeof(tweet)!='undefined' && sentiment.words.length>0){
                                if( tweet.coordinates!=null && typeof(tweet.coordinates) != 'undefined'){
                                    var lat = tweet.coordinates.coordinates[1];
                                    var lng = tweet.coordinates.coordinates[0];
                                    var latlng = lat+','+lng;
                                    
                                    if( typeof(dataCoordinateMappedSentiment[latlng])!='undefined'){
                                        var myData = dataCoordinateMappedSentiment[latlng];
                                        myData.tweetCount++;
                                        myData.averageSentiment = myData.averageSentiment + sentiment.score;
                                    }
                                    else{
                                        var myData = new MiniTData(lat, lng);
                                        myData.averageSentiment = sentiment.score;
                                        dataCoordinateMappedSentiment[latlng] = myData;
                                    }
                                    
                                    //streamCallback(myData);
                                }
                                else{
                                    var timezoneFound = getTimeZone(tweet);
                                    if(timezoneFound!=null){
                                        var lat = timezoneFound.lat;
                                        var lng = timezoneFound.lng;
                                        var latlng = lat+','+lng;
                                        
                                        if( typeof(dataCoordinateMappedSentiment[latlng])!='undefined'){
                                            dataCoordinateMappedSentiment[latlng].tweetCount++;
                                        }
                                        else{
                                            var myData = new MiniTData(lat, lng);
                                            dataCoordinateMappedSentiment[latlng] = myData;
                                        }
                                        //streamCallback(myData);
                                    }
                                }<commit_msg>sentiment calculation fix<commit_after>
var mongo = require('mongodb');
var config = require('../conf');

    var mongoClient = mongo.MongoClient;

    
    var tweetsWithCoordinates = {};
    
    var getCoordinateViaTimeZone = false;
    
    var TDataObj = function (date,type,sentiment,tweet){
                        this.date = date;
                        this.type = type;
                        this.sentiment = sentiment;
                        this.tweet = tweet;
                    };

    var Coordinates = function (lat,lng){
        this.lat = lat
        this.lng = lng;
    }
    var MiniTData = function (lat, lng){
        this.coordinates = new Coordinates(lat,lng);
        this.tweetCount = 1;
        this.averageSentiment = 0;
    }
    
    var startReadingStream = function (){



        // Connect to the db //strat mango db before trying to connect.
        mongoClient.connect("mongodb://"+config.ip+":27017/wardolph", function(err, db) {
          if(!err) {
            console.log("tread: We are connected to mongo db");
            var collection = db.collection('feminism');//try feminismCoordinate for smaller set
            var timezoneCol = db.collection('timezone');
            var readyToDraw = db.collection('feminism_readyToDraw');
            readyToDraw.drop();
            var timezone = null;
            var dataCoordinateMappedSentiment = {};


            //start debugging section;
            var totalTweetCount = 1;
            collection.count(function(err, countRet) {
              totalTweetCount = countRet;
                console.log('total collections elements: '+totalTweetCount);
            });
            var currentStreamCount = 0;
            var printCompletionPercentageIntervalId = 0;
            var printCompletionPercentage = function(){
              var percentage = (currentStreamCount/totalTweetCount*100);
              console.log('completed percentage % : '+ percentage);
              if(percentage >= 100){
                clearInterval(printCompletionPercentageIntervalId);
              }
            }
            printCompletionPercentageIntervalId = setInterval(printCompletionPercentage, 30000);//print percentage every 30 sec
            //end debugging section;

            var getTimeZone = function(tweet){
                var timezoneFound = null;
                for(var i=0; i<timezone.length; i++){
                    if(typeof(tweet.user.time_zone) != 'undefined' && tweet.user.time_zone!=null && tweet.user.time_zone == timezone[i].name){
                        timezoneFound = timezone[i];
                        break;
                    }
                }
                return timezoneFound;
            }

            var storeInLatLongMap = function(lat, lng, sentiment){
                var latlng = lat+','+lng;
                if( typeof(dataCoordinateMappedSentiment[latlng])!='undefined'){
                    var myData = dataCoordinateMappedSentiment[latlng];
                    myData.tweetCount++;
                    myData.averageSentiment = myData.averageSentiment + sentiment.score;
                }
                else{
                    var myData = new MiniTData(lat, lng);
                    myData.averageSentiment = sentiment.score;
                    dataCoordinateMappedSentiment[latlng] = myData;
                }
            }

            var initStream = function(){
                var stream = collection.find().stream();

                var firststream = true;
                stream.on("data", function(item) {

                    for (key in item) {
                        
                        if (key!='_id' && item.hasOwnProperty(key)) {
                            var sentiment = item[key].sentiment;
                            var tweet = item[key].tweet;
                            if(tweet!=null && typeof(tweet)!='undefined' && sentiment.words.length>0){
                                if( tweet.coordinates!=null && typeof(tweet.coordinates) != 'undefined'){
                                    var lat = tweet.coordinates.coordinates[1];
                                    var lng = tweet.coordinates.coordinates[0];
                                    
                                    storeInLatLongMap(lat,lng,sentiment);
                                    
                                }
                                else{
                                    var timezoneFound = getTimeZone(tweet);
                                    if(timezoneFound!=null){
                                        var lat = timezoneFound.lat;
                                        var lng = timezoneFound.lng;
                                        
                                        storeInLatLongMap(lat,lng,sentiment);
                                    }
                                }
{'fraction_non_alphanumeric': 0.08771929824561403, 'fraction_numerical': 0.005176876617773943, 'mean_word_length': 1.1638664177556524, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 56, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4027998', 'n_tokens_mistral': 2490, 'n_tokens_neox': 2297, 'n_words': 471}