<commit_before>/**
 * Created by tigran on 7/20/15.
 */

require("./model");

var mongoose = require("mongoose")
    , Twitter = require('twitter-node-client').Twitter
    , async = require('async')
    , City = mongoose.model("City")
    , Country = mongoose.model("Country")
    , SocialPost = mongoose.model("SocialPost")
    , SocialUser = mongoose.model("SocialUser")
    , Token = mongoose.model("Token");

function twoDigits(d) {
    if(0 <= d && d < 10) return "0" + d.toString();
    if(-10 < d && d < 0) return "-0" + (-1*d).toString();
    return d.toString();
}

Date.prototype.toMysqlFormat = function() {
    return this.getUTCFullYear() + "-" + twoDigits(1 + this.getUTCMonth()) + "-" + twoDigits(this.getUTCDate());
};


Token.findOne({social: "twitter"}, function (err, twitter_keys) {
    var config = {
        "consumerKey": twitter_keys.token_data["consumer_key"],
        "consumerSecret": twitter_keys.token_data["consumer_secret"],
        "accessToken": twitter_keys.token_data["access_token"],
        "accessTokenSecret": twitter_keys.token_data["access_token_secret"]
    };
    var twitter = new Twitter(config);

    async.forever(function (next) {
        City.find({}, function (e, cities) {
            async.eachSeries(cities, function (city, cb) {
                var date = new Date();

                twitter.getSearch({
                    q: 'since:' + date.toMysqlFormat()
                    , geocode: city.latitude + "," + city.longitude + ",5000mi"
                    , count: 100
                    , result_type: "popular"
                }, function (err, response, body) {
                    console.log('ERROR [%s]', body);
                    setTimeout(function () {
                        cb();
                    }, 2500);
                }, function (data) {
                    if(typeof data !== 'object')
                    {
                        data = JSON.parse(data);
                    }
                    async.eachSeries(data["statuses"], function (post_data, next_post) {
                        SocialUser.findOne({user_id: post_data["user"]["id_str"]}, function (uerr, user) {
                            if(uerr)
                            {
                                console.log(err);
                                next_post(err);
                            }
                            else
                            {
                                var post_user;
                                var save_post_fn = function () {
                                    SocialPost.find({post_id: post_data["id_str"]}, function (perror, found_posts) {
                                        if (perror)
                                        {<commit_msg>adding print index<commit_after>/**
 * Created by tigran on 7/20/15.
 */

require("./model");

var mongoose = require("mongoose")
    , Twitter = require('twitter-node-client').Twitter
    , async = require('async')
    , City = mongoose.model("City")
    , Country = mongoose.model("Country")
    , SocialPost = mongoose.model("SocialPost")
    , SocialUser = mongoose.model("SocialUser")
    , Token = mongoose.model("Token");

function twoDigits(d) {
    if(0 <= d && d < 10) return "0" + d.toString();
    if(-10 < d && d < 0) return "-0" + (-1*d).toString();
    return d.toString();
}

Date.prototype.toMysqlFormat = function() {
    return this.getUTCFullYear() + "-" + twoDigits(1 + this.getUTCMonth()) + "-" + twoDigits(this.getUTCDate());
};

var request_index = 0;

Token.findOne({social: "twitter"}, function (err, twitter_keys) {
    var config = {
        "consumerKey": twitter_keys.token_data["consumer_key"],
        "consumerSecret": twitter_keys.token_data["consumer_secret"],
        "accessToken": twitter_keys.token_data["access_token"],
        "accessTokenSecret": twitter_keys.token_data["access_token_secret"]
    };
    var twitter = new Twitter(config);

    async.forever(function (next) {
        City.find({}, function (e, cities) {
            async.eachSeries(cities, function (city, cb) {
                console.log(request_index);
                request_index++;
                var date = new Date();

                twitter.getSearch({
                    q: 'since:' + date.toMysqlFormat()
                    , geocode: city.latitude + "," + city.longitude + ",5000mi"
                    , count: 100
                    , result_type: "popular"
                }, function (err, response, body) {
                    console.log('ERROR [%s]', body);
                    setTimeout(function () {
                        cb();
                    }, 2500);
                }, function (data) {
                    if(typeof data !== 'object')
                    {
                        data = JSON.parse(data);
                    }
                    async.eachSeries(data["statuses"], function (post_data, next_post) {
                        SocialUser.findOne({user_id: post_data["user"]["id_str"]}, function (uerr, user) {
                            if(uerr)
                            {
                                console.log(err);
                                next_post(err);
                            }
                            else
                            {
                                var post_user;
                                var save_post_fn = function () {
                                    SocialPost.find({post_id: post_data["id_str"]}, function (perror, found_posts) {
                                        if (perror)
                                        {
{'fraction_non_alphanumeric': 0.13332143494556487, 'fraction_numerical': 0.009459218275923612, 'mean_word_length': 1.460052677787533, 'pattern_counts': {'":': 8, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5407192', 'n_tokens_mistral': 1570, 'n_tokens_neox': 1477, 'n_words': 279}