<commit_before>var mongoRepository = require('./mongoRepository.js');
var eventRepository = require('./eventRepository.js');
var Q = require('q');
var moment = require("moment");
var platformService = require('./platformService.js');

var appIds = ["app-id-9adce70ae3ef5c4c8389910b9abb95b1", //github
    "app-id-07be2f65eccd23895b75313a1eb8283b", //lastfm
    "app-id-tw4f3dsd91d9a3e715ff98bb9eedbd0a", //twitter
    "app-id-gf4f3dsd93d9a3e715ff98bb9eedbd0a", //google fit
    "app-id-fsq9954f966461936d47b7b4ba1e6f2e", //Four Square
    "app-id-1bb5f1c77f0df722a9b1bc650a41988a", //Instagram
    "app-id-hn80883506c93b80dcf2c3780d223eaa", //Hacker News
    "app-id-st39ec18b3ad1dc3f3783f856c033418", //Strava
    "app-id-rtbbd9175a69edbc4lol5d2747b638a7"]; //Rescue time

var findStreamsForAppIds = function (appIds) {
    var query = {
        "appId": {
            "$in": appIds
        }
    };
    var projection = {
        "streamid": 1
    };
    return mongoRepository.find("stream", query, projection)
        .then(function (streams) {
            return _.forEach(streams, function (stream) {
                return stream.streamid;
            });
        });
};

var getLatestSyncField = function (streamId) {
    var deferred = q.defer();
    var filterSpec = {
        'payload.streamid': streamId
    };
    var orderSpec = {
        "payload.latestSyncField": -1
    };
    var options = {
        "limit": 1
    };
    var query = {
        'filterSpec': JSON.stringify(filterSpec),
        'orderSpec': JSON.stringify(orderSpec),
        'options': JSON.stringify(options)
    };
    platformService.filter(query)
        .then(function (result) {
            deferred.resolve(result[0].payload.latestSyncField);
        }, function (err) {
            deferred.reject(err);
        });
    return deferred.promise;
};
var updateLatestSyncField = function (streamId, latestSyncField) {
    var deferred = q.defer();
    var query = {"streamid": streamId};
    var updateObject = {
        $set: {"latestSyncField": latestSyncField}
    };
    mongoRepository.update('stream', query, updateObject)
        .then(function () {
            deferred.resolve(updateObject);
        }, function (err) {
            deferred.reject(err);
        });
    return deferred.promise;
};

findStreamsForAppIds(appIds)
    .then(function (streamIds) {
        _.forEach(streamIds, function(streamId){
            var query = {
                "payload.streamid": streamId,
                "event.createdOn": {
                    "$gt": moment("2015-03-13T00:00:00.000Z").toISOString()
                }
            };
            return eventRepository.remove("oneself",query)
                .then(function(noOfEventsRemoved){
                    console.log("No Of Events Removed: ", noOfEventsRemoved);
                    return getLatestSyncField(streamId);
                })
                .then(function(latestSyncField){
                    return updateLatestSyncField(streamId, latestSyncField)
                })
        })
    });<commit_msg>-- If the user has no integration data after migration, set latestSyncFieldto default<commit_after>var mongoRepository = require('./mongoRepository.js');
var eventRepository = require('./eventRepository.js');
var Q = require('q');
var moment = require("moment");
var platformService = require('./platformService.js');

var appIds = ["app-id-9adce70ae3ef5c4c8389910b9abb95b1", //github
    "app-id-07be2f65eccd23895b75313a1eb8283b", //lastfm
    "app-id-tw4f3dsd91d9a3e715ff98bb9eedbd0a", //twitter
    "app-id-gf4f3dsd93d9a3e715ff98bb9eedbd0a"]; //Google Fit

var findStreamsForAppIds = function (appIds) {

};

var getLatestSyncField = function (streamId) {
    var deferred = q.defer();
    var filterSpec = {
        'payload.streamid': streamId
    };
    var orderSpec = {
        "payload.latestSyncField": -1
    };
    var options = {
        "limit": 1
    };
    var query = {
        'filterSpec': JSON.stringify(filterSpec),
        'orderSpec': JSON.stringify(orderSpec),
        'options': JSON.stringify(options)
    };
    platformService.filter(query)
        .then(function (result) {
            if (_.isEmpty(result)) {
                deferred.resolve();
            }
            deferred.resolve(result[0].payload.latestSyncField);
        }, function (err) {
            deferred.reject(err);
        });
    return deferred.promise;
};
var updateLatestSyncField = function (streamId, latestSyncField) {
    var deferred = q.defer();
    var query = {"streamid": streamId};
    var updateObject = {
        $set: {"latestSyncField": latestSyncField}
    };
    mongoRepository.update('stream', query, updateObject)
        .then(function () {
            deferred.resolve(updateObject);
        }, function (err) {
            deferred.reject(err);
        });
    return deferred.promise;
};

var query = {
    "appId": {
        "$in": appIds
    }
};
var projection = {};

mongoRepository.find("stream", query, projection)
    .then(function (streams) {
        _.forEach(streams, function (stream) {
            var query = {
                "payload.streamid": stream.streamid,
                "event.createdOn": {
                    "$gt": moment("2015-03-13T00:00:00.000Z").toISOString()
                }
            };
            return eventRepository.remove("oneself", query)
                .then(function (noOfEventsRemoved) {
                    console.log("No Of Events Removed: ", noOfEventsRemoved);
                    return getLatestSyncField(streamId);
                })
                .then(function (latestSyncField) {
                    if (_.isEmpty(latestSyncField)) {
                        if (stream.appId === "app-id-tw4f3dsd91d9a3e715ff98bb9eedbd0a") {
                            latestSyncField = "00000000";
                        }
                        else if(stream.appId === "app-id-gf4f3dsd93d9a3e715ff98bb9eedbd0a"){
                            latestSyncField = null;
                        }
                        else {
                            latestSyncField = new Date(1970,1,1).toISOString();
                        }
                    }
                    return updateLatestSyncField(streamId, latestSyncField)
                })
        });
    });
{'fraction_non_alphanumeric': 0.12507926442612555, 'fraction_numerical': 0.04787571337983513, 'mean_word_length': 2.0881057268722465, 'pattern_counts': {'":': 19, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3624488', 'n_tokens_mistral': 2075, 'n_tokens_neox': 1899, 'n_words': 316}