<commit_before>/**
 * Created by <NAME> (shladkou) on 2/26/14.
 */
var util = require('util');
var config = require('../config');
var log = require('../libs/log')(module);
var Module = require('../models/module').Module;
var Page = require('../models/page').Page;
var Issue = require('../models/issue').Issue;
var _ = require('underscore');
var async = require('async');

var JiraApi = require('jira').JiraApi;
var response = null;

var epicsList = [];
var issuesList = [];
var epicIssueMap = {};
var linkedIssueUniqList = [];
var updateInProgress = false;

exports.rememberResponse = function (res) {
    response = res;
    UpdateProgress(0, "page");
    UpdateProgress(0, "issues");
};

var UpdateProgress = function (progress, type) {
    if(response != null) {
        response.write("event: progress\n");
        response.write('data: {"' + type + '":' + progress.toString() + '}\n\n');
    }
    if (progress > 0) {
        LogProgress("**********" + type + " Progress " + progress.toString() + "% **********");
    }
};

var LogProgress = function (text, error) {
    if (response && error == null) {
        response.write("event: logmessage\n");
        response.write("data: " + text + "\n\n");
    }
    if (error) {
        log.error(text);
        log.error(error);
        if (response) {
            response.write("event: errmessage\n");
            var errorText = error == null ? "" : error.message;
            response.write("data: " + text + ", reason - " + errorText + "\n\n");
        }
    }
    else {
        log.info(text);
    }
};

exports.updateJiraInfo = function (full, jiraUser, jiraPassword, callback) {
    if (updateInProgress) {
        callback();
    }

    updateInProgress = true;

    LogProgress("**** Step 0: async processing");

    var jira = new JiraApi(config.get("jiraAPIProtocol"), config.get("jiraUrl"), config.get("jiraPort"), jiraUser, jiraPassword, '2');

    async.series([
        //step 1
        function (callback) {
            //grab all modules
            LogProgress("**** Step 1: collect modules");
            Step1CollectModules(jira, callback);
        },
        //step 2
        function (callback) {
            LogProgress("**** Step 2: collect pages");
            //grab pages list
            Step2CollectPages(jira, full, callback);
        },
        //step 3
        function (callback) {
            //process pages
            LogProgress("**** Step 3: process " + issuesList.length + " pages");
            Step3ProcessPages(jira, callback);
        },
        //step 4
        function (callback) {
            // process linked issues pages
            LogProgress("**** Step 4: process " + linkedIssueUniqList.length + " blockers");
            Step4ProcessBlockers(jira, callback);
        },
        //step 5
        function (callback) {
            LogProgress("**** Step 5: Update End");
            response.end();
            updateInProgress = false;
            callback();
        }
    ],
    //optional callback
    function(err) {
        if(err) {
            LogProgress("**** Update Failed ****", err);
        }
        else {
            LogProgress("**** Update Succeed ****");
        }
    }
    );
    callback();
};

function Step1CollectModules(jira, callback) {
    var requestString = "project = PLEX-UXC AND issuetype = epic AND summary ~ Module AND NOT summary ~ automation ORDER BY key ASC";
    epicsList = [];

    UpdateProgress(0, "page");
    UpdateProgress(0, "issues");

    var loopError = true;
    async.whilst(function() {
            return loopError;
        },
        function(callback) {
        jira.searchJira(requestString, { fields: ["summary", "duedate", "assignee"] }, function (error, epics) {
            if (error) {
                callback(error);
            }
            if (epics != null) {
                async.eachSeries(epics.issues, function (epic, callback) {
                        Module.findOne({ key: epic.key }, function (err, module) {
                            if (!module) {
                                module = new Module();
                            }
                            module.key = epic.key;
                            module.summary = epic.fields.summary;
                            module.duedate = epic.fields.duedate == null ? null : new Date(epic.fields.duedate);
                            module.assignee = epic.fields.assignee == null ? "Unassigned" : epic.fields.assignee.name;
                            module.save(function () {
                                epicsList.push(epic.key);
                                LogProgress(epic.key + " : " + " Module Collected");
                                callback();
                            })
                        });
                    },
                    function (err) {
                        if (err) {
                            LogProgress("Collect modules error happened!", err);
                        }
                        LogProgress(epicsList.length + " : " + " Modules Collected");
                        if(err == null) {
                            loopError = false;
                        }
                        callback(err);
                    });
            }
        });
    },
    function(err) {
        if(err) {
            LogProgress("Restarting Loop");
        }
        callback(err);
    });
}

function Step2CollectPages(jira, full, callback) {
    issuesList = [];
    epicIssueMap = {};

    async.eachLimit(epicsList, 10, function (epic, callback) {
            CollectPagesFromJira(jira, full, epic, callback);
        },
        function (err) {
            callback(err);
        }
    );
}

function Step3ProcessPages(jira, callback) {
    var counter = 0;
    var lastProgress = 0;
    linkedIssueUniqList = {};

    async.eachLimit(issuesList, 10, function (issueKey, callback) {
            var currentProgress = Math.floor((++counter * 100) / issuesList.length);
            if (lastProgress != currentProgress) {
                lastProgress = currentProgress;
                UpdateProgress(currentProgress, "page");
            }
            ProcessPageFromJira(jira, issueKey, counter, callback);
        },
        function (err) {
            callback(err);
        }
    );
}

function Step4ProcessBlockers(jira, callback) {
    var counter = 0;
    var lastProgress = 0;
    var keys = Object.keys(linkedIssueUniqList);
    async.eachLimit(keys, 10, function (linkedIssueKey, callback) {
            var currentProgress = Math.floor((++counter * 100) / keys.length);
            if (lastProgress != currentProgress) {
                lastProgress = currentProgress;
                UpdateProgress(currentProgress, "issues");
            }
            var linkedIssue = linkedIssueUniqList[linkedIssueKey];
            ProcessBlockersFromJira(jira, linkedIssue, counter, callback);
        },
        function (err) {
            callback(err);
        }
    );
}

function CollectPagesFromJira(jira, full, moduleKey, callback) {
    var queryString = full ?
        util.format("project = PLEXUXC AND issuetype = Story AND 'Epic Link' in (%s)", moduleKey) :
        util.format("project = PLEXUXC AND issuetype = Story AND 'Epic Link' in (%s) AND updated > -3d", moduleKey);

    var loopError = true;
    async.whilst(function() {
            return loopError;
        },
        function(callback) {
            LogProgress("**** collect pages for module: " + moduleKey);
            jira.searchJira(queryString, { fields: ["summary"] }, function (error, stories) {
                if (error) {
                    LogProgress("Collect pages error happened!", error);
                    LogProgress("Restarting Loop for: "+moduleKey, error);
                }
                if (stories != null) {
                    async.eachSeries(stories.issues, function (story, callback) {
                            issuesList.push(story.key);
                            epicIssueMap[story.key] = moduleKey;
                            callback();
                        },
                        function (err) {
                            if (err) {
                                LogProgress("Restarting Loop for:"+story.key, err);
                            }
                            else {
                                loopError = false;
                            }
                            callback();
                        }
                    );
                }
                else {
                    loopError = false;
                    callback();
                }
            });
        },
        function(err) {
            LogProgress(moduleKey + " : " + " : Page Collected");
            callback();
        }
    );
}

function ProcessPageFromJira(jira, issueKey, counter, callback) {
    var loopError = true;
    async.whilst(function() {
            return loopError;
        },
        function(callback) {
            jira.findIssue(issueKey + "?expand=changelog", function (error, issue) {
                if (error) {
                    LogProgress("Collect pages error happened!", error);
                    LogProgress("Restarting Loop for: "+issueKey, error);
                }
                else if (issue != null) {
                    SavePage(jira, issue, function (error, dbPage) {
                        if (error) {
                            LogProgress("Restarting Loop for:"+issueKey, error);
                        }
                        ProcessLinkedIssues(issue, dbPage);
                        loopError = false;
                        callback();
                    });
                }
                else {
                    loopError = false;
                    callback();
                }
            });
        },
        function(err) {
            LogProgress(counter + ":" + issueKey + " : Page Collected");
            callback();
        }
    );
}

function ProcessLinkedIssues(issue, dbPage) {
    if(dbPage != null) {
        _.each(issue.fields.issuelinks, function (linkedIssueItem) {
            var linkedIssue = linkedIssueItem.inwardIssue ? linkedIssueItem.inwardIssue : linkedIssueItem.outwardIssue;
            if (linkedIssue.fields.issuetype.name != "Story") {
                if (_.isUndefined(linkedIssueUniqList[linkedIssue.key])) {
                    linkedIssueUniqList[linkedIssue.key] = {
                        linkedIssueKey: linkedIssue.key,
                        linkedPages: [
                            {
                                key: issue.key,
                                _id: dbPage._id,
                                linkType: linkedIssueItem.type.inward
                            }
                        ]
                    };
                } else {
                    linkedIssueUniqList[linkedIssue.key].linkedPages.push({
                        key: issue.key,
                        _id: dbPage._id,
                        linkType: linkedIssueItem.type.inward});
                }
            }
        });
    }
}

function ProcessBlockersFromJira(jira, linkedIssue, counter, callback) {
    var loopError = true;
    async.whilst(function() {
            return loopError;
        },
        function(callback) {
            jira.findIssue(linkedIssue.linkedIssueKey, function (error, jiraLinkedIssue) {
                if (error || jiraLinkedIssue == null) {
                    LogProgress("Collect issues error happened!", error);
                    LogProgress("Restarting Loop for:"+linkedIssue.linkedIssueKey, error);
                }
                else {
                    SaveLinkedIssue(jiraLinkedIssue, counter, callback);
                }
            });
        },
        function(err) {
            LogProgress(counter + ":" + linkedIssue.linkedIssueKey + " : Issue Collected");
            callback();
        }
    );
}

function SaveLinkedIssue(linkedIssue, counter, callback) {<commit_msg>fixes for reprocessing 502 error<commit_after>/**
 * Created by <NAME> (shladkou) on 2/26/14.
 */
var util = require('util');
var config = require('../config');
var log = require('../libs/log')(module);
var Module = require('../models/module').Module;
var Page = require('../models/page').Page;
var Issue = require('../models/issue').Issue;
var _ = require('underscore');
var async = require('async');

var JiraApi = require('jira').JiraApi;
var response = null;

var epicsList = [];
var issuesList = [];
var epicIssueMap = {};
var linkedIssueUniqList = [];
var updateInProgress = false;

exports.rememberResponse = function (res) {
    response = res;
    UpdateProgress(0, "page");
    UpdateProgress(0, "issues");
};

var UpdateProgress = function (progress, type) {
    if(response != null) {
        response.write("event: progress\n");
        response.write('data: {"' + type + '":' + progress.toString() + '}\n\n');
    }
    if (progress > 0) {
        LogProgress("**********" + type + " Progress " + progress.toString() + "% **********");
    }
};

var LogProgress = function (text, error) {
    if (response && error == null) {
        response.write("event: logmessage\n");
        response.write("data: " + text + "\n\n");
    }
    if (error) {
        log.error(text);
        log.error(error);
        if (response) {
            response.write("event: errmessage\n");
            var errorText = error == null ? "" : error.message;
            response.write("data: " + text + ", reason - " + errorText + "\n\n");
        }
    }
    else {
        log.info(text);
    }
};

exports.updateJiraInfo = function (full, jiraUser, jiraPassword, callback) {
    if (updateInProgress) {
        callback();
    }

    updateInProgress = true;

    LogProgress("**** Step 0: async processing");

    var jira = new JiraApi(config.get("jiraAPIProtocol"), config.get("jiraUrl"), config.get("jiraPort"), jiraUser, jiraPassword, '2');

    async.series([
        //step 1
        function (callback) {
            //grab all modules
            LogProgress("**** Step 1: collect modules");
            Step1CollectModules(jira, callback);
        },
        //step 2
        function (callback) {
            LogProgress("**** Step 2: collect pages");
            //grab pages list
            Step2CollectPages(jira, full, callback);
        },
        //step 3
        function (callback) {
            //process pages
            LogProgress("**** Step 3: process " + issuesList.length + " pages");
            Step3ProcessPages(jira, callback);
        },
        //step 4
        function (callback) {
            // process linked issues pages
            LogProgress("**** Step 4: process " + linkedIssueUniqList.length + " blockers");
            Step4ProcessBlockers(jira, callback);
        },
        //step 5
        function (callback) {
            LogProgress("**** Step 5: Update End");
            response.end();
            updateInProgress = false;
            callback();
        }
    ],
    //optional callback
    function(err) {
        if(err) {
            LogProgress("**** Update Failed ****", err);
        }
        else {
            LogProgress("**** Update Succeed ****");
        }
    }
    );
    callback();
};

function Step1CollectModules(jira, callback) {
    var requestString = "project = PLEX-UXC AND issuetype = epic AND summary ~ Module AND NOT summary ~ automation ORDER BY key ASC";
    epicsList = [];

    UpdateProgress(0, "page");
    UpdateProgress(0, "issues");

    var loopError = true;
    async.whilst(function() {
            return loopError;
        },
        function(callback) {
        jira.searchJira(requestString, { fields: ["summary", "duedate", "assignee"] }, function (error, epics) {
            if (error) {
                callback(error);
            }
            if (epics != null) {
                async.eachSeries(epics.issues, function (epic, callback) {
                        Module.findOne({ key: epic.key }, function (err, module) {
                            if (!module) {
                                module = new Module();
                            }
                            module.key = epic.key;
                            module.summary = epic.fields.summary;
                            module.duedate = epic.fields.duedate == null ? null : new Date(epic.fields.duedate);
                            module.assignee = epic.fields.assignee == null ? "Unassigned" : epic.fields.assignee.name;
                            module.save(function () {
                                epicsList.push(epic.key);
                                LogProgress(epic.key + " : " + " Module Collected");
                                callback();
                            })
                        });
                    },
                    function (err) {
                        if (err) {
                            LogProgress("Collect modules error happened!", err);
                        }
                        LogProgress(epicsList.length + " : " + " Modules Collected");
                        if(err == null) {
                            loopError = false;
                        }
                        callback(err);
                    });
            }
        });
    },
    function(err) {
        if(err) {
            LogProgress("Restarting Loop");
        }
        callback(err);
    });
}

function Step2CollectPages(jira, full, callback) {
    issuesList = [];
    epicIssueMap = {};

    async.eachLimit(epicsList, 10, function (epic, callback) {
            CollectPagesFromJira(jira, full, epic, callback);
        },
        function (err) {
            callback(err);
        }
    );
}

function Step3ProcessPages(jira, callback) {
    var counter = 0;
    var lastProgress = 0;
    linkedIssueUniqList = {};

    async.eachLimit(issuesList, 10, function (issueKey, callback) {
            var currentProgress = Math.floor((++counter * 100) / issuesList.length);
            if (lastProgress != currentProgress) {
                lastProgress = currentProgress;
                UpdateProgress(currentProgress, "page");
            }
            ProcessPageFromJira(jira, issueKey, counter, callback);
        },
        function (err) {
            callback(err);
        }
    );
}

function Step4ProcessBlockers(jira, callback) {
    var counter = 0;
    var lastProgress = 0;
    var keys = Object.keys(linkedIssueUniqList);
    async.eachLimit(keys, 10, function (linkedIssueKey, callback) {
            var currentProgress = Math.floor((++counter * 100) / keys.length);
            if (lastProgress != currentProgress) {
                lastProgress = currentProgress;
                UpdateProgress(currentProgress, "issues");
            }
            var linkedIssue = linkedIssueUniqList[linkedIssueKey];
            ProcessBlockersFromJira(jira, linkedIssue, counter, callback);
        },
        function (err) {
            callback(err);
        }
    );
}

function CollectPagesFromJira(jira, full, moduleKey, callback) {
    var queryString = full ?
        util.format("project = PLEXUXC AND issuetype = Story AND 'Epic Link' in (%s)", moduleKey) :
        util.format("project = PLEXUXC AND issuetype = Story AND 'Epic Link' in (%s) AND updated > -3d", moduleKey);

    var loopError = true;
    async.whilst(function() {
            return loopError;
        },
        function(callback) {
            LogProgress("**** collect pages for module: " + moduleKey);
            jira.searchJira(queryString, { fields: ["summary"] }, function (error, stories) {
                if (error) {
                    LogProgress("Collect pages error happened!", error);
                    LogProgress("Restarting Loop for: "+moduleKey, error);
                }
                if (stories != null) {
                    async.eachSeries(stories.issues, function (story, callback) {
                            issuesList.push(story.key);
                            epicIssueMap[story.key] = moduleKey;
                            callback();
                        },
                        function (err) {
                            if (err) {
                                LogProgress("Restarting Loop for:"+story.key, err);
                            }
                            else {
                                loopError = false;
                            }
                            callback();
                        }
                    );
                }
                else {
                    loopError = false;
                    callback();
                }
            });
        },
        function(err) {
            LogProgress(moduleKey + " : " + " : Page Collected");
            callback();
        }
    );
}

function ProcessPageFromJira(jira, issueKey, counter, callback) {
    var loopError = true;
    async.whilst(function() {
            return loopError;
        },
        function(callback) {
            jira.findIssue(issueKey + "?expand=changelog", function (error, issue) {
                if (error) {
                    LogProgress("Collect pages error happened!", error);
                    LogProgress("Restarting Loop for: "+issueKey, error);
                }
                else if (issue != null) {
                    SavePage(jira, issue, function (error, dbPage) {
                        if (error) {
                            LogProgress("Restarting Loop for:"+issueKey, error);
                        }
                        ProcessLinkedIssues(issue, dbPage);
                        loopError = false;
                        callback();
                    });
                }
                else {
                    loopError = false;
                    callback();
                }
            });
        },
        function(err) {
            LogProgress(counter + ":" + issueKey + " : Page Collected");
            callback();
        }
    );
}

function ProcessLinkedIssues(issue, dbPage) {
    if(dbPage != null) {
        _.each(issue.fields.issuelinks, function (linkedIssueItem) {
            var linkedIssue = linkedIssueItem.inwardIssue ? linkedIssueItem.inwardIssue : linkedIssueItem.outwardIssue;
            if (linkedIssue.fields.issuetype.name != "Story") {
                if (_.isUndefined(linkedIssueUniqList[linkedIssue.key])) {
                    linkedIssueUniqList[linkedIssue.key] = {
                        linkedIssueKey: linkedIssue.key,
                        linkedPages: [
                            {
                                key: issue.key,
                                _id: dbPage._id,
                                linkType: linkedIssueItem.type.inward
                            }
                        ]
                    };
                } else {
                    linkedIssueUniqList[linkedIssue.key].linkedPages.push({
                        key: issue.key,
                        _id: dbPage._id,
                        linkType: linkedIssueItem.type.inward});
                }
            }
        });
    }
}

function ProcessBlockersFromJira(jira, linkedIssue, counter, callback) {
    var loopError = true;
    async.whilst(function() {
            return loopError;
        },
        function(callback) {
            jira.findIssue(linkedIssue.linkedIssueKey, function (error, jiraLinkedIssue) {
                if (error || jiraLinkedIssue == null) {
                    LogProgress("Collect issues error happened!", error);
                    LogProgress("Restarting Loop for:"+linkedIssue.linkedIssueKey, error);
                    callback();
                }
                else {
                    loopError = false;
                    SaveLinkedIssue(jiraLinkedIssue, counter, callback);
                }
            });
        },
        function(err) {
            LogProgress(counter + ":" + linkedIssue.linkedIssueKey + " : Issue Collected");
            callback();
        }
    );
}

function SaveLinkedIssue(linkedIssue, counter, callback) {
{'fraction_non_alphanumeric': 0.12032041172076036, 'fraction_numerical': 0.004025898563957832, 'mean_word_length': 1.4524173027989822, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1330317', 'n_tokens_mistral': 6405, 'n_tokens_neox': 6023, 'n_words': 1309}