<commit_before>"use strict";
var wowprogressAPI = process.require('api/wowProgress.js');
var logger = process.require("api/logger.js").get("logger");
var async = require('async');
var characterUpdateModel = process.require("models/characterUpdateModel.js");
var guildUpdateModel = process.require("models/guildUpdateModel.js");
var characterService = process.require("services/characterService.js");
var guildService = process.require("services/guildService.js");
var bnetAPI = process.require("api/bnet.js");

module.exports.parseWowProgress = function(callback){
    var self = this;
    async.waterfall([
        function(callback){
            wowprogressAPI.getAds(function(error,wowProgressGuildAds,wowProgressCharacterAds) {
                callback(error,wowProgressGuildAds,wowProgressCharacterAds);
            });
        },
        function(wowProgressGuildAds, wowProgressCharacterAds, callback){
            async.series([
                function(callback) {
                    async.eachSeries(wowProgressGuildAds, function (wowProgressGuildAd, callback) {
                        self.insertWoWProgressGuildAd(wowProgressGuildAd,function(error){
                            callback(error);
                        });
                    }, function (error) {
                        callback(error);
                    });
                },
                function(callback){
                    async.eachSeries(wowProgressCharacterAds,function(wowProgressCharacterAd,callback){
                        self.insertWoWProgressCharacterAd(wowProgressCharacterAd,function(error){
                            callback(error);
                        });
                    },function(error){
                        callback(error);
                    });
                }
            ],function(error){
                callback(error);
            });
        }
    ],function(error){
        callback(error);
    });
};

module.exports.insertWoWProgressGuildAd = function(wowProgressGuildAd,callback){
    async.waterfall([
        function (callback) {
            bnetAPI.getGuild(wowProgressGuildAd.region, wowProgressGuildAd.realm, wowProgressGuildAd.name, function (error, guild) {
                callback(error, guild);
            });
        },
        function (guild, callback) {
            //Force name with bnet response (case or russian realm name)
            wowProgressGuildAd.realm = guild.realm;
            wowProgressGuildAd.name = guild.name;
            guildService.get(wowProgressGuildAd.region, wowProgressGuildAd.realm, wowProgressGuildAd.name, function (error, guild) {
                callback(error, guild);
            });
        },
        function (guild, callback) {
            if (guild==null || (guild!=null && guild.ad==null))
                async.parallel([
                    function (callback) {
                        guildService.insertOrUpdateAd(wowProgressGuildAd.region, wowProgressGuildAd.realm, wowProgressGuildAd.name, 0, wowProgressGuildAd, function (error) {
                            callback(error);
                        });
                    },
                    function (callback) {
                        guildUpdateModel.insertOrUpdate(wowProgressGuildAd.region, wowProgressGuildAd.realm, wowProgressGuildAd.name, 10, function (error) {
                            if (!error)
                                logger.info("Insert guild to update " + wowProgressGuildAd.region + "-" + wowProgressGuildAd.realm + "-" + wowProgressGuildAd.name + " priority 10");
                            callback(error);
                        });
                    }
                ], function (error) {
                    callback(error)
                });
            else{
                logger.debug("Guild is already in datase " + wowProgressGuildAd.region + "-" + wowProgressGuildAd.realm + "-" + wowProgressGuildAd.name);
                callback();
            }
        }
    ], function (error) {
        callback(error);
    });
};

module.exports.insertWoWProgressCharacterAd = function(wowProgressCharacterAd,callback){
    async.waterfall([
        function(callback) {
            bnetAPI.getCharacter(wowProgressCharacterAd.region, wowProgressCharacterAd.realm, wowProgressCharacterAd.name, function (error, character) {
                callback(error,character);
            });
        },
        function(character,callback) {
            //Force name with bnet response (case or russian realm name)
            wowProgressCharacterAd.realm = character.realm;
            wowProgressCharacterAd.name = character.name;
            characterService.get(wowProgressCharacterAd.region, wowProgressCharacterAd.realm, wowProgressCharacterAd.name, function (error, character) {
                callback(error,character);
            });
        },
        function(character,callback) {
            if (!character || (character && !character.ad))
                async.parallel([
                    function (callback) {
                        characterService.insertOrUpdateAd(wowProgressCharacterAd.region, wowProgressCharacterAd.realm, wowProgressCharacterAd.name, 0, wowProgressCharacterAd, function (error) {
                            callback(error);
                        });
                    },
                    function (callback) {
                        characterUpdateModel.insertOrUpdate(wowProgressCharacterAd.region, wowProgressCharacterAd.realm, wowProgressCharacterAd.name, 10, function (error) {
                            if (!error)
                                logger.info("Insert character to update " + wowProgressCharacterAd.region + "-" + wowProgressCharacterAd.realm + "-" + wowProgressCharacterAd.name+" priority 10");
                            callback(error);
                        });
                    },
                    function (callback) {
                        if (wowProgressCharacterAd.guild)
                            guildUpdateModel.insertOrUpdate(wowProgressCharacterAd.region, wowProgressCharacterAd.realm, wowProgressCharacterAd.guild, 10, function (error) {
                                if (!error)
                                    logger.info("Insert guild to update " + wowProgressCharacterAd.region + "-" + wowProgressCharacterAd.realm + "-" + wowProgressCharacterAd.guild+" priority 10");
                                callback(error);
                            });
                        else
                            callback();
                    }
                ], function (error) {
                    callback(error);
                });<commit_msg>Correct bug on ad & wowprogress<commit_after>"use strict";
var wowprogressAPI = process.require('api/wowProgress.js');
var logger = process.require("api/logger.js").get("logger");
var async = require('async');
var characterUpdateModel = process.require("models/characterUpdateModel.js");
var guildUpdateModel = process.require("models/guildUpdateModel.js");
var characterService = process.require("services/characterService.js");
var guildService = process.require("services/guildService.js");
var bnetAPI = process.require("api/bnet.js");

module.exports.parseWowProgress = function(callback){
    var self = this;
    async.waterfall([
        function(callback){
            wowprogressAPI.getAds(function(error,wowProgressGuildAds,wowProgressCharacterAds) {
                callback(error,wowProgressGuildAds,wowProgressCharacterAds);
            });
        },
        function(wowProgressGuildAds, wowProgressCharacterAds, callback){
            async.series([
                function(callback) {
                    async.eachSeries(wowProgressGuildAds, function (wowProgressGuildAd, callback) {
                        self.insertWoWProgressGuildAd(wowProgressGuildAd,function(error){
                            callback(error);
                        });
                    }, function (error) {
                        callback(error);
                    });
                },
                function(callback){
                    async.eachSeries(wowProgressCharacterAds,function(wowProgressCharacterAd,callback){
                        self.insertWoWProgressCharacterAd(wowProgressCharacterAd,function(error){
                            callback(error);
                        });
                    },function(error){
                        callback(error);
                    });
                }
            ],function(error){
                callback(error);
            });
        }
    ],function(error){
        callback(error);
    });
};

module.exports.insertWoWProgressGuildAd = function(wowProgressGuildAd,callback){
    async.waterfall([
        function (callback) {
            bnetAPI.getGuild(wowProgressGuildAd.region, wowProgressGuildAd.realm, wowProgressGuildAd.name, function (error, guild) {
                callback(error, guild);
            });
        },
        function (guild, callback) {
            //Force name with bnet response (case or russian realm name)
            wowProgressGuildAd.realm = guild.realm;
            wowProgressGuildAd.name = guild.name;
            guildService.get(wowProgressGuildAd.region, wowProgressGuildAd.realm, wowProgressGuildAd.name, function (error, guild) {
                callback(error, guild);
            });
        },
        function (guild, callback) {
            if (!guild || (guild && !guild.ad) ||(guild && guild.ad && !guild.ad.updated))
                async.parallel([
                    function (callback) {
                        guildService.insertOrUpdateAd(wowProgressGuildAd.region, wowProgressGuildAd.realm, wowProgressGuildAd.name, 0, wowProgressGuildAd, function (error) {
                            callback(error);
                        });
                    },
                    function (callback) {
                        guildUpdateModel.insertOrUpdate(wowProgressGuildAd.region, wowProgressGuildAd.realm, wowProgressGuildAd.name, 10, function (error) {
                            if (!error)
                                logger.info("Insert guild to update " + wowProgressGuildAd.region + "-" + wowProgressGuildAd.realm + "-" + wowProgressGuildAd.name + " priority 10");
                            callback(error);
                        });
                    }
                ], function (error) {
                    callback(error)
                });
            else{
                logger.debug("Guild is already in datase " + wowProgressGuildAd.region + "-" + wowProgressGuildAd.realm + "-" + wowProgressGuildAd.name);
                callback();
            }
        }
    ], function (error) {
        callback(error);
    });
};

module.exports.insertWoWProgressCharacterAd = function(wowProgressCharacterAd,callback){
    async.waterfall([
        function(callback) {
            bnetAPI.getCharacter(wowProgressCharacterAd.region, wowProgressCharacterAd.realm, wowProgressCharacterAd.name, function (error, character) {
                callback(error,character);
            });
        },
        function(character,callback) {
            //Force name with bnet response (case or russian realm name)
            wowProgressCharacterAd.realm = character.realm;
            wowProgressCharacterAd.name = character.name;
            characterService.get(wowProgressCharacterAd.region, wowProgressCharacterAd.realm, wowProgressCharacterAd.name, function (error, character) {
                callback(error,character);
            });
        },
        function(character,callback) {
            if (!character || (character && !character.ad) || (character && character.ad && !character.ad.updated))
                async.parallel([
                    function (callback) {
                        characterService.insertOrUpdateAd(wowProgressCharacterAd.region, wowProgressCharacterAd.realm, wowProgressCharacterAd.name, 0, wowProgressCharacterAd, function (error) {
                            callback(error);
                        });
                    },
                    function (callback) {
                        characterUpdateModel.insertOrUpdate(wowProgressCharacterAd.region, wowProgressCharacterAd.realm, wowProgressCharacterAd.name, 10, function (error) {
                            if (!error)
                                logger.info("Insert character to update " + wowProgressCharacterAd.region + "-" + wowProgressCharacterAd.realm + "-" + wowProgressCharacterAd.name+" priority 10");
                            callback(error);
                        });
                    },
                    function (callback) {
                        if (wowProgressCharacterAd.guild)
                            guildUpdateModel.insertOrUpdate(wowProgressCharacterAd.region, wowProgressCharacterAd.realm, wowProgressCharacterAd.guild, 10, function (error) {
                                if (!error)
                                    logger.info("Insert guild to update " + wowProgressCharacterAd.region + "-" + wowProgressCharacterAd.realm + "-" + wowProgressCharacterAd.guild+" priority 10");
                                callback(error);
                            });
                        else
                            callback();
                    }
                ], function (error) {
                    callback(error);
                });
{'fraction_non_alphanumeric': 0.09911253635617868, 'fraction_numerical': 0.0020881497501677978, 'mean_word_length': 1.7440147329650093, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3618607', 'n_tokens_mistral': 3383, 'n_tokens_neox': 3150, 'n_words': 504}