<commit_before>#!/usr/bin/env node
"use strict";

/**
 * Node.js : overwatch-js
 * Access overwatch heroes and profile informations
 *
 **/

const cheerio = require('cheerio');
const rp = require('request-promise');
const _ = require('lodash/core');

const url = 'https://playoverwatch.com/en-us/career/';
const searchUrl = 'https://playoverwatch.com/en-us/search/account-by-name/';

const GAMETYPES = ['competitive', 'quickplay'];
const PLATFORMS = {
    XboxLive: "xbl",
    Playstation: "psn",
    PC: "pc"
}

let OverwatchProvider = function () {
    var self = this;

    String.prototype.sanitize = function () {
        return this.trim().replace(" - ", "_").replace(/\s/g, "_").toLowerCase();
    }

    String.prototype.toTimestamp = function () {
        if (this.indexOf(':') > 0) {
            let intervals = this.split(':').reverse();
            intervals.at0 = (idx) => intervals[idx] || 0;
            // UTC date gives us milliseconds since the unix epoch
            return Date.UTC(1970, 0, intervals.at0(3) + 1, intervals.at0(2), intervals.at0(1), intervals.at0(0));
        }

        var swap = this;
        if (swap.endsWith('s')) swap = this.substr(0, this.length - 1); // remove trailing s
        if (swap.endsWith("second")) return parseInt(swap) * 1000;
        if (swap.endsWith("minute")) return parseInt(swap) * 60000;
        if (swap.endsWith("hour")) return parseInt(swap) * 3600000;
        if (swap.endsWith("day")) return parseInt(swap) * 86400000;

        return parseInt(swap);
    }

    String.prototype.cast = function () {
        if (this.indexOf('.') > 0) return parseFloat(this);
        if (this.indexOf(':') > 0 || this.split(' ').length > 1) return this.toTimestamp();
        return parseInt(this.replace(/,/g, ''));
    }

    let getUrl = (platform, region, tag) => {

        switch (platform) {
            case PLATFORMS.PC:
                region = "/" + region
                break;
            case PLATFORMS.Playstation:
            case PLATFORMS.XboxLive:
            default:
                //// No region must be specified
                region = "";
                break;
        }

        return url + `${platform}${region}/${tag}`;
    };

    let getSearchUrl = (nickname) => {
        return searchUrl + nickname;
    };

    let parseProfile = ($) => {
        var stats = {};
        stats.nick = $('.header-masthead').text();
        stats.level = parseInt($('div.player-level div').first().text());
        stats.avatar = $('.player-portrait').attr('src');
        stats.rank = parseInt($('div.competitive-rank > div').first().text());

        if (stats.rank)
            stats.rankPicture = $('div.competitive-rank > img').attr('src');

        stats.platform = $('#profile-platforms > a').text();

        return stats;
    };

    let parseFeaturedStats = ($, gameType) => {
        var stats = {};
        _.each($(`#${gameType} > section.highlights-section div.card-content`), (item) => {
            var item = $(item);
            stats[item.find('.card-copy').text().sanitize()] = item.find('.card-heading').text().cast();
        });

        return stats;
    }

    let parseHeroesStats = ($, gametype, overallOnly = false) => {
        var heroesMap = [];
        var stats = {};
        _.each($(`#${gametype} > .career-stats-section option`), (item) => {
            heroesMap.push({ name: item.attribs['option-id'].toLowerCase().sanitize(), value: item.attribs['value'] });

            if (overallOnly)
                return false;
        });

        //// Seeking heroe datas
        _.each(heroesMap, (map) => {
            stats[map.name] = {};

            _.each($(`#${gametype} [data-category-id="${map.value}"]`), (slide) => {
                var e = $(slide);
                _.each(e.find('tbody > tr'), (stat) => {
                    stats[map.name][stat.children[0].children[0].data.sanitize()] = stat.children[1].children[0].data.cast();
                });
            });

            if (overallOnly)
                return false;
        });

        return stats;
    }

    let parseAchievements = ($) => {
        var categories = [];
        var stats = {};
        _.each($(`select[data-group-id="achievements"] option`), (item) => {
            categories.push({ name: item.attribs['option-id'].toLowerCase(), value: item.attribs['value'] });
        });

        // Seeking achievements stats
        return _.map(categories, (category) => {
            var ctn  = $(`[data-category-id="${category.value}"]`);
            return {    
                acquired : ctn.find('.achievement-card').attr('class').indexOf('m-disabled') < 0,
                thumbnail : ctn.find('.media-card-fill').attr('src'),
                title : ctn.find('.tooltip-tip > .h5').text(),
                description : ctn.find('.tooltip-tip > .h6').text()
            }
        });
    }

    let handle = (err) => {
        console.log(err);
        switch (err.response.statusCode) {
            case 404:
                throw new Error('PROFILE_NOT_FOUND');
                break;
            case 500:
                throw new Error('TECHNICAL_EXCEPTION_HTML_STRUCTURE_MAY_HAVE_CHANGED')
                break;
            default:
                throw new Error('TECHNICAL_EXCEPTION_NOT_IDENTIFIED')
                break;
        }
    }

    self.getOverall = (platform, region, tag) => {
        return self.getAll(platform, region, tag, true);
    }

    self.getAll = (platform, region, tag, overallOnly) => {
        var baseurl = getUrl(platform, region, tag);
        return rp(baseurl).then((context) => {

            var result = {};
            var promises = [];
            const $ = cheerio.load(context);<commit_msg>Fix : Achievements list depth<commit_after>#!/usr/bin/env node
"use strict";

/**
 * Node.js : overwatch-js
 * Access overwatch heroes and profile informations
 *
 **/

const cheerio = require('cheerio');
const rp = require('request-promise');
const _ = require('lodash/core');

const url = 'https://playoverwatch.com/en-us/career/';
const searchUrl = 'https://playoverwatch.com/en-us/search/account-by-name/';

const GAMETYPES = ['competitive', 'quickplay'];
const PLATFORMS = {
    XboxLive: "xbl",
    Playstation: "psn",
    PC: "pc"
}

let OverwatchProvider = function () {
    var self = this;

    String.prototype.sanitize = function () {
        return this.trim().replace(" - ", "_").replace(/\s/g, "_").toLowerCase();
    }

    String.prototype.toTimestamp = function () {
        if (this.indexOf(':') > 0) {
            let intervals = this.split(':').reverse();
            intervals.at0 = (idx) => intervals[idx] || 0;
            // UTC date gives us milliseconds since the unix epoch
            return Date.UTC(1970, 0, intervals.at0(3) + 1, intervals.at0(2), intervals.at0(1), intervals.at0(0));
        }

        var swap = this;
        if (swap.endsWith('s')) swap = this.substr(0, this.length - 1); // remove trailing s
        if (swap.endsWith("second")) return parseInt(swap) * 1000;
        if (swap.endsWith("minute")) return parseInt(swap) * 60000;
        if (swap.endsWith("hour")) return parseInt(swap) * 3600000;
        if (swap.endsWith("day")) return parseInt(swap) * 86400000;

        return parseInt(swap);
    }

    String.prototype.cast = function () {
        if (this.indexOf('.') > 0) return parseFloat(this);
        if (this.indexOf(':') > 0 || this.split(' ').length > 1) return this.toTimestamp();
        return parseInt(this.replace(/,/g, ''));
    }

    let getUrl = (platform, region, tag) => {

        switch (platform) {
            case PLATFORMS.PC:
                region = "/" + region
                break;
            case PLATFORMS.Playstation:
            case PLATFORMS.XboxLive:
            default:
                //// No region must be specified
                region = "";
                break;
        }

        return url + `${platform}${region}/${tag}`;
    };

    let getSearchUrl = (nickname) => {
        return searchUrl + nickname;
    };

    let parseProfile = ($) => {
        var stats = {};
        stats.nick = $('.header-masthead').text();
        stats.level = parseInt($('div.player-level div').first().text());
        stats.avatar = $('.player-portrait').attr('src');
        stats.rank = parseInt($('div.competitive-rank > div').first().text());

        if (stats.rank)
            stats.rankPicture = $('div.competitive-rank > img').attr('src');

        stats.platform = $('#profile-platforms > a').text();

        return stats;
    };

    let parseFeaturedStats = ($, gameType) => {
        var stats = {};
        _.each($(`#${gameType} > section.highlights-section div.card-content`), (item) => {
            var item = $(item);
            stats[item.find('.card-copy').text().sanitize()] = item.find('.card-heading').text().cast();
        });

        return stats;
    }

    let parseHeroesStats = ($, gametype, overallOnly = false) => {
        var heroesMap = [];
        var stats = {};
        _.each($(`#${gametype} > .career-stats-section option`), (item) => {
            heroesMap.push({ name: item.attribs['option-id'].toLowerCase().sanitize(), value: item.attribs['value'] });

            if (overallOnly)
                return false;
        });

        //// Seeking heroe datas
        _.each(heroesMap, (map) => {
            stats[map.name] = {};

            _.each($(`#${gametype} [data-category-id="${map.value}"]`), (slide) => {
                var e = $(slide);
                _.each(e.find('tbody > tr'), (stat) => {
                    stats[map.name][stat.children[0].children[0].data.sanitize()] = stat.children[1].children[0].data.cast();
                });
            });

            if (overallOnly)
                return false;
        });

        return stats;
    }

    let parseAchievements = ($) => {
        var categories = [];
        var stats = {};
        var achievements = [];
        _.each($(`select[data-group-id="achievements"] option`), (item) => {
            categories.push({ name: item.attribs['option-id'].toLowerCase(), value: item.attribs['value'] });
        });

        // Seeking achievements stats
        _.each(categories, (category) => {
            var ctns  = $(`[data-category-id="${category.value}"] > ul > div`);

            _.each(ctns, (ctn) => {
                var ctn = $(ctn);
                achievements.push({    
                    acquired : ctn.find('.achievement-card').attr('class').indexOf('m-disabled') < 0,
                    thumbnail : ctn.find('.media-card-fill').attr('src'),
                    title : ctn.find('.tooltip-tip > .h5').text(),
                    description : ctn.find('.tooltip-tip > .h6').text(),
                    category : category.name
                })
            });
        });

        return achievements;
    }

    let handle = (err) => {
        console.log(err);
        switch (err.response.statusCode) {
            case 404:
                throw new Error('PROFILE_NOT_FOUND');
                break;
            case 500:
                throw new Error('TECHNICAL_EXCEPTION_HTML_STRUCTURE_MAY_HAVE_CHANGED')
                break;
            default:
                throw new Error('TECHNICAL_EXCEPTION_NOT_IDENTIFIED')
                break;
        }
    }

    self.getOverall = (platform, region, tag) => {
        return self.getAll(platform, region, tag, true);
    }

    self.getAll = (platform, region, tag, overallOnly) => {
        var baseurl = getUrl(platform, region, tag);
        return rp(baseurl).then((context) => {

            var result = {};
            var promises = [];
            const $ = cheerio.load(context);
{'fraction_non_alphanumeric': 0.17095619567070053, 'fraction_numerical': 0.010056246804158855, 'mean_word_length': 1.954431017119839, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 66, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3464323', 'n_tokens_mistral': 3597, 'n_tokens_neox': 3442, 'n_words': 680}