<commit_before>/**
 * Helper functions for handling blacklisted titles.
 */

'use strict';

const BLACKLIST = require('../../etc/feed/blacklist');
const escape = require('escape-string-regexp');
const HTTPError = require('../util').HTTPError;


/**
 * Articles with less than this proportion of pageviews on either desktop or
 * mobile are likely bot traffic and will be filtered from the output.
 *
 * Must be in the interval [0, 1].
 */
const BOT_FILTER_THRESHOLD = 0.1;

function isBlacklisted(title) {
    return BLACKLIST.indexOf(title) !== -1;
}

function filterSpecialPages(pages, mainPageTitle) {
    const mainPageRegExp = new RegExp(`^${escape(mainPageTitle)}$`, 'i');
    return pages.filter((page) => {
        return page.ns === 0 && !mainPageRegExp.test(page.title);
    });
}

/**
 * Implements an editor-proposed heuristic in which top-pageview articles with
 * almost all desktop pageviews or almost all mobile pageviews, rather than a
 * relatively even mix of the two, are presumed to be inflated by bot traffic
 * and not of sufficient human interest to include in the feed.
 */
function filterBotTraffic(allPlatformsMostRead, desktopMostRead) {

    if (BOT_FILTER_THRESHOLD < 0 || BOT_FILTER_THRESHOLD > 1) {
        throw new HTTPError({
            status: 500,
            type: 'internal_error',
            title: 'Internal error',
            detail: 'An internal error occurred'
        });
    }

    // Create an object with title keys for speedy desktop article lookups when
    // iterating over combined-platform top pageviews
    const desktopArticles = (function() {
        const result = {};
        desktopMostRead.forEach((entry) => {
            result[entry.article] = entry;
        });
        return result;
    }());
    return allPlatformsMostRead.filter((entry) => {
        const title = entry.article;
        const totalViews = entry.views;
        const desktopViews = desktopArticles[title] && desktopArticles[title].views;
        return (desktopViews
            && desktopViews >= totalViews * BOT_FILTER_THRESHOLD
            && desktopViews <= totalViews * (1 - BOT_FILTER_THRESHOLD))
            && !isBlacklisted(title);
    });
}

module.exports = {
    filterSpecialPages,
    filterBotTraffic
};<commit_msg>Hygiene: simplify most-read-filter.filterBotTraffic()<commit_after>/**
 * Helper functions for handling blacklisted titles.
 */

'use strict';

const BLACKLIST = require('../../etc/feed/blacklist');
const escape = require('escape-string-regexp');
const HTTPError = require('../util').HTTPError;


/**
 * Articles with less than this proportion of pageviews on either desktop or
 * mobile are likely bot traffic and will be filtered from the output.
 *
 * Must be in the interval [0, 1].
 */
const BOT_FILTER_THRESHOLD = 0.1;

function isBlacklisted(title) {
    return BLACKLIST.indexOf(title) !== -1;
}

function filterSpecialPages(pages, mainPageTitle) {
    const mainPageRegExp = new RegExp(`^${escape(mainPageTitle)}$`, 'i');
    return pages.filter((page) => {
        return page.ns === 0 && !mainPageRegExp.test(page.title);
    });
}

/**
 * Implements an editor-proposed heuristic in which top-pageview articles with
 * almost all desktop pageviews or almost all mobile pageviews, rather than a
 * relatively even mix of the two, are presumed to be inflated by bot traffic
 * and not of sufficient human interest to include in the feed.
 */
function filterBotTraffic(allPlatformsMostRead, desktopMostRead) {

    if (BOT_FILTER_THRESHOLD < 0 || BOT_FILTER_THRESHOLD > 1) {
        throw new HTTPError({
            status: 500,
            type: 'internal_error',
            title: 'Internal error',
            detail: 'An internal error occurred'
        });
    }

    // Create an object with title keys for speedy desktop article lookups when
    // iterating over combined-platform top pageviews
    const desktopArticles = {};
    desktopMostRead.forEach((entry) => {
        desktopArticles[entry.article] = entry;
    });
    return allPlatformsMostRead.filter((entry) => {
        const title = entry.article;
        const totalViews = entry.views;
        const desktopViews = desktopArticles[title] && desktopArticles[title].views;
        return (desktopViews >= totalViews * BOT_FILTER_THRESHOLD
            && desktopViews <= totalViews * (1 - BOT_FILTER_THRESHOLD))
            && !isBlacklisted(title);
    });
}

module.exports = {
    filterSpecialPages,
    filterBotTraffic
};
{'fraction_non_alphanumeric': 0.1096226836347399, 'fraction_numerical': 0.0053583389149363695, 'mean_word_length': 3.1404805914972274, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4412614', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1243, 'n_words': 380}