<commit_before>'use strict';

function decodeEntities(encodedString) {
    const translate_re = /&(nbsp|amp|quot|lt|gt);/g;
    const translate = {
        nbsp: ' ',
        amp: '&',
        quot: '"',
        lt: '<',
        gt: '>'
    };
    return encodedString.replace(translate_re, function(match, entity) {
        return translate[entity];
    }).replace(/&#(\d+);/gi, function(match, numStr) {
        const num = parseInt(numStr, 10);
        return String.fromCharCode(num);
    });
}

module.exports = {<commit_msg>Fix hex HTML entities<commit_after>'use strict';

function decodeEntities(encodedString) {
    const translate_re = /&(nbsp|amp|quot|lt|gt);/g;
    const translate = {
        nbsp: ' ',
        amp: '&',
        quot: '"',
        lt: '<',
        gt: '>'
    };
    return encodedString.replace(translate_re, function(match, entity) {
        return translate[entity];
    }).replace(/&#(x)?(\d+);/gi, function(match, isHex, numStr) {
        const num = parseInt(numStr, isHex ? 16 : 10);
        return String.fromCharCode(num);
    });
}

module.exports = {
{'fraction_non_alphanumeric': 0.17399267399267399, 'fraction_numerical': 0.005494505494505495, 'mean_word_length': 2.6312292358803986, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5673452', 'n_tokens_mistral': 363, 'n_tokens_neox': 350, 'n_words': 66}