<commit_before>'use strict';

var _           = require('lodash'),
    v           = require('voca'),
    path        = require('path'),
    cheerio     = require('cheerio');

/**
 * default options
 * @type {Object}
 */
var defaultOptions = {
        moreRegExp: /\s*<!--\s*more\s*-->/i,
        stripTags: true,
        pruneLength: 140,
        pruneString: '…'
    };


/**
 * plugin expose function
 *
 * @param  {Object} options custom options
 * @return {Function}       plugin function
 */
module.exports = function(options) {
    options = _.merge({}, defaultOptions, options);
    return function(files, metalsmith, done) {
        setImmediate(done);
        _.forEach(files, function(file, filename) {
            if (!isHtml(filename)) {
                return;
            }
            var excerpt = file.excerpt ||
                            getExcerptByMoreTag(file, options.moreRegExp) ||
                            getExcerptByFirstParagraph(file) ||
                             '';
            if (options.stripTags) {
                excerpt = v.stripTags(excerpt);
                if (options.pruneLength > 0) {
                    excerpt = v.prune(excerpt, options.pruneLength, options.pruneString);
                }
            }
            file.excerpt = excerpt && excerpt.length > 0 ? excerpt : false;
        });
    };
};


/**
 * retrieve excerpt from file object by extracting contents until a 'more' tag
 *
 * @param  {Object} file   file object
 * @param  {RegExp} regExp 'more' tag regexp
 * @return {mixed}         excerpt string or false
 */
function getExcerptByMoreTag(file, regExp) {
    var excerpt = false,
        contents = file.contents.toString();
    contents = cheerio.load('<root>' + contents + '</root>')('root').html();
    var match = contents.search(regExp);
    if (match > -1) {
        excerpt = contents.slice(0, Buffer.byteLength(contents.slice(0, match)));
        excerpt = v.unescapeHtml(excerpt);
    }
    return excerpt;
}


/**
 * retrieve excerpt from file object by extracting the first p's contents
 *
 * @param  {Object} file file object
 * @return {mixed}       excerpt string or false
 */
function getExcerptByFirstParagraph(file) {
    var $ = cheerio.load(file.contents.toString()),
        p = $('p').first(),
        excerpt = p.length ? p.html().trim() : false;
    if (excerpt) {
        excerpt = v.unescapeHtml(excerpt);
    }
    return excerpt;
}

<commit_msg>Disable decodeEntities on cheerio load to work with UTF-8 characters<commit_after>'use strict';

var _           = require('lodash'),
    v           = require('voca'),
    path        = require('path'),
    cheerio     = require('cheerio');

/**
 * default options
 * @type {Object}
 */
var defaultOptions = {
        moreRegExp: /\s*<!--\s*more\s*-->/i,
        stripTags: true,
        pruneLength: 140,
        pruneString: '…'
    };


/**
 * plugin expose function
 *
 * @param  {Object} options custom options
 * @return {Function}       plugin function
 */
module.exports = function(options) {
    options = _.merge({}, defaultOptions, options);
    return function(files, metalsmith, done) {
        setImmediate(done);
        _.forEach(files, function(file, filename) {
            if (!isHtml(filename)) {
                return;
            }
            var excerpt = file.excerpt ||
                            getExcerptByMoreTag(file, options.moreRegExp) ||
                            getExcerptByFirstParagraph(file) ||
                             '';
            if (options.stripTags) {
                excerpt = v.stripTags(excerpt);
                if (options.pruneLength > 0) {
                    excerpt = v.prune(excerpt, options.pruneLength, options.pruneString);
                }
            }
            file.excerpt = excerpt && excerpt.length > 0 ? excerpt : false;
        });
    };
};


/**
 * retrieve excerpt from file object by extracting contents until a 'more' tag
 *
 * @param  {Object} file   file object
 * @param  {RegExp} regExp 'more' tag regexp
 * @return {mixed}         excerpt string or false
 */
function getExcerptByMoreTag(file, regExp) {
    var excerpt = false,
        contents = file.contents.toString();
    contents = cheerio.load('<root>' + contents + '</root>', {decodeEntities: false})('root').html();
    var match = contents.search(regExp);
    if (match > -1) {
        excerpt = contents.slice(0, match);
        excerpt = v.unescapeHtml(excerpt);
    }
    return excerpt;
}


/**
 * retrieve excerpt from file object by extracting the first p's contents
 *
 * @param  {Object} file file object
 * @return {mixed}       excerpt string or false
 */
function getExcerptByFirstParagraph(file) {
    var $ = cheerio.load(file.contents.toString(), {decodeEntities: false}),
        p = $('p').first(),
        excerpt = p.length ? p.html().trim() : false;
    if (excerpt) {
        excerpt = v.unescapeHtml(excerpt);
    }
    return excerpt;
}


{'fraction_non_alphanumeric': 0.13309134906231096, 'fraction_numerical': 0.003226456946965114, 'mean_word_length': 2.084577114427861, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1243993', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1418, 'n_words': 350}