<commit_before>// expose to the world
module.exports.stripHTML = stripHTML;

/**
 * <p>Converts a HTML stringo into plaintext format that resembles Markdown</p>
 *
 * <p>Only good for simple and valid HTML, probably breaks on eveything else</p>
 *
 * <p>Placeholders:</p>
 *
 * <ul>
 *     <li>-\u0000\u0000- for newline</li>
 *     <li>-\u0001\u0001- for a space</li>
 *     <li>-\u0002\u0002- temporary newlines</li>
 * </ul>
 *
 * @param {String} str HTML string to convert
 * @return {String} Plaintext that resembles Markdown
 */
function stripHTML(str){
    str = (str || "").toString("utf-8").trim();

    // remove head
    str = str.replace(/<head[\s\S]{1,}?\/head>/gi, '');

    // replace newlines
    str = str.replace(/\r?\n|\r/g,"-\u0002\u0002-");

    // convert block element endings to linebreak markers
    str = str.replace(/<(?:\/p|br|\/tr|\/table|\/div)>/g,"-\u0000\u0000--\u0000\u0000-");

    // H1-H6, add underline or prepend with #
    str = str.replace(/<[hH](\d)[^>]*>(.*?)<\/[hH]\d[^>]*>/g,function(match, level, content){
        var line = "",
            symbol, // line symbol char
            len;
<commit_msg>Translate self closing break tags to newline<commit_after>// expose to the world
module.exports.stripHTML = stripHTML;

/**
 * <p>Converts a HTML stringo into plaintext format that resembles Markdown</p>
 *
 * <p>Only good for simple and valid HTML, probably breaks on eveything else</p>
 *
 * <p>Placeholders:</p>
 *
 * <ul>
 *     <li>-\u0000\u0000- for newline</li>
 *     <li>-\u0001\u0001- for a space</li>
 *     <li>-\u0002\u0002- temporary newlines</li>
 * </ul>
 *
 * @param {String} str HTML string to convert
 * @return {String} Plaintext that resembles Markdown
 */
function stripHTML(str){
    str = (str || "").toString("utf-8").trim();

    // remove head
    str = str.replace(/<head[\s\S]{1,}?\/head>/gi, '');

    // replace newlines
    str = str.replace(/\r?\n|\r/g,"-\u0002\u0002-");

    // convert block element endings to linebreak markers
    str = str.replace(/<(?:\/p|br\s*\/*|\/tr|\/table|\/div)>/g,"-\u0000\u0000--\u0000\u0000-");

    // H1-H6, add underline or prepend with #
    str = str.replace(/<[hH](\d)[^>]*>(.*?)<\/[hH]\d[^>]*>/g,function(match, level, content){
        var line = "",
            symbol, // line symbol char
            len;

{'fraction_non_alphanumeric': 0.20774193548387096, 'fraction_numerical': 0.044731182795698925, 'mean_word_length': 2.7038216560509554, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6908959', 'n_tokens_mistral': 915, 'n_tokens_neox': 814, 'n_words': 194}