<commit_before>/*
Copyright 2015, 2016 OpenMarket Ltd

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

'use strict';

var React = require('react');
var sanitizeHtml = require('sanitize-html');
var highlight = require('highlight.js');
var linkifyMatrix = require('./linkify-matrix');
import escape from 'lodash/escape';
import emojione from 'emojione';
import classNames from 'classnames';

emojione.imagePathSVG = 'emojione/svg/';
emojione.imageType = 'svg';

const EMOJI_REGEX = new RegExp(emojione.unicodeRegexp+"+", "gi");
const COLOR_REGEX = /^[#a-z0-9]+$/;
const ALLOWED_CSS = {
    "background-color": COLOR_REGEX,
    "color": COLOR_REGEX,
};

/* modified from https://github.com/Ranks/emojione/blob/master/lib/js/emojione.js
 * because we want to include emoji shortnames in title text
 */
export function unicodeToImage(str) {
    let replaceWith, unicode, alt;
    const mappedUnicode = emojione.mapUnicodeToShort();

    str = str.replace(emojione.regUnicode, function(unicodeChar) {
        if ( (typeof unicodeChar === 'undefined') || (unicodeChar === '') || (!(unicodeChar in emojione.jsEscapeMap)) ) {
            // if the unicodeChar doesnt exist just return the entire match
            return unicodeChar;
        }
        else {
            // get the unicode codepoint from the actual char
            unicode = emojione.jsEscapeMap[unicodeChar];

            // depending on the settings, we'll either add the native unicode as the alt tag, otherwise the shortname
            alt = (emojione.unicodeAlt) ? emojione.convert(unicode.toUpperCase()) : mappedUnicode[unicode];
            const title = mappedUnicode[unicode];

            replaceWith = `<img class="mx_emojione" title="${title}" alt="${alt}" src="${emojione.imagePathSVG}${unicode}.svg${emojione.cacheBustParam}"/>`;
            return replaceWith;
        }
    });

    return str;
}

export function stripParagraphs(html: string): string {
    const contentDiv = document.createElement('div');
    contentDiv.innerHTML = html;

    if (contentDiv.children.length === 0) {
        return contentDiv.innerHTML;
    }

    let contentHTML = "";
    for (let i=0; i<contentDiv.children.length; i++) {
        const element = contentDiv.children[i];
        if (element.tagName.toLowerCase() === 'p') {
            contentHTML += element.innerHTML + '<br />';
        } else {
            const temp = document.createElement('div');
            temp.appendChild(element.cloneNode(true));
            contentHTML += temp.innerHTML;
        }
    }

    return contentHTML;
}

var sanitizeHtmlParams = {
    allowedTags: [
        'font', // custom to matrix for IRC-style font coloring
        'del', // for markdown
        // deliberately no h1/h2 to stop people shouting.
        'h3', 'h4', 'h5', 'h6', 'blockquote', 'p', 'a', 'ul', 'ol',
        'nl', 'li', 'b', 'i', 'u', 'strong', 'em', 'strike', 'code', 'hr', 'br', 'div',
        'table', 'thead', 'caption', 'tbody', 'tr', 'th', 'td', 'pre', 'span',
    ],
    allowedAttributes: {
        // custom ones first:
        font: ['color', 'data-mx-bg-color', 'data-mx-color', 'style'], // custom to matrix
        span: ['data-mx-bg-color', 'data-mx-color', 'style'], // custom to matrix
        a: ['href', 'name', 'target', 'rel'], // remote target: custom to matrix
        // We don't currently allow img itself by default, but this
        // would make sense if we did
        img: ['src'],
    },
    // Lots of these won't come up by default because we don't allow them
    selfClosing: ['img', 'br', 'hr', 'area', 'base', 'basefont', 'input', 'link', 'meta'],
    // URL schemes we permit
    allowedSchemes: ['http', 'https', 'ftp', 'mailto'],

    // DO NOT USE. sanitize-html allows all URL starting with '//'
    // so this will always allow links to whatever scheme the
    // host page is served over.
    allowedSchemesByTag: {},

    transformTags: { // custom to matrix
        // add blank targets to all hyperlinks except vector URLs
        'a': function(tagName, attribs) {
            if (attribs.href) {
                attribs.target = '_blank'; // by default

                var m;
                // FIXME: horrible duplication with linkify-matrix
                m = attribs.href.match(linkifyMatrix.VECTOR_URL_PATTERN);
                if (m) {
                    attribs.href = m[1];
                    delete attribs.target;
                }

                m = attribs.href.match(linkifyMatrix.MATRIXTO_URL_PATTERN);
                if (m) {
                    var entity = m[1];
                    if (entity[0] === '@') {
                        attribs.href = '#/user/' + entity;
                    }
                    else if (entity[0] === '#' || entity[0] === '!') {
                        attribs.href = '#/room/' + entity;
                    }
                    delete attribs.target;
                }
            }
            attribs.rel = 'noopener'; // https://mathiasbynens.github.io/rel-noopener/
            return { tagName: tagName, attribs : attribs };
        },
        '*': function(tagName, attribs) {
            // Delete any style previously assigned
            delete attribs.style;

            // Sanitise and transform data-mx-color and data-mx-bg-color to their CSS
            // equivalents
            const customCSSMapper = {
                'data-mx-color': 'color',
                'data-mx-bg-color': 'background-color',
                // $customAttributeKey: $cssAttributeKey
            };

            let style = "";
            Object.keys(customCSSMapper).forEach((customAttributeKey) => {
                const cssAttributeKey = customCSSMapper[customAttributeKey];
                const customAttributeValue = attribs[customAttributeKey];
                if (customAttributeValue &&
                    typeof customAttributeValue === 'string' &&
                    /#[0-9a-fA-F]{6}/.test(customAttributeValue)
                ) {
                    style += cssAttributeKey + ":" + customAttributeValue + ";";
                }
            });

            if (style) {
                attribs.style = style;
            }

            return { tagName: tagName, attribs: attribs };
        },
    },
};

class BaseHighlighter {
    constructor(highlightClass, highlightLink) {
        this.highlightClass = highlightClass;
        this.highlightLink = highlightLink;
    }

    /**
     * apply the highlights to a section of text
     *
     * @param {string} safeSnippet The snippet of text to apply the highlights
     *     to.
     * @param {string[]} safeHighlights A list of substrings to highlight,
     *     sorted by descending length.<commit_msg>Remove custom attribs as consumed<commit_after>/*
Copyright 2015, 2016 OpenMarket Ltd

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

'use strict';

var React = require('react');
var sanitizeHtml = require('sanitize-html');
var highlight = require('highlight.js');
var linkifyMatrix = require('./linkify-matrix');
import escape from 'lodash/escape';
import emojione from 'emojione';
import classNames from 'classnames';

emojione.imagePathSVG = 'emojione/svg/';
emojione.imageType = 'svg';

const EMOJI_REGEX = new RegExp(emojione.unicodeRegexp+"+", "gi");
const COLOR_REGEX = /^[#a-z0-9]+$/;
const ALLOWED_CSS = {
    "background-color": COLOR_REGEX,
    "color": COLOR_REGEX,
};

/* modified from https://github.com/Ranks/emojione/blob/master/lib/js/emojione.js
 * because we want to include emoji shortnames in title text
 */
export function unicodeToImage(str) {
    let replaceWith, unicode, alt;
    const mappedUnicode = emojione.mapUnicodeToShort();

    str = str.replace(emojione.regUnicode, function(unicodeChar) {
        if ( (typeof unicodeChar === 'undefined') || (unicodeChar === '') || (!(unicodeChar in emojione.jsEscapeMap)) ) {
            // if the unicodeChar doesnt exist just return the entire match
            return unicodeChar;
        }
        else {
            // get the unicode codepoint from the actual char
            unicode = emojione.jsEscapeMap[unicodeChar];

            // depending on the settings, we'll either add the native unicode as the alt tag, otherwise the shortname
            alt = (emojione.unicodeAlt) ? emojione.convert(unicode.toUpperCase()) : mappedUnicode[unicode];
            const title = mappedUnicode[unicode];

            replaceWith = `<img class="mx_emojione" title="${title}" alt="${alt}" src="${emojione.imagePathSVG}${unicode}.svg${emojione.cacheBustParam}"/>`;
            return replaceWith;
        }
    });

    return str;
}

export function stripParagraphs(html: string): string {
    const contentDiv = document.createElement('div');
    contentDiv.innerHTML = html;

    if (contentDiv.children.length === 0) {
        return contentDiv.innerHTML;
    }

    let contentHTML = "";
    for (let i=0; i<contentDiv.children.length; i++) {
        const element = contentDiv.children[i];
        if (element.tagName.toLowerCase() === 'p') {
            contentHTML += element.innerHTML + '<br />';
        } else {
            const temp = document.createElement('div');
            temp.appendChild(element.cloneNode(true));
            contentHTML += temp.innerHTML;
        }
    }

    return contentHTML;
}

var sanitizeHtmlParams = {
    allowedTags: [
        'font', // custom to matrix for IRC-style font coloring
        'del', // for markdown
        // deliberately no h1/h2 to stop people shouting.
        'h3', 'h4', 'h5', 'h6', 'blockquote', 'p', 'a', 'ul', 'ol',
        'nl', 'li', 'b', 'i', 'u', 'strong', 'em', 'strike', 'code', 'hr', 'br', 'div',
        'table', 'thead', 'caption', 'tbody', 'tr', 'th', 'td', 'pre', 'span',
    ],
    allowedAttributes: {
        // custom ones first:
        font: ['color', 'data-mx-bg-color', 'data-mx-color', 'style'], // custom to matrix
        span: ['data-mx-bg-color', 'data-mx-color', 'style'], // custom to matrix
        a: ['href', 'name', 'target', 'rel'], // remote target: custom to matrix
        // We don't currently allow img itself by default, but this
        // would make sense if we did
        img: ['src'],
    },
    // Lots of these won't come up by default because we don't allow them
    selfClosing: ['img', 'br', 'hr', 'area', 'base', 'basefont', 'input', 'link', 'meta'],
    // URL schemes we permit
    allowedSchemes: ['http', 'https', 'ftp', 'mailto'],

    // DO NOT USE. sanitize-html allows all URL starting with '//'
    // so this will always allow links to whatever scheme the
    // host page is served over.
    allowedSchemesByTag: {},

    transformTags: { // custom to matrix
        // add blank targets to all hyperlinks except vector URLs
        'a': function(tagName, attribs) {
            if (attribs.href) {
                attribs.target = '_blank'; // by default

                var m;
                // FIXME: horrible duplication with linkify-matrix
                m = attribs.href.match(linkifyMatrix.VECTOR_URL_PATTERN);
                if (m) {
                    attribs.href = m[1];
                    delete attribs.target;
                }

                m = attribs.href.match(linkifyMatrix.MATRIXTO_URL_PATTERN);
                if (m) {
                    var entity = m[1];
                    if (entity[0] === '@') {
                        attribs.href = '#/user/' + entity;
                    }
                    else if (entity[0] === '#' || entity[0] === '!') {
                        attribs.href = '#/room/' + entity;
                    }
                    delete attribs.target;
                }
            }
            attribs.rel = 'noopener'; // https://mathiasbynens.github.io/rel-noopener/
            return { tagName: tagName, attribs : attribs };
        },
        '*': function(tagName, attribs) {
            // Delete any style previously assigned
            delete attribs.style;

            // Sanitise and transform data-mx-color and data-mx-bg-color to their CSS
            // equivalents
            const customCSSMapper = {
                'data-mx-color': 'color',
                'data-mx-bg-color': 'background-color',
                // $customAttributeKey: $cssAttributeKey
            };

            let style = "";
            Object.keys(customCSSMapper).forEach((customAttributeKey) => {
                const cssAttributeKey = customCSSMapper[customAttributeKey];
                const customAttributeValue = attribs[customAttributeKey];
                if (customAttributeValue &&
                    typeof customAttributeValue === 'string' &&
                    /#[0-9a-fA-F]{6}/.test(customAttributeValue)
                ) {
                    style += cssAttributeKey + ":" + customAttributeValue + ";";
                    delete attribs[customAttributeKey];
                }
            });

            if (style) {
                attribs.style = style;
            }

            return { tagName: tagName, attribs: attribs };
        },
    },
};

class BaseHighlighter {
    constructor(highlightClass, highlightLink) {
        this.highlightClass = highlightClass;
        this.highlightLink = highlightLink;
    }

    /**
     * apply the highlights to a section of text
     *
     * @param {string} safeSnippet The snippet of text to apply the highlights
     *     to.
     * @param {string[]} safeHighlights A list of substrings to highlight,
     *     sorted by descending length.
{'fraction_non_alphanumeric': 0.12728658536585366, 'fraction_numerical': 0.0041574279379157425, 'mean_word_length': 2.1561338289962824, 'pattern_counts': {'":': 6, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1019825', 'n_tokens_mistral': 4160, 'n_tokens_neox': 3970, 'n_words': 1155}