<commit_before>import TurndownService from 'turndown';

import identity from '@proton/utils/identity';

const EMPTY_LINE_KEEPER = '%%%PROTON-EMPTY-LINE%%%';

/**
 * Transform HTML to text
 * Append lines before the content if it starts with a Signature
 */
export const toText = (html: string, convertImages = false) => {
    const turndownService = new TurndownService({
        bulletListMarker: '-',
        strongDelimiter: '' as any,
        emDelimiter: '' as any,
        hr: '',
    });

    const replaceBreakLine = {
        filter: 'br',
        replacement(content: string, node: HTMLElement) {
            // ex <li>monique<br></li>
            if (node.parentElement?.lastChild === node && node.parentElement.textContent) {
                return node.parentElement.nodeName !== 'LI' ? '\n' : '';
            }

            // Turndown remove multiple empty lines by default
            // So we use special anchors on each line break to for keeping all lines
            return `${EMPTY_LINE_KEEPER}\n`;
        },
    } as TurndownService.Rule;

    const replaceImg = {
        filter: 'img',
        replacement(content: string, element: HTMLElement) {
            if (!convertImages) {
                return '';
            }

            const image = element as HTMLImageElement;

            // needed for the automatic conversion done by pgp/inline, otherwise the conversion happens and people forget that they have selected this for some contacts
            const attribute = image.alt || image.src;
            return attribute ? `[${attribute}]` : '';
        },
    } as TurndownService.Rule;

    const replaceAnchor = {
        filter: 'a',
        replacement(content: string, node: HTMLElement) {
            return node.textContent;
        },
    } as TurndownService.Rule;

    const replaceDiv = {
        filter: ['div'],
        replacement(content: string, node: HTMLElement) {
            if (node.children.length > 0 && (node.lastChild?.nodeName === 'BR' || node.lastChild?.nodeName === 'DIV')) {
                return content;
            }

            return `${content}\n`;
        },
    } as TurndownService.Rule;

    turndownService.use([
        () => turndownService.addRule('replaceAnchor', replaceAnchor),
        () => turndownService.addRule('replaceDiv', replaceDiv),
        () => turndownService.addRule('replaceImg', replaceImg),
        () => turndownService.addRule('replaceBreakLine', replaceBreakLine),
    ]);

    /**
     * Override turndown to NOT escape any HTML. For example MONO_TLS_PROVIDER -> MONO\_TLS\_PROVIDER.
     * Just return the value that is passed in.
     * Fixes https://github.com/ProtonMail/Angular/issues/6556
     */
    turndownService.escape = identity;

    let output = turndownService.turndown(html);

    // Remove anchors to meant to keep empty lines
    const emptyLineRegex = new RegExp(EMPTY_LINE_KEEPER, 'gi');
    output = output.replace(emptyLineRegex, '');

    return output;
};
<commit_msg>Remove hidden tags from parsed HTML<commit_after>import TurndownService from 'turndown';

import identity from '@proton/utils/identity';

const EMPTY_LINE_KEEPER = '%%%PROTON-EMPTY-LINE%%%';

/**
 * Transform HTML to text
 * Append lines before the content if it starts with a Signature
 */
export const toText = (html: string, convertImages = false) => {
    const turndownService = new TurndownService({
        bulletListMarker: '-',
        strongDelimiter: '' as any,
        emDelimiter: '' as any,
        hr: '',
    });

    const replaceBreakLine = {
        filter: 'br',
        replacement(content: string, node: HTMLElement) {
            // ex <li>monique<br></li>
            if (node.parentElement?.lastChild === node && node.parentElement.textContent) {
                return node.parentElement.nodeName !== 'LI' ? '\n' : '';
            }

            // Turndown remove multiple empty lines by default
            // So we use special anchors on each line break to for keeping all lines
            return `${EMPTY_LINE_KEEPER}\n`;
        },
    } as TurndownService.Rule;

    const replaceImg = {
        filter: 'img',
        replacement(content: string, element: HTMLElement) {
            if (!convertImages) {
                return '';
            }

            const image = element as HTMLImageElement;

            // needed for the automatic conversion done by pgp/inline, otherwise the conversion happens and people forget that they have selected this for some contacts
            const attribute = image.alt || image.src;
            return attribute ? `[${attribute}]` : '';
        },
    } as TurndownService.Rule;

    const replaceAnchor = {
        filter: 'a',
        replacement(content: string, node: HTMLElement) {
            return node.textContent;
        },
    } as TurndownService.Rule;

    const replaceDiv = {
        filter: ['div'],
        replacement(content: string, node: HTMLElement) {
            if (node.children.length > 0 && (node.lastChild?.nodeName === 'BR' || node.lastChild?.nodeName === 'DIV')) {
                return content;
            }

            return `${content}\n`;
        },
    } as TurndownService.Rule;

    const removeHiddenTags = {
        filter: (node: HTMLElement) => node.style.display === 'none' || node.style.visibility === 'hidden',
        replacement: () => '',
    } as TurndownService.Rule;

    turndownService.use([
        () => turndownService.addRule('replaceAnchor', replaceAnchor),
        () => turndownService.addRule('replaceDiv', replaceDiv),
        () => turndownService.addRule('replaceImg', replaceImg),
        () => turndownService.addRule('replaceBreakLine', replaceBreakLine),
        () => turndownService.addRule('removeHiddenTags', removeHiddenTags),
    ]);

    /**
     * Override turndown to NOT escape any HTML. For example MONO_TLS_PROVIDER -> MONO\_TLS\_PROVIDER.
     * Just return the value that is passed in.
     * Fixes https://github.com/ProtonMail/Angular/issues/6556
     */
    turndownService.escape = identity;

    let output = turndownService.turndown(html);

    // Remove anchors to meant to keep empty lines
    const emptyLineRegex = new RegExp(EMPTY_LINE_KEEPER, 'gi');
    output = output.replace(emptyLineRegex, '');

    return output;
};

{'fraction_non_alphanumeric': 0.11997457492451931, 'fraction_numerical': 0.0015890672175433022, 'mean_word_length': 2.418794133623031, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 26, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5620803', 'n_tokens_mistral': 1761, 'n_tokens_neox': 1702, 'n_words': 461}