<commit_before>/* eslint no-redeclare: "off", block-scoped-var: "off" */
const _ = require('lodash');
const esprima = require('esprima');
const JSDOM = require('jsdom').JSDOM;
const htmlMinifier = require('html-minifier');
const errors = require('../errors');
const Text = require('./Text');
const mozilla = require('source-map');

function extractEncodingFromText(text) {
    let metaCharset;
    for (const metaTagString of (text.match(/<meta[^>]+>/ig) || [])) {
        if (/\bhttp-equiv=([\"\']|)\s*Content-Type\s*\1/i.test(metaTagString)) {
            const matchContent = metaTagString.match(/\bcontent=([\"\']|)\s*text\/html;\s*charset=([\w\-]*)\s*\1/i);
            if (matchContent) {
                metaCharset = matchContent[2];
            }
        } else {
            const matchSimpleCharset = metaTagString.match(/\bcharset=([\"\']|)\s*([\w\-]*)\s*\1/i);
            if (matchSimpleCharset) {
                metaCharset = matchSimpleCharset[2];
            }
        }
    }
    return metaCharset; // Will be undefined if not found
}

class Html extends Text {
    init(config = {}) {
        super.init(config);
        if (typeof config.isFragment !== 'undefined') {
            this._isFragment = config.isFragment;
            config.isFragment = undefined;
        }
    }

    get encoding() {
        if (!this._encoding) {
            // An explicit encoding (Content-Type header, data: url charset, assetConfig) takes precedence, but if absent we should
            // look for a <meta http-equiv='Content-Type' ...> tag with a charset before falling back to the defaultEncoding (utf-8)
            if (typeof this._text === 'string') {
                this._encoding = extractEncodingFromText(this._text) || this.defaultEncoding;
            } else if (this._rawSrc) {
                this._encoding = extractEncodingFromText(this._rawSrc.toString('binary', 0, Math.min(1024, this._rawSrc.length))) || this.defaultEncoding;
            } else {
                this._encoding = this.defaultEncoding;
            }
        }
        return this._encoding;
    }

    set encoding(encoding) {
        // An intended side effect of getting this.parseTree before deleting this._rawSrc is that we're sure
        // that the raw source has been decoded into this._text before the original encoding is thrown away.
        const parseTree = this.parseTree;
        if (parseTree.head) {
            const existingMetaElements = parseTree.head.getElementsByTagName('meta');
            let contentTypeMetaElement;
            for (let i = 0 ; i < existingMetaElements.length ; i += 1) {
                const metaElement = existingMetaElements[i];
                if (metaElement.hasAttribute('charset') || /^content-type$/i.test(metaElement.getAttribute('http-equiv'))) {
                    contentTypeMetaElement = metaElement;
                    break;
                }
            }
            if (!contentTypeMetaElement) {
                contentTypeMetaElement = parseTree.createElement('meta');
                parseTree.head.insertBefore(contentTypeMetaElement, parseTree.head.firstChild);
                this.markDirty();
            }
            if (contentTypeMetaElement.hasAttribute('http-equiv')) {
                if ((contentTypeMetaElement.getAttribute('content') || '').toLowerCase() !== 'text/html; charset=' + encoding) {
                    contentTypeMetaElement.setAttribute('content', 'text/html; charset=' + encoding);
                    this.markDirty();
                }
            } else {
                // Simple <meta charset="...">
                if (contentTypeMetaElement.getAttribute('charset') !== encoding) {
                    contentTypeMetaElement.setAttribute('charset', encoding);
                    this.markDirty();
                }
            }
        }
        if (encoding !== this.encoding) {
            this._encoding = encoding;
            this.markDirty();
        }
    }

    unload() {
        super.unload();
        this._templateReplacements = {};
    }

    get text() {
        if (typeof this._text !== 'string') {
            if (this._parseTree) {
                this._text = this.isFragment ? this._parseTree.innerHTML : this._serialize();
                const templateReplacements = this._templateReplacements;
                this._text = Object.keys(templateReplacements).reduce(
                    (text, key) => text.replace(key, templateReplacements[key]),
                    this._text
                );
            } else {
                this._text = this._getTextFromRawSrc();
            }
            if (this.isMinified) {
                this._text = htmlMinifier.minify(this._text, _.defaults(this.htmlMinifierOptions || {}, {
                    // https://github.com/kangax/html-minifier#options-quick-reference
                    maxLineLength: Infinity,
                    collapseWhitespace: true,
                    collapseBooleanAttributes: true,
                    removeEmptyAttributes: true,
                    removeAttributeQuotes: true,
                    // useShortDoctype: true, // Replaces any DOCTYPE with the HTML5 one.
                    // removeOptionalTags: true, // Omits </head>, </body>, </html> etc. Too obtrusive?
                    removeComments: true,
                    removeScriptTypeAttributes: true,
                    removeStyleLinkTypeAttributes: true,
                    ignoreCustomComments: [
                        /^\s*\/?(?:ko|hz)(?:\s|$)/,
                        /^ASSETGRAPH DOCUMENT (?:START|END) MARKER$|^#|^\[if|^<!\[endif\]|^esi/
                    ]
                }));
            }
        }
        return this._text;
    }

    set text(text) {
        this.unload();

        this._text = text;
        if (this.assetGraph) {
            this.populate();
        }
        this.markDirty();
    }

    get parseTree() {
        if (!this._parseTree) {
            let text;
            if (typeof this._text === 'string') {
                text = this._text;
            } else {
                text = this._getTextFromRawSrc();
            }
            const templateReplacements = this._templateReplacements = {};
            text = text.replace(/<([%\?])[^\1]*?\1>/g, function (match, sub1, offset) {
                const key = '⋖' + offset + '⋗';
                templateReplacements[key] = match;
                return key;
            });

            const isFragment = this.isFragment;
            let document;
            try {
                const jsdom = new JSDOM(isFragment ? '<body>' + text + '</body>' : text, {
                    includeNodeLocations: true,
                    runScripts: 'outside-only'
                });
                document = jsdom.window.document;
                this._serialize = jsdom.serialize.bind(jsdom);
                this._nodeLocation = jsdom.nodeLocation.bind(jsdom);
            } catch (e) {
                const err = new errors.ParseError({message: 'Parse error in ' + this.urlOrDescription + '\n' + e.message, asset: this});
                if (this.assetGraph) {
                    this.assetGraph.warn(err);
                } else {
                    throw err;
                }
            }
            if (isFragment) {
                // Install the properties of document on the HtmlBodyElement used as the parse tree:
                for (const propertyName in document) {<commit_msg>Eliminate some console.log noise by providing a virtual console for all jsdom instance<commit_after>/* eslint no-redeclare: "off", block-scoped-var: "off" */
const _ = require('lodash');
const esprima = require('esprima');
const { JSDOM, VirtualConsole } = require('jsdom');
const virtualConsole = new VirtualConsole();
const htmlMinifier = require('html-minifier');
const errors = require('../errors');
const Text = require('./Text');
const mozilla = require('source-map');

function extractEncodingFromText(text) {
    let metaCharset;
    for (const metaTagString of (text.match(/<meta[^>]+>/ig) || [])) {
        if (/\bhttp-equiv=([\"\']|)\s*Content-Type\s*\1/i.test(metaTagString)) {
            const matchContent = metaTagString.match(/\bcontent=([\"\']|)\s*text\/html;\s*charset=([\w\-]*)\s*\1/i);
            if (matchContent) {
                metaCharset = matchContent[2];
            }
        } else {
            const matchSimpleCharset = metaTagString.match(/\bcharset=([\"\']|)\s*([\w\-]*)\s*\1/i);
            if (matchSimpleCharset) {
                metaCharset = matchSimpleCharset[2];
            }
        }
    }
    return metaCharset; // Will be undefined if not found
}

class Html extends Text {
    init(config = {}) {
        super.init(config);
        if (typeof config.isFragment !== 'undefined') {
            this._isFragment = config.isFragment;
            config.isFragment = undefined;
        }
    }

    get encoding() {
        if (!this._encoding) {
            // An explicit encoding (Content-Type header, data: url charset, assetConfig) takes precedence, but if absent we should
            // look for a <meta http-equiv='Content-Type' ...> tag with a charset before falling back to the defaultEncoding (utf-8)
            if (typeof this._text === 'string') {
                this._encoding = extractEncodingFromText(this._text) || this.defaultEncoding;
            } else if (this._rawSrc) {
                this._encoding = extractEncodingFromText(this._rawSrc.toString('binary', 0, Math.min(1024, this._rawSrc.length))) || this.defaultEncoding;
            } else {
                this._encoding = this.defaultEncoding;
            }
        }
        return this._encoding;
    }

    set encoding(encoding) {
        // An intended side effect of getting this.parseTree before deleting this._rawSrc is that we're sure
        // that the raw source has been decoded into this._text before the original encoding is thrown away.
        const parseTree = this.parseTree;
        if (parseTree.head) {
            const existingMetaElements = parseTree.head.getElementsByTagName('meta');
            let contentTypeMetaElement;
            for (let i = 0 ; i < existingMetaElements.length ; i += 1) {
                const metaElement = existingMetaElements[i];
                if (metaElement.hasAttribute('charset') || /^content-type$/i.test(metaElement.getAttribute('http-equiv'))) {
                    contentTypeMetaElement = metaElement;
                    break;
                }
            }
            if (!contentTypeMetaElement) {
                contentTypeMetaElement = parseTree.createElement('meta');
                parseTree.head.insertBefore(contentTypeMetaElement, parseTree.head.firstChild);
                this.markDirty();
            }
            if (contentTypeMetaElement.hasAttribute('http-equiv')) {
                if ((contentTypeMetaElement.getAttribute('content') || '').toLowerCase() !== 'text/html; charset=' + encoding) {
                    contentTypeMetaElement.setAttribute('content', 'text/html; charset=' + encoding);
                    this.markDirty();
                }
            } else {
                // Simple <meta charset="...">
                if (contentTypeMetaElement.getAttribute('charset') !== encoding) {
                    contentTypeMetaElement.setAttribute('charset', encoding);
                    this.markDirty();
                }
            }
        }
        if (encoding !== this.encoding) {
            this._encoding = encoding;
            this.markDirty();
        }
    }

    unload() {
        super.unload();
        this._templateReplacements = {};
    }

    get text() {
        if (typeof this._text !== 'string') {
            if (this._parseTree) {
                this._text = this.isFragment ? this._parseTree.innerHTML : this._serialize();
                const templateReplacements = this._templateReplacements;
                this._text = Object.keys(templateReplacements).reduce(
                    (text, key) => text.replace(key, templateReplacements[key]),
                    this._text
                );
            } else {
                this._text = this._getTextFromRawSrc();
            }
            if (this.isMinified) {
                this._text = htmlMinifier.minify(this._text, _.defaults(this.htmlMinifierOptions || {}, {
                    // https://github.com/kangax/html-minifier#options-quick-reference
                    maxLineLength: Infinity,
                    collapseWhitespace: true,
                    collapseBooleanAttributes: true,
                    removeEmptyAttributes: true,
                    removeAttributeQuotes: true,
                    // useShortDoctype: true, // Replaces any DOCTYPE with the HTML5 one.
                    // removeOptionalTags: true, // Omits </head>, </body>, </html> etc. Too obtrusive?
                    removeComments: true,
                    removeScriptTypeAttributes: true,
                    removeStyleLinkTypeAttributes: true,
                    ignoreCustomComments: [
                        /^\s*\/?(?:ko|hz)(?:\s|$)/,
                        /^ASSETGRAPH DOCUMENT (?:START|END) MARKER$|^#|^\[if|^<!\[endif\]|^esi/
                    ]
                }));
            }
        }
        return this._text;
    }

    set text(text) {
        this.unload();

        this._text = text;
        if (this.assetGraph) {
            this.populate();
        }
        this.markDirty();
    }

    get parseTree() {
        if (!this._parseTree) {
            let text;
            if (typeof this._text === 'string') {
                text = this._text;
            } else {
                text = this._getTextFromRawSrc();
            }
            const templateReplacements = this._templateReplacements = {};
            text = text.replace(/<([%\?])[^\1]*?\1>/g, function (match, sub1, offset) {
                const key = '⋖' + offset + '⋗';
                templateReplacements[key] = match;
                return key;
            });

            const isFragment = this.isFragment;
            let document;
            try {
                const jsdom = new JSDOM(isFragment ? '<body>' + text + '</body>' : text, {
                    virtualConsole,
                    includeNodeLocations: true,
                    runScripts: 'outside-only'
                });
                document = jsdom.window.document;
                this._serialize = jsdom.serialize.bind(jsdom);
                this._nodeLocation = jsdom.nodeLocation.bind(jsdom);
            } catch (e) {
                const err = new errors.ParseError({message: 'Parse error in ' + this.urlOrDescription + '\n' + e.message, asset: this});
                if (this.assetGraph) {
                    this.assetGraph.warn(err);
                } else {
                    throw err;
                }
            }
            if (isFragment) {
                // Install the properties of document on the HtmlBodyElement used as the parse tree:
                for (const propertyName in document) {
{'fraction_non_alphanumeric': 0.1242713301536831, 'fraction_numerical': 0.0022522522522522522, 'mean_word_length': 1.6644899399929403, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1156803', 'n_tokens_mistral': 4038, 'n_tokens_neox': 3820, 'n_words': 834}