<commit_before>define(["vendor/mithril"], function(mDiscard) {

    "use strict"
    
    // This constructs a nested Mithril object with only specific HTML tags allowed
    // No attributes are allowed.
    // A css class (from a short approved list) can be set on a tag using a ".className" after the opening tag name.
    // For example: <span.narrafirma-special-warning>Warning!!!<span>
    
    const debugLogging = false
    
    // 1 is normal tag that needs to be closed; 2 is self-closing tag (br and hr)
    const allowedHTMLTags = {
        // a: 1,
        address: 1,
        article: 1,
        b: 1,
        big: 1,
        blockquote: 1,
        br: 2,
        caption: 1,
        cite: 1,
        code: 1,
        del: 1,
        div: 1,
        dd: 1,
        d1: 1,
        dt: 1,
        em: 1,
        h1: 1,
        h2: 1,
        h3: 1,
        h4: 1,
        h5: 1,
        h6: 1,
        hr: 2,
        i: 1,
        // img
        kbd: 1,
        li: 1,
        ol: 1,
        p: 1,
        pre: 1,
        s: 1,
        small: 1,
        span: 1,
        sup: 1,
        sub: 1,
        strong: 1,
        strike: 1,
        table: 1,
        td: 1,
        th: 1,
        tr: 1,
        u: 1,
        ul: 1
    }
    
    const allowedCSSClasses = {
        "narrafirma-special-warning": 1
    }
    
    function isURLAcceptable(url, configuration) {
        if (!url) return ""
        if (url.substring(0, 5) === "http:" || url.substring(0, 6) === "https:") return url
        if (configuration.baseURL) return configuration.baseURL + url
        return ""
    }
    
    function isTagAllowed(tagName, configuration) {
        if (allowedHTMLTags[tagName]) return true
        
        if (!configuration) configuration = {}
        
        if (tagName === "a" && configuration.allowLinks) return true
        if (tagName === "img" && configuration.allowImages) return true
        return false
    }
    
    function generateVDOM(nodes, configuration) {
        if (!nodes) return []
        // console.log("generateVDOM nodes length", nodes.length)
      
        const result = []
        
        for (let i = 0; i < nodes.length; i++) {
            const node = nodes[i]
            // console.log("nodeType", node.nodeType)
    
            switch (node.nodeType) {
            case 1:
                let tagName = node.tagName.toLowerCase()
                
                // console.log("element", node)
                if (!isTagAllowed(tagName, configuration)) {
                    if (debugLogging) console.log("disallowed tag", tagName)
                    tagName = "span"
                }
                
                // TODO: Allow more attributes maybe
                let attributes = {}
                for (let j = 0; j < node.attributes.length; j++) {
                    const attribute = node.attributes[j]
                    if (attribute.name === "class") {
                        const theClassOrClasses = attribute.value
                        if (theClassOrClasses in allowedCSSClasses) {
                            attributes["class"] = theClassOrClasses
                        } else {
                            if (debugLogging) console.log("WARN: CSS class not allowed", theClassOrClasses)
                        }
                    }
                    if (configuration.allowLinks && attribute.name === "href") {
                        let url = attribute.value
                        if (url && url.substring(0, 2) === "//") {
                            url = window.location.protocol + url
                        }
                        const usableURL = isURLAcceptable(url, configuration)
                        if (usableURL) {
                            attributes["href"] = usableURL
                        }
                    }
                    if (configuration.allowImages && attribute.name === "src") {
                        let url = attribute.value
                        if (url && url.substring(0, 2) === "//") {
                            url = window.location.protocol + url
                        }
                        const usableURL = isURLAcceptable(url, configuration)
                        if (usableURL) {
                            attributes["src"] = usableURL
                        }
                    }
                    if (configuration.allowImages && (attribute.name === "width" || attribute.name === "height")) {
                        const size = parseInt(attribute.value)
                        if (!isNaN(size) && size > 1) {
                            attributes[attribute.name] = size
                        } else {
                            // Tracking image
                            console.log("discarding likely tracking image")
                            tagName = "span"
                            attributes = {}
                            break
                        }
                    }                    
                }
                
                if (tagName === "a") {
                    attributes["rel"] = "nofollow"
                }
                
                if (tagName === "img") {
                    if (!attributes["width"] || !attributes["height"]) {
                        console.log("discarding likely tracking image (2)", attributes["src"])
                        tagName = "span"
                        attributes = {}
                    }
                }
                
                const children = generateVDOM(node.childNodes, configuration)
                const vdom = m(tagName, attributes, children)
                result.push(vdom)
                break
            case 3:
                if (node.data) {
                    // console.log("adding text node", node.data)
                    result.push(node.data)
                }<commit_msg>Change image sanitation to let images through without width or height info<commit_after>define(["vendor/mithril"], function(mDiscard) {

    "use strict"
    
    // This constructs a nested Mithril object with only specific HTML tags allowed
    // No attributes are allowed.
    // A css class (from a short approved list) can be set on a tag using a ".className" after the opening tag name.
    // For example: <span.narrafirma-special-warning>Warning!!!<span>
    
    const debugLogging = false
    
    // 1 is normal tag that needs to be closed; 2 is self-closing tag (br and hr)
    const allowedHTMLTags = {
        // a: 1,
        address: 1,
        article: 1,
        b: 1,
        big: 1,
        blockquote: 1,
        br: 2,
        caption: 1,
        cite: 1,
        code: 1,
        del: 1,
        div: 1,
        dd: 1,
        d1: 1,
        dt: 1,
        em: 1,
        h1: 1,
        h2: 1,
        h3: 1,
        h4: 1,
        h5: 1,
        h6: 1,
        hr: 2,
        i: 1,
        // img
        kbd: 1,
        li: 1,
        ol: 1,
        p: 1,
        pre: 1,
        s: 1,
        small: 1,
        span: 1,
        sup: 1,
        sub: 1,
        strong: 1,
        strike: 1,
        table: 1,
        td: 1,
        th: 1,
        tr: 1,
        u: 1,
        ul: 1
    }
    
    const allowedCSSClasses = {
        "narrafirma-special-warning": 1
    }
    
    function isURLAcceptable(url, configuration) {
        if (!url) return ""
        if (url.substring(0, 5) === "http:" || url.substring(0, 6) === "https:") return url
        if (configuration.baseURL) return configuration.baseURL + url
        return ""
    }
    
    function isTagAllowed(tagName, configuration) {
        if (allowedHTMLTags[tagName]) return true
        
        if (!configuration) configuration = {}
        
        if (tagName === "a" && configuration.allowLinks) return true
        if (tagName === "img" && configuration.allowImages) return true
        return false
    }
    
    function generateVDOM(nodes, configuration) {
        if (!nodes) return []
        // console.log("generateVDOM nodes length", nodes.length)
      
        const result = []
        
        for (let i = 0; i < nodes.length; i++) {
            const node = nodes[i]
            // console.log("nodeType", node.nodeType)
    
            switch (node.nodeType) {
            case 1:
                let tagName = node.tagName.toLowerCase()
                
                // console.log("element", node)
                if (!isTagAllowed(tagName, configuration)) {
                    if (debugLogging) console.log("disallowed tag", tagName)
                    tagName = "span"
                }
                
                // TODO: Allow more attributes maybe
                let attributes = {}
                for (let j = 0; j < node.attributes.length; j++) {
                    const attribute = node.attributes[j]
                    if (attribute.name === "class") {
                        const theClassOrClasses = attribute.value
                        if (theClassOrClasses in allowedCSSClasses) {
                            attributes["class"] = theClassOrClasses
                        } else {
                            if (debugLogging) console.log("WARN: CSS class not allowed", theClassOrClasses)
                        }
                    }
                    if (configuration.allowLinks && attribute.name === "href") {
                        let url = attribute.value
                        if (url && url.substring(0, 2) === "//") {
                            url = window.location.protocol + url
                        }
                        const usableURL = isURLAcceptable(url, configuration)
                        if (usableURL) {
                            attributes["href"] = usableURL
                        }
                    }
                    if (configuration.allowImages && attribute.name === "src") {
                        let url = attribute.value
                        if (url && url.substring(0, 2) === "//") {
                            url = window.location.protocol + url
                        }
                        const usableURL = isURLAcceptable(url, configuration)
                        if (usableURL) {
                            attributes["src"] = usableURL
                        }
                    }
                    if (configuration.allowImages && (attribute.name === "width" || attribute.name === "height")) {
                        const size = parseInt(attribute.value)
                        if (!isNaN(size) && size > 1) {
                            attributes[attribute.name] = size
                        } else {
                            // Tracking image
                            console.log("discarding likely tracking image")
                            tagName = "span"
                            attributes = {}
                            break
                        }
                    }                    
                }
                
                if (tagName === "a") {
                    attributes["rel"] = "nofollow"
                }
                
                if (tagName === "img") {
                    if (attributes["width"] === 0 || attributes["width"] === 1 || attributes["height"] === 0 || attributes["height"] === 1) {
                        console.log("discarding likely tracking image (2)", attributes["src"])
                        tagName = "span"
                        attributes = {}
                    }
                }
                
                const children = generateVDOM(node.childNodes, configuration)
                const vdom = m(tagName, attributes, children)
                result.push(vdom)
                break
            case 3:
                if (node.data) {
                    // console.log("adding text node", node.data)
                    result.push(node.data)
                }
{'fraction_non_alphanumeric': 0.10287443267776097, 'fraction_numerical': 0.011430492519751218, 'mean_word_length': 1.0672341904100069, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1513861', 'n_tokens_mistral': 3115, 'n_tokens_neox': 2876, 'n_words': 799}