<commit_before>#= require trix/models/text
#= require trix/dom

class Trix.HTMLParser
  styleMap =
    bold: ["font-weight", "bold"]
    italic: ["font-style", "italic"]

  allowedAttributes = "style href src width height".split(" ")

  @parse: (html) ->
    parser = new this html
    parser.parse()
    parser

  constructor: (@html) ->
    @text = new Trix.Text

  createHiddenContainer: ->
    @container = sanitizeHTML(squish(@html))
    @container.style["display"] = "none"
    document.body.appendChild(@container)

  removeHiddenContainer: ->
    document.body.removeChild(@container)

  parse: ->
    @createHiddenContainer()
    walker = document.createTreeWalker(@container, NodeFilter.SHOW_ALL)
    @processNode(walker.currentNode) while walker.nextNode()
    @removeHiddenContainer()

  processNode: (node) ->
    switch node.nodeType
      when Node.TEXT_NODE
        @processTextNode(node)
      when Node.ELEMENT_NODE
        @processElementNode(node)

  processTextNode: (node) ->
    @appendString(node.textContent, getAttributes(node.parentNode))

  processElementNode: (node) ->
    switch node.tagName.toLowerCase()
      when "br"
        @appendString("\n", getAttributes(node))
      when "img"
        attachment = { type: "image" }
        attachment[key] = node[key] for key in ["src", "width", "height"]
        @appendAttachment(attachment, getAttributes(node))

  appendString: (string, attributes) ->
    text = Trix.Text.textForStringWithAttributes(string, attributes)
    @text.appendText(text)

  appendAttachment: (attachment, attributes) ->
    text = Trix.Text.textForAttachmentWithAttributes(attachment, attributes)
    @text.appendText(text)

  getText: ->
    @text

  getAttributes = (element) ->
    attributes = {}
    style = window.getComputedStyle(element)

    for attribute, [property, value] of styleMap
      if style[property] is value
        attributes[attribute] = true

    if link = Trix.DOM.closest(element, "a")
      attributes["href"] = link.getAttribute("href")

    attributes

  squish = (string) ->
    string.trim().replace(/\n/g, " ").replace(/\s{2,}/g, " ")

  sanitizeHTML = (html) ->
    container = document.createElement("div")
    container.innerHTML = html
    walker = document.createTreeWalker(container, NodeFilter.SHOW_ELEMENT)

    while walker.nextNode()
      element = walker.currentNode
      for {name} in element.attributes
        element.removeAttribute(name) unless name in allowedAttributes

    container<commit_msg>Closure around attributes object<commit_after>#= require trix/models/text
#= require trix/dom

class Trix.HTMLParser
  styleMap =
    bold: ["font-weight", "bold"]
    italic: ["font-style", "italic"]

  allowedAttributes = "style href src width height".split(" ")

  @parse: (html) ->
    parser = new this html
    parser.parse()
    parser

  constructor: (@html) ->
    @text = new Trix.Text

  createHiddenContainer: ->
    @container = sanitizeHTML(squish(@html))
    @container.style["display"] = "none"
    document.body.appendChild(@container)

  removeHiddenContainer: ->
    document.body.removeChild(@container)

  parse: ->
    @createHiddenContainer()
    walker = document.createTreeWalker(@container, NodeFilter.SHOW_ALL)
    @processNode(walker.currentNode) while walker.nextNode()
    @removeHiddenContainer()

  processNode: (node) ->
    switch node.nodeType
      when Node.TEXT_NODE
        @processTextNode(node)
      when Node.ELEMENT_NODE
        @processElementNode(node)

  processTextNode: (node) ->
    @appendString(node.textContent, getAttributes(node.parentNode))

  processElementNode: (node) ->
    switch node.tagName.toLowerCase()
      when "br"
        @appendString("\n", getAttributes(node))
      when "img"
        attachment = { type: "image" }
        attachment[key] = node[key] for key in ["src", "width", "height"]
        @appendAttachment(attachment, getAttributes(node))

  appendString: (string, attributes) ->
    text = Trix.Text.textForStringWithAttributes(string, attributes)
    @text.appendText(text)

  appendAttachment: (attachment, attributes) ->
    text = Trix.Text.textForAttachmentWithAttributes(attachment, attributes)
    @text.appendText(text)

  getText: ->
    @text

  getAttributes = (element) ->
    attributes = {}
    style = window.getComputedStyle(element)

    for attribute, [property, value] of styleMap
      if style[property] is value
        attributes[attribute] = true

    if link = Trix.DOM.closest(element, "a")
      attributes["href"] = link.getAttribute("href")

    attributes

  squish = (string) ->
    string.trim().replace(/\n/g, " ").replace(/\s{2,}/g, " ")

  sanitizeHTML = (html) ->
    container = document.createElement("div")
    container.innerHTML = html
    walker = document.createTreeWalker(container, NodeFilter.SHOW_ELEMENT)

    while walker.nextNode()
      element = walker.currentNode
      for attribute in element.attributes
        do ({name}) ->
          element.removeAttribute(name) unless name in allowedAttributes

    container
{'fraction_non_alphanumeric': 0.12813921297211786, 'fraction_numerical': 0.00039549139806209217, 'mean_word_length': 3.4290718038528896, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '787361', 'n_tokens_mistral': 1509, 'n_tokens_neox': 1469, 'n_words': 312}