<commit_before>import Parser from 'htmlparser2/lib/Parser'

export default function(incrementalDom) {
  const {
    attr,
    elementClose,
    elementOpen,
    elementOpenEnd,
    elementOpenStart,
    elementVoid,
    text,
  } = incrementalDom

  const sanitizeName = name => name.replace(/[^-:\w]/g, '')

  const iDOMParser = new Parser(
    {
      onopentag: name => elementOpenEnd(sanitizeName(name)),
      onopentagname: name => elementOpenStart(sanitizeName(name)),
      onattribute: (name, value) => {
        const sanitizedName = sanitizeName(name)
        if (sanitizedName !== '') attr(sanitizedName, value)
      },
      ontext: text,
      onclosetag: name => elementClose(sanitizeName(name)),
    },
    {
      decodeEntities: true,
      lowerCaseAttributeNames: false,
      lowerCaseTags: false,
    }
  )

  const wrapIncrementalDOM = html =>
    typeof html === 'function' ? html() : iDOMParser.write(html)

  return {
    renderAttrsToArray(token) {
      if (!token.attrs) return []
      return token.attrs.reduce((v, a) => v.concat(a), [])
    },

    renderInline(tokens, options, env) {
      return () => {
        tokens.forEach((current, i) => {
          const { type } = current

          if (this.rules[type] !== undefined) {
            wrapIncrementalDOM(this.rules[type](tokens, i, options, env, this))
          } else {
            this.renderToken(tokens, i, options)()
          }
        })
      }
    },

    renderToken(tokens, idx) {
      return () => {
        const token = tokens[idx]
        if (token.hidden) return

        if (token.nesting === -1) {
          elementClose(token.tag)
        } else {
          const func = token.nesting === 0 ? elementVoid : elementOpen

          func.apply(
            this,
            [token.tag, '', []].concat(this.renderAttrsToArray(token))
          )
        }
      }
    },

    render(tokens, options, env) {
      return () => {
        tokens.forEach((current, i) => {
          const { type } = current

          if (type === 'inline') {
            this.renderInline(current.children, options, env)()
          } else if (this.rules[type] !== undefined) {
            wrapIncrementalDOM(this.rules[type](tokens, i, options, env, this))
          } else {
            this.renderToken(tokens, i, options, env)()
          }
        })
        iDOMParser.end()
        iDOMParser.reset()
      }
    },
  }
}
<commit_msg>Prevent twice element closing by htmlparser2<commit_after>import Parser from 'htmlparser2/lib/Parser'

export default function(incrementalDom) {
  const {
    attr,
    elementClose,
    elementOpen,
    elementOpenEnd,
    elementOpenStart,
    elementVoid,
    text,
  } = incrementalDom

  const sanitizeName = name => name.replace(/[^-:\w]/g, '')

  const iDOMParser = new Parser(
    {
      onopentag: name => elementOpenEnd(sanitizeName(name)),
      onopentagname: name => elementOpenStart(sanitizeName(name)),
      onattribute: (name, value) => {
        const sanitizedName = sanitizeName(name)
        if (sanitizedName !== '') attr(sanitizedName, value)
      },
      ontext: text,
      onclosetag: name => elementClose(sanitizeName(name)),
    },
    {
      decodeEntities: true,
      lowerCaseAttributeNames: false,
      lowerCaseTags: false,
    }
  )

  const wrapIncrementalDOM = html =>
    typeof html === 'function' ? html() : iDOMParser.write(html)

  return {
    renderAttrsToArray(token) {
      if (!token.attrs) return []
      return token.attrs.reduce((v, a) => v.concat(a), [])
    },

    renderInline(tokens, options, env) {
      return () => {
        tokens.forEach((current, i) => {
          const { type } = current

          if (this.rules[type] !== undefined) {
            wrapIncrementalDOM(this.rules[type](tokens, i, options, env, this))
          } else {
            this.renderToken(tokens, i, options)()
          }
        })
      }
    },

    renderToken(tokens, idx) {
      return () => {
        const token = tokens[idx]
        if (token.hidden) return

        if (token.nesting === -1) {
          elementClose(token.tag)
        } else {
          const func = token.nesting === 0 ? elementVoid : elementOpen

          func.apply(
            this,
            [token.tag, '', []].concat(this.renderAttrsToArray(token))
          )
        }
      }
    },

    render(tokens, options, env) {
      return () => {
        tokens.forEach((current, i) => {
          const { type } = current

          if (type === 'inline') {
            this.renderInline(current.children, options, env)()
          } else if (this.rules[type] !== undefined) {
            wrapIncrementalDOM(this.rules[type](tokens, i, options, env, this))
          } else {
            this.renderToken(tokens, i, options, env)()
          }
        })
        iDOMParser.reset()
      }
    },
  }
}

{'fraction_non_alphanumeric': 0.137782340862423, 'fraction_numerical': 0.001437371663244353, 'mean_word_length': 2.07124842370744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3706085', 'n_tokens_mistral': 1503, 'n_tokens_neox': 1464, 'n_words': 292}