<commit_before>LaTeX = {} unless LaTeX

LaTeX.text2latex = (text) ->
  latex = @html2latex(@cleanHTML(text))
  return BetterBibTeXBraceBalancer.parse(latex) if latex.indexOf("\\{") >= 0 || latex.indexOf("\\textleftbrace") >= 0 || latex.indexOf("\\}") >= 0 || latex.indexOf("\\textrightbrace") >= 0
  return latex

LaTeX.cleanHTML = (text) ->
  html = ''
  for chunk, i in text.split(/(<\/?(?:i|italic|b|sub|sup|pre|span)(?:[^a-z][^>]*)?>)/i)
    if i % 2 == 0 # text
      html += LaTeX.he.escape(chunk)
    else
      switch
        when chunk.match(/^<pre/i)
          html += '<![CDATA['
        when chunk.match(/^<\/pre/i)
          html += ']]>'
        else
          html += chunk
  return html

LaTeX.html2latex = (html) ->
  return (new @HTML(html)).latex.trim().replace(/\n\n\n+/g, "\n\n")

class LaTeX.HTML
  constructor: (html) ->
    @latex = ''
    @stack = []
    @mapping = (if Translator.unicode then LaTeX.toLaTeX.unicode else LaTeX.toLaTeX.ascii)

    LaTeX.HTMLtoDOM.Parser(html, @)

  start: (tag, attrs, unary) ->
    tag = {name: tag, attrs: {}}
    for attr in attrs
      tag.attrs[attr.name.toLowerCase()] = attr.value<commit_msg>fixes #257<commit_after>LaTeX = {} unless LaTeX

LaTeX.text2latex = (text) ->
  latex = @html2latex(@cleanHTML(text))
  return BetterBibTeXBraceBalancer.parse(latex) if latex.indexOf("\\{") >= 0 || latex.indexOf("\\textleftbrace") >= 0 || latex.indexOf("\\}") >= 0 || latex.indexOf("\\textrightbrace") >= 0
  return latex

LaTeX.cleanHTML = (text) ->
  html = ''
  cdata = false
  for chunk, i in text.split(/(<\/?(?:i|italic|b|sub|sup|pre|span)(?:[^>a-z][^>]*)?>)/i)
    switch
      when i % 2 == 0 # text
        html += LaTeX.he.escape(chunk)

      when chunk.match(/^<pre/i)
        html += '<![CDATA['
        cdata = true

      when chunk.match(/^<\/pre/i)
        html += ']]>'
        cdata = false

      else
        html += chunk

  html += ']]>' if cdata

  return html

LaTeX.html2latex = (html) ->
  return (new @HTML(html)).latex.trim().replace(/\n\n\n+/g, "\n\n")

class LaTeX.HTML
  constructor: (html) ->
    @latex = ''
    @stack = []
    @mapping = (if Translator.unicode then LaTeX.toLaTeX.unicode else LaTeX.toLaTeX.ascii)

    LaTeX.HTMLtoDOM.Parser(html, @)

  start: (tag, attrs, unary) ->
    tag = {name: tag, attrs: {}}
    for attr in attrs
      tag.attrs[attr.name.toLowerCase()] = attr.value
{'fraction_non_alphanumeric': 0.20385906040268456, 'fraction_numerical': 0.008808724832214765, 'mean_word_length': 2.738244514106583, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5822746', 'n_tokens_mistral': 945, 'n_tokens_neox': 875, 'n_words': 174}