<commit_before>module Scrub 
  # 1: regularize
  def self.regularize_body(html)
    replacements = [
                   [ '&#151;', '&#8212;' ],
                   [ '&#146;', '&#8217;' ],
                   [ '&#145;', '&#8216;' ],
                   [ '&#148;', '&#8221;' ],
                   [ '&#147;', '&#8220;' ],
                   [ '&nbsp;', ' ' ],
                   ]
    replacements.each do |replace|
      html.gsub!(replace.first, replace.last)
    end
    html.gsub!(/[\s]+/, " ")
    doc = Nokogiri::HTML(html)
    body = doc.xpath('//body').first
    Scrub.remove_scripts(body) if body
  end

  # 2: get node from MyWebsites

  # 3: to xhtml
  def self.to_xhtml(body)
    nodeset = body.children
    nodeset = Scrub.remove_blanks(nodeset)
    nodeset = Scrub.remove_surrounding(nodeset)
    body = nodeset.to_xhtml(:encoding => 'utf8').gsub("&#13;", '')
  end

  # 4: sanitize
  def self.sanitize_html(html)
    html = Sanitize.clean(
      html, 
      :elements => [ 'a', 'b', 'big', 'blockquote', 'br', 'center', 
                     'div', 'dt', 'em', 'i', 'h1', 'h2', 'h3', 'h4', 
                     'h5', 'h6', 'hr', 'img', 'li', 'p', 'small', 
                     'strike', 'strong', 'sub', 'sup', 'u'], 
      :attributes => { 'a' => ['href'],
                       'img' => ['align', 'alt', 'height', 'src', 'title', 'width'] })
    html.gsub!(/<a><\/a>/, "")
    html.gsub!("Alright", "All right")
    html.gsub!("alright", "all right")
    html.gsub!("\n", " ")
    html.gsub!(/ +<p>/, "<p>")
    html.gsub!(/<p> +/, "<p>")
    html.gsub!(/ +<\/p>/, "</p>")
    html.gsub!(/<\/p> +/, "</p>")
    html.gsub!(/ +<br \/>/, "<br \>")
    html.gsub!(/<br \/> +/, "<br \>")
    html.gsub!(/<br \/><\/p>/, "</p>")
    html.gsub!(/<p><br \/>/, "<p>")
    html.gsub!(/(<br \/>){3,}/, '<hr />')
    html.gsub!(/(<p><\/p>){2,}/, '<hr />')
    html.gsub!(/<p><hr \/>/, '<hr />')
    html.gsub!(/(<hr \/>\s*){2,}/, '<hr />')
    html.gsub!(/(<p><\/p>)/, "")
    html.gsub!(/<hr \/>/, '<hr width="80%"/>')
    "<body>" + html + "</body>"
  end

  def self.html_to_text(html)
    return "" unless html
    text = html.gsub(/<a .*?>(.*?)<\/a>/m) {|s| " [#{$1}] " unless $1.blank?}
    text = text.gsub(/<\/?b>/, "\*")
    text = text.gsub(/<\/?big>/, "\*")<commit_msg>small fix to make scrubbing cleaner<commit_after>module Scrub 
  # 1: regularize
  def self.regularize_body(html)
    replacements = [
                   [ '&#151;', '&#8212;' ],
                   [ '&#146;', '&#8217;' ],
                   [ '&#145;', '&#8216;' ],
                   [ '&#148;', '&#8221;' ],
                   [ '&#147;', '&#8220;' ],
                   [ '&nbsp;', ' ' ],
                   ]
    replacements.each do |replace|
      html.gsub!(replace.first, replace.last)
    end
    html.gsub!(/[\s]+/, " ")
    doc = Nokogiri::HTML(html)
    body = doc.xpath('//body').first
    Scrub.remove_scripts(body) if body
  end

  # 2: get node from MyWebsites

  # 3: to xhtml
  def self.to_xhtml(body)
    nodeset = body.children
    nodeset = Scrub.remove_blanks(nodeset)
    nodeset = Scrub.remove_surrounding(nodeset)
    body = nodeset.to_xhtml(:encoding => 'utf8').gsub("&#13;", '')
  end

  # 4: sanitize
  def self.sanitize_html(html)
    html = Sanitize.clean(
      html, 
      :elements => [ 'a', 'b', 'big', 'blockquote', 'br', 'center', 
                     'div', 'dt', 'em', 'i', 'h1', 'h2', 'h3', 'h4', 
                     'h5', 'h6', 'hr', 'img', 'li', 'p', 'small', 
                     'strike', 'strong', 'sub', 'sup', 'u'], 
      :attributes => { 'a' => ['href'],
                       'img' => ['align', 'alt', 'height', 'src', 'title', 'width'] })
    html.gsub!(/<a><\/a>/, "")
    html.gsub!("Alright", "All right")
    html.gsub!("alright", "all right")
    html.gsub!("\n", " ")
    html.gsub!(/ +<p>/, "<p>")
    html.gsub!(/<p> +/, "<p>")
    html.gsub!(/ +<\/p>/, "</p>")
    html.gsub!(/<\/p> +/, "</p>")
    html.gsub!(/ +<br \/>/, "<br \>")
    html.gsub!(/<br \/> +/, "<br \>")
    html.gsub!(/<br \/><\/p>/, "</p>")
    html.gsub!(/<p><br \/>/, "<p>")
    html.gsub!(/(<br \/>){3,}/, '<hr />')
    html.gsub!(/(<br \/>){2,}/, '<p>')
    html.gsub!(/(<p><\/p>){2,}/, '<hr />')
    html.gsub!(/<p><hr \/>/, '<hr />')
    html.gsub!(/(<hr \/>\s*){2,}/, '<hr />')
    html.gsub!(/(<p><\/p>)/, "")
    html.gsub!(/<hr \/>/, '<hr width="80%"/>')
    "<body>" + html + "</body>"
  end

  def self.html_to_text(html)
    return "" unless html
    text = html.gsub(/<a .*?>(.*?)<\/a>/m) {|s| " [#{$1}] " unless $1.blank?}
    text = text.gsub(/<\/?b>/, "\*")
    text = text.gsub(/<\/?big>/, "\*")
{'fraction_non_alphanumeric': 0.2973031283710895, 'fraction_numerical': 0.02394822006472492, 'mean_word_length': 1.9679897567221512, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 95, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6910320', 'n_tokens_mistral': 1947, 'n_tokens_neox': 1861, 'n_words': 326}