<commit_before>require 'tempfile'

module HTMLDiff
  
  def self.log_to_stdout=(value)
   @@log_to_stdout = value
  end

  def self.diff(a, b)

    f1 = Tempfile.new("crabgrass-diff-a")
    f1.write a
    f1.close
          
    f2 = Tempfile.new("crabgrass-diff-b")
    f2.write b
    f2.close

    arguments = [PYTHON_COMMAND, HTML_DIFF_COMMAND, f1.path, f2.path]
    success, output = cmd(*arguments)
    f1.unlink
    f2.unlink

    return output
  end

  def self.cmd(*args)
    cmdstr = Escape.shell_command(args)
    log cmdstr
    output = `#{cmdstr}`
    return [$?.success?, output]
  end

  def self.log(*args)
    ActiveRecord::Base.logger.info "HTML_Diff --- " + args.join(' ')
    puts args.join(' ') if @@log_to_stdout
  end

end
<commit_msg>oops, HtmlDiff should be a class, not a module.<commit_after>require 'tempfile'

class HTMLDiff
  cattr_accessor :log_to_stdout

  def self.diff(a, b)

    f1 = Tempfile.new("crabgrass-diff-a")
    f1.write a
    f1.close
          
    f2 = Tempfile.new("crabgrass-diff-b")
    f2.write b
    f2.close

    arguments = [PYTHON_COMMAND, HTML_DIFF_COMMAND, f1.path, f2.path]
    success, output = cmd(*arguments)
    f1.unlink
    f2.unlink

    return output
  end

  def self.cmd(*args)
    cmdstr = Escape.shell_command(args)
    log cmdstr
    output = `#{cmdstr}`
    return [$?.success?, output]
  end

  def self.log(*args)
    ActiveRecord::Base.logger.info "HTML_Diff --- " + args.join(' ')
    puts args.join(' ') if log_to_stdout
  end

end

{'fraction_non_alphanumeric': 0.126657824933687, 'fraction_numerical': 0.013262599469496022, 'mean_word_length': 2.6894865525672373, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3988467', 'n_tokens_mistral': 582, 'n_tokens_neox': 571, 'n_words': 121}