<commit_before>require 'rubygems'
require 'diff/lcs'
require File.dirname(__FILE__) + '/result_processor'

def escape_content(s)
  CGI.escapeHTML(s).gsub(" ", "&nbsp;")
end

class DiffToHtml
  INTEGRATION_MAP = {
    :mediawiki => { :search_for => /\[\[([^\[\]]+)\]\]/, :replace_with => '#{url}/\1' },
    :redmine => { :search_for => /\b(?:refs|fixes)\s*\#(\d+)/i, :replace_with => '#{url}/issues/show/\1' },
    :bugzilla => { :search_for => /\bBUG\s*(\d+)/i, :replace_with => '#{url}/show_bug.cgi?id=\1' }
  }.freeze
  
  attr_accessor :file_prefix, :current_file_name
  attr_reader :result

  def initialize(previous_dir = nil, config = nil)
    @previous_dir = previous_dir
    @config = config || {}
    @lines_added = 0
  end

  def range_info(range)
    matches = range.match(/^@@ \-(\S+) \+(\S+)/)
    return matches[1..2].map { |m| m.split(',')[0].to_i }
  end

  def line_class(line)
    if line[:op] == :removal
      return " class=\"r\""
    elsif line[:op] == :addition
      return " class=\"a\""
    else
      return ''
    end
  end

  def add_block_to_results(block, escape)
    return if block.empty?
    block.each do |line|
      add_line_to_result(line, escape)
    end
  end

  def lines_per_diff
    @config['lines_per_diff']
  end

  def add_separator<commit_msg>Eliminate warning: instance variable @#{name} not initialized<commit_after>require 'rubygems'
require 'diff/lcs'
require File.dirname(__FILE__) + '/result_processor'

def escape_content(s)
  CGI.escapeHTML(s).gsub(" ", "&nbsp;")
end

class DiffToHtml
  INTEGRATION_MAP = {
    :mediawiki => { :search_for => /\[\[([^\[\]]+)\]\]/, :replace_with => '#{url}/\1' },
    :redmine => { :search_for => /\b(?:refs|fixes)\s*\#(\d+)/i, :replace_with => '#{url}/issues/show/\1' },
    :bugzilla => { :search_for => /\bBUG\s*(\d+)/i, :replace_with => '#{url}/show_bug.cgi?id=\1' }
  }.freeze
  
  attr_accessor :file_prefix, :current_file_name
  attr_reader :result

  def initialize(previous_dir = nil, config = nil)
    @previous_dir = previous_dir
    @config = config || {}
    @lines_added = 0
    @file_added = false
    @file_removed = false
    @binary = false
  end

  def range_info(range)
    matches = range.match(/^@@ \-(\S+) \+(\S+)/)
    return matches[1..2].map { |m| m.split(',')[0].to_i }
  end

  def line_class(line)
    if line[:op] == :removal
      return " class=\"r\""
    elsif line[:op] == :addition
      return " class=\"a\""
    else
      return ''
    end
  end

  def add_block_to_results(block, escape)
    return if block.empty?
    block.each do |line|
      add_line_to_result(line, escape)
    end
  end

  def lines_per_diff
    @config['lines_per_diff']
  end

  def add_separator
{'fraction_non_alphanumeric': 0.18865826538176428, 'fraction_numerical': 0.005189028910303929, 'mean_word_length': 3.1523076923076925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2442133', 'n_tokens_mistral': 1041, 'n_tokens_neox': 1002, 'n_words': 197}