<commit_before># -*- encoding : utf-8 -*-
class RubyPreprocessor < BasePreprocessor

  attr :filename
  attr :execute
  attr :compile

  def initialize(attribut)
    super(attribut)
    @filename = 'code.rb'
    @execute = 'ruby $FILE$'
    @compile = ''
  end

  def process_code(code_msg)
    i=0
    codes = ''
    code_msg.each_line do |s|
      codes += s.chomp + " # #{$prefix}(#{i+1}#{$prefix})\n" + "#{$prefix}line(#{i})\n"
      i += 1
    end
    insert_logic + codes + "\n"
  end

  def postprocess_error(line, code, file)

    #remove filepath
    index_begin = line.index('/') #filepath starts with /
    index_end = line.index($prefix+file) #filepath ends with filename
    if index_begin and index_end and index_begin < index_end #found a filepath?
      index_end += "#{$prefix+file}".length #add the lenght of the filename to the end
      line.slice!(index_begin...index_end) #remove the filepath

      #chance the linenumber
      index_line_end = line.index(':', index_begin+1) #find the : after the linenumber
      line_number = line[index_begin+1...index_line_end] #get the linenumber between the two :
      i = 1 #Set a counter
      new_line = '' #Set a result string
      code.each_line() do |l| #search in the executed code for the right line. In every line is a comment with the original linenumber
        if i == line_number.to_i #find the line from the errormessage
          line_begin=l.index("#{$prefix}(") #find the begin of the original linenumber in the comment
          line_end=l.index("#{$prefix})") #find the end of the original linenumber in the comment
          if line_begin and line_end #found something?
            new_line = l[line_begin+"#{$prefix}(".length...line_end] #Set the new linenumber to the number in the comment
          end
        end
        i += 1
      end
      line.slice!(index_begin+1...index_line_end) #remove the old linenumber from the error
      line = line.insert(index_begin+1, new_line) #add the new linenumber to the error

      if new_line == ''
        line.slice!(index_begin..index_begin+1)
      else
        line = line.insert(index_begin, 'line') #add a line to the error instead of the filepath
      end
    end
    line
  end

  def preprocess_filename
  end

  def preprocess_execute
  end

  def preprocess_compile
  end<commit_msg>clean up code<commit_after># -*- encoding : utf-8 -*-
class RubyPreprocessor < BasePreprocessor

  attr :filename
  attr :execute
  attr :compile

  def initialize(attribut)
    super(attribut)
    @filename = 'code.rb'
    @execute = 'ruby $FILE$'
    @compile = ''
  end

  def process_code(code_msg)
    i=0
    codes = ''
    code_msg.each_line do |s|
      codes += s.chomp + " # #{$prefix}(#{i+1}#{$prefix})\n" + "#{$prefix}line(#{i})\n"
      i += 1
    end
    insert_logic + codes + "\n"
  end

  def postprocess_error(line, code, file)

    #remove filepath
    index_begin = line.index('/') #filepath starts with /
    index_end = line.index($prefix+file) #filepath ends with filename
    if index_begin and index_end and index_begin < index_end #found a filepath?
      index_end += "#{$prefix+file}".length #add the lenght of the filename to the end
      line.slice!(index_begin...index_end) #remove the filepath

      #chance the linenumber
      index_line_end = line.index(':', index_begin+1) #find the : after the linenumber
      line_number = line[index_begin+1...index_line_end] #get the linenumber between the two :
      i = 1 #Set a counter
      new_line = '' #Set a result string
      code.each_line() do |l| #search in the executed code for the right line. In every line is a comment with the original linenumber
        if i == line_number.to_i #find the line from the errormessage
          line_begin=l.index("#{$prefix}(") #find the begin of the original linenumber in the comment
          line_end=l.index("#{$prefix})") #find the end of the original linenumber in the comment
          if line_begin and line_end #found something?
            new_line = l[line_begin+"#{$prefix}(".length...line_end] #Set the new linenumber to the number in the comment
          end
        end
        i += 1
      end
      line.slice!(index_begin+1...index_line_end) #remove the old linenumber from the error
      line = line.insert(index_begin+1, new_line) #add the new linenumber to the error

      if new_line == '' #is there a result for the new linenumber?
        line.slice!(index_begin..index_begin+1) #remove the : around the old number
      else
        line = line.insert(index_begin, 'line') #add a line to the error instead of the filepath
      end
    end
    line
  end

  def preprocess_filename
  end

  def preprocess_execute
  end

  def preprocess_compile
  end
{'fraction_non_alphanumeric': 0.10470762085708255, 'fraction_numerical': 0.004644289634789952, 'mean_word_length': 2.8836065573770493, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '243929', 'n_tokens_mistral': 1495, 'n_tokens_neox': 1441, 'n_words': 485}