<commit_before>#!/usr/bin/env ruby

require_relative 'Grammar'

#
# Preprocess the emerald code and add notion of indentation so it may be parsed
# by a context free grammar. Removes all whitespace and adds braces to denote
# indentation.
#
module PreProcessor
  def self.process_emerald(file_name)
    input = File.open(file_name, "r").read

    in_literal = false;
    current_indent = 0; new_indent = 0; b_count = 0; output = ''

    input.each_line do |line|
      # remove any blank lines for intermediate form
      next if line.lstrip.length == 0

      new_indent = line.length - line.lstrip.length

      if new_indent > current_indent
        unless in_literal
          output += "{\n"; b_count += 1
          current_indent = new_indent
        end
      elsif new_indent < current_indent && new_indent != 0
        if in_literal
          output += "$\n"
          in_literal = false
          for i in 2..((current_indent - new_indent) / 2) do
            output += "}\n"; b_count -= 1
          end
        else
          for i in 1..((current_indent - new_indent) / 2) do
            output += "}\n"; b_count -= 1
          end
        end
        current_indent = new_indent
      end

      if in_literal
        # Crop off only Emerald indent whitespace
        # to preserve whitespace in the literal
        output += (line[current_indent..-1] || "")
          .gsub('$', '\$') 
        # $ is our end character, so 
        # we need to escape it in the literal
      else
        output += line.lstrip
      end

      if line.rstrip.end_with?("->")
        in_literal = true
        current_indent += 2
      end
    end

    # Print out remaining braces.
    for i in 1..b_count do
      output += "}\n"
    end

    output
  end
end<commit_msg>Minor changes to PreProcessor.rb.<commit_after>#!/usr/bin/env ruby

require_relative 'Grammar'

#
# Preprocess the emerald code and add notion of indentation so it may be parsed
# by a context free grammar. Removes all whitespace and adds braces to denote
# indentation.
#
module PreProcessor
  def self.process_emerald(file_name)
    input = File.open(file_name, "r").read

    in_literal = false;
    current_indent = 0; new_indent = 0; b_count = 0; output = ''

    input.each_line do |line|
      # remove any blank lines for intermediate form
      next if line.lstrip.length == 0

      new_indent = line.length - line.lstrip.length

      if new_indent > current_indent
        unless in_literal
          output += "{\n"; b_count += 1
          current_indent = new_indent
        end
      elsif new_indent < current_indent && new_indent != 0
        if in_literal
          output += "$\n"
          in_literal = false
          for i in 2..((current_indent - new_indent) / 2) do
            output += "}\n"; b_count -= 1
          end
        else
          for i in 1..((current_indent - new_indent) / 2) do
            output += "}\n"; b_count -= 1
          end
        end
        current_indent = new_indent
      end

      if in_literal
        # Crop off only Emerald indent whitespace to preserve
        # whitespace in the literal
        output += (line[current_indent..-1] || "").gsub('$', '\$')
        # $ is our end character, so we need to escape it in
        # the literal
      else
        output += line.lstrip
      end

      if line.rstrip.end_with?("->")
        in_literal = true
        current_indent += 2
      end
    end

    # Print out remaining braces.
    for i in 1..b_count do
      output += "}\n"
    end

    output
  end
end
{'fraction_non_alphanumeric': 0.09360022554271215, 'fraction_numerical': 0.00845785170566676, 'mean_word_length': 1.9105824446267432, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4859302', 'n_tokens_mistral': 1102, 'n_tokens_neox': 1047, 'n_words': 350}