<commit_before>module Normalize

  class Text < Treetop::Runtime::SyntaxNode
    def to_s
      elements.reduce('') { |s,e| s+="#{e}" }
    end
  end

  class Delete < Treetop::Runtime::SyntaxNode
    def to_s
      ''
    end
  end

  class Normalize < Treetop::Runtime::SyntaxNode
    def to_s
      t=text_value
      t=t.downcase
      t=t.gsub(/\t/,' ') # convert tabs to spaces
      t=t.gsub(/  */,'') # remove spaces
      t=t.gsub(/;/,"\n") # split semicolon-delimited statemtns
    end
  end

  class Verbatim < Treetop::Runtime::SyntaxNode
    def to_s
      text_value
    end
  end

end<commit_msg>upcase instead of downcase (f90 sect. 3.1.1)<commit_after>module Normalize

  class Text < Treetop::Runtime::SyntaxNode
    def to_s
      elements.reduce('') { |s,e| s+="#{e}" }
    end
  end

  class Delete < Treetop::Runtime::SyntaxNode
    def to_s
      ''
    end
  end

  class Normalize < Treetop::Runtime::SyntaxNode
    def to_s
      t=text_value
      t=t.upcase
      t=t.gsub(/\t/,' ') # convert tabs to spaces
      t=t.gsub(/  */,'') # remove spaces
      t=t.gsub(/;/,"\n") # split semicolon-delimited statemtns
    end
  end

  class Verbatim < Treetop::Runtime::SyntaxNode
    def to_s
      text_value
    end
  end

end
{'fraction_non_alphanumeric': 0.13189448441247004, 'fraction_numerical': 0.003996802557953637, 'mean_word_length': 2.458563535911602, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5856739', 'n_tokens_mistral': 473, 'n_tokens_neox': 455, 'n_words': 103}