<commit_before>module Normalize

  class Text < Treetop::Runtime::SyntaxNode
    def to_s
      elements.reduce("") { |s,e| s+="#{e}" }
    end
  end

  class Delete < Treetop::Runtime::SyntaxNode
    def to_s
      ""
    end
  end

  class Directive < Treetop::Runtime::SyntaxNode
    def to_s
      t=text_value
      t="\n"+t
    end
  end

  class Normalize < Treetop::Runtime::SyntaxNode
    def to_s
      t=text_value
      # Join continuation lines
      t=t.gsub(/&$(\n&?)?/,"")
      # Split semicolon-delimited statement lines
      t=t.gsub(/\s*;\s*/,"\n")
      # Attempting a multiline match of the regular expression below on a string
      # representing a huge source file is incredibly slow. Breaking the string
      # into lines improves performance significantly. 
      a=t.split("\n")
      a.each_index do |i|
        l=a[i]
        # Convert instances of F90:1016 char-string-edit-desc to quoted strings
        # to preserve case and whitespace.
        h=false
        p="\(.*?[0-9]{1,5}[ \t]format[ \t]*\\(.*?\)\([0-9]+\)[ \t]*[hH]\(.*?\)\\)\(.*\)"
        r=Regexp.new(p,true)
        while m=r.match(l)
          h=true
          p1=m[3][0..m[2].to_i-1]
          p2=m[3].sub(/^#{p1}/,"")
          l="#{m[1]}'#{p1}'#{p2})#{m[4]}"
        end
        # If a F90:1016 conversion occurred, quoted strings have been introduced
        # and it is no longer safe to change case or whitespace. Note, though,
        # that these conversions can only happen in the first normalization pass,
        # so that the second pass can normalize case and whitespace.
        unless h
          # Make upper-case characters lower-case
          l=l.downcase
          # Remove tabs & spaces
          l=l.gsub(/[ \t]+/,"")
        end
        a[i]=l
      end
      t=a.join("\n")
      # Remove blank lines
      t=t.gsub(/\n\n+/,"\n")
    end
  end

  class Quoted < Treetop::Runtime::SyntaxNode
    def to_s
      t=text_value
      t=t.gsub(/^!.*/,"")    # no comment lines
      t=t.gsub(/\n\n+/,"\n") # no blank lines
      t=t.gsub(/&$\n&?/,"")  # join continuation lines
    end
  end

  class SMS < Treetop::Runtime::SyntaxNode
    def to_s
      t=text_value
      t=t.downcase          # make upper-case characters lower-case
      t=t.gsub(/[ \t]+/,"") # remove tabs & spaces
      t="\n"+t
    end
  end

end

# <EMAIL><commit_msg>normalize functions: explicitly return t in last statement<commit_after>module Normalize

  class Text < Treetop::Runtime::SyntaxNode
    def to_s
      elements.reduce("") { |s,e| s+="#{e}" }
    end
  end

  class Delete < Treetop::Runtime::SyntaxNode
    def to_s
      ""
    end
  end

  class Directive < Treetop::Runtime::SyntaxNode
    def to_s
      t=text_value
      t="\n"+t
    end
  end

  class Normalize < Treetop::Runtime::SyntaxNode
    def to_s
      t=text_value
      # Join continuation lines
      t=t.gsub(/&$(\n&?)?/,"")
      # Split semicolon-delimited statement lines
      t=t.gsub(/\s*;\s*/,"\n")
      # Attempting a multiline match of the regular expression below on a string
      # representing a huge source file is incredibly slow. Breaking the string
      # into lines improves performance significantly. 
      a=t.split("\n")
      a.each_index do |i|
        l=a[i]
        # Convert instances of F90:1016 char-string-edit-desc to quoted strings
        # to preserve case and whitespace.
        h=false
        p="\(.*?[0-9]{1,5}[ \t]format[ \t]*\\(.*?\)\([0-9]+\)[ \t]*[hH]\(.*?\)\\)\(.*\)"
        r=Regexp.new(p,true)
        while m=r.match(l)
          h=true
          p1=m[3][0..m[2].to_i-1]
          p2=m[3].sub(/^#{p1}/,"")
          l="#{m[1]}'#{p1}'#{p2})#{m[4]}"
        end
        # If a F90:1016 conversion occurred, quoted strings have been introduced
        # and it is no longer safe to change case or whitespace. Note, though,
        # that these conversions can only happen in the first normalization pass,
        # so that the second pass can normalize case and whitespace.
        unless h
          # Make upper-case characters lower-case
          l=l.downcase
          # Remove tabs & spaces
          l=l.gsub(/[ \t]+/,"")
        end
        a[i]=l
      end
      t=a.join("\n")
      # Remove blank lines
      t=t.gsub(/\n\n+/,"\n")
      t
    end
  end

  class Quoted < Treetop::Runtime::SyntaxNode
    def to_s
      t=text_value
      t=t.gsub(/^!.*/,"")    # no comment lines
      t=t.gsub(/\n\n+/,"\n") # no blank lines
      t=t.gsub(/&$\n&?/,"")  # join continuation lines
      t
    end
  end

  class SMS < Treetop::Runtime::SyntaxNode
    def to_s
      t=text_value
      t=t.downcase          # make upper-case characters lower-case
      t=t.gsub(/[ \t]+/,"") # remove tabs & spaces
      t="\n"+t
      t
    end
  end

end

# <EMAIL>
{'fraction_non_alphanumeric': 0.15257774994781884, 'fraction_numerical': 0.012523481527864746, 'mean_word_length': 2.015733165512901, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7504017', 'n_tokens_mistral': 1710, 'n_tokens_neox': 1647, 'n_words': 427}