<commit_before>#reference: 
# http://www.java2s.com/Code/Ruby/Language-Basics/Rubysreservedwords.htm
require_relative 'method_data'
require_relative 'attribute_data'

module Languages

  # Token types used to identify keywords
  CLASS_TOKEN = 1 # Executes the block passed to the method.
  ATTRIBUTE_TOKEN = 2
  DEF_TOKEN = 3
  VISIBILITY_TOKEN = 4
  END_TOKEN = 5 # Code, enclosed in { and }, to run when the program ends.
  IF_TOKEN = 6 # Executes code block if true. Closes with end.
  WHILE_TOKEN = 7 # Executes the block passed to the method.
  DO_TOKEN = 8 # Begins a block and executes code in that block(end)
  BEGIN_TOKEN = 9 # Code, enclosed in { and }, to run before the program runs.
  ALIAS_TOKEN = 10 # Creates an alias: method, operator, or global variable.
  AND_TOKEN = 11 # Logical operator same as && except and has lower precedence.
  BEGIN_DOWN_TOKEN = 12 # Begins a code block or group of statements (end).
  BREAK_TOKEN = 13 # Terminates a while or until loop or a method.
  CASE_TOKEN = 14 # Compares an expression with a matching when clause (end).
# LOST NUMBER 15, ADD NEW KEYWORD HERE.
  DEFINED_TOKEN = 16 # Determines: variable, method, super method, or block.
  ELSE_TOKEN = 17 # Executes if previous conditional.
  ELSIF_TOKEN = 18 # Executes if previous conditional, in if or elsif.
  ENSURE_TOKEN = 19 # Always executes at block termination.
  FALSE_TOKEN = 20 # Logical or Boolean false, instance of FalseClass.
  TRUE_TOKEN = 21 # Instance of TrueClass.
  FOR_TOKEN = 22 # Begins a for loop; used with in.
  MODULE_TOKEN = 23 # Defines a module; closes with end.
  NEXT_TOKEN = 24 # Jumps before a loop's conditional.
  NIL_TOKEN = 25 # Object of NilClass.
  NOT_TOKEN = 26 # Logical operator, same as !.
  OR_TOKEN = 27 # Logical operator, same as ||.
  REDO_TOKEN = 28 # Jumps after a loop's conditional.
  RESCUE_TOKEN = 29 # Evaluates an expression after an exception is raised.
  RETRY_TOKEN = 30 # Repeats a method call outside of rescue.
  RETURN_TOKEN = 31 # Returns a value from a method or block.
  SELF_TOKEN = 32 # Current object (invoked by a method).
  SUPER_TOKEN = 33 # Calls method of the same name in the superclass.
  THEN_TOKEN = 34 # A continuation for if, unless, and when.
  UNDEF_TOKEN = 35 # Makes a method in current class undefined.
  UNLESS_TOKEN = 36 # Executes code block if conditional statement is false.
  UNTIL_TOKEN = 37 # Executes code block while conditional statement is false.
  WHEN_TOKEN = 38 # Starts a clause (one or more) under case.
  YIELD_TOKEN = 39 # Executes the block passed to the method.

  # This class implements useful methods for help to handling Ruby syntax.
  class RubySyntaxSupport

    def apply_regex(line, regex)
      result = line.scan(regex)[0]
      if not result
        return nil
      end
      return result
    end

    # Verify if line has the keyword and return it, otherwise return nil
    # @param line String to apply the regex
    # @return Return nil if anything is find, or the class name
    def get_class_name(line)
      regexExpression = /^\s*class\b[ |\t]+\s*(.*)\b/
      return apply_regex(line, regexExpression)
    end

    def handle_string(line)
      return line.gsub!(/\s+|:|@|=/,"")
    end

    # Verify if a line has an attribute. If it has attribute, first the 
    # function capture all lines and remove "@" or ":" and whitespace, finally
    # it splits the string by "," and return an array. Otherwise it returns 
    # nil.
    # @param line to inpect for find attribute.
    # @return Return nil if not find attribute or an array with the attribute.
    def get_attribute(line)
      regexExp = /^\s*(?:@|attr_(?:accessor|read|write))(.*)$/
      result = apply_regex(line, regexExp)
      unless result
        return nil
      end
      # Split string by comma
      result = result.join("")
      result = result.split(",")
      listOfAttributes = []
      result.each do |variable|
        if variable =~ /=/
          if variable.scan(/=/).count > 1
            next
          end
          variable = variable.scan(/.*=/)
          if variable =~ /\./
            return nil
          end
          variable = handle_string(variable.join(""))
          listOfAttributes.push(variable)
          next
        elsif variable =~ /\./
          return nil
        else
          variable = handle_string(variable)
          listOfAttributes.push(variable)
        end
      end
      return listOfAttributes.join("")
    end

    def get_method(line)
      regexExpression = /^\s*def\b\s*(\w*)\b/<commit_msg>Fixed attribute collection, however it is time to refactory.<commit_after>#reference: 
# http://www.java2s.com/Code/Ruby/Language-Basics/Rubysreservedwords.htm
require_relative 'method_data'
require_relative 'attribute_data'

module Languages

  # Token types used to identify keywords
  CLASS_TOKEN = 1 # Executes the block passed to the method.
  ATTRIBUTE_TOKEN = 2
  DEF_TOKEN = 3
  VISIBILITY_TOKEN = 4
  END_TOKEN = 5 # Code, enclosed in { and }, to run when the program ends.
  IF_TOKEN = 6 # Executes code block if true. Closes with end.
  WHILE_TOKEN = 7 # Executes the block passed to the method.
  DO_TOKEN = 8 # Begins a block and executes code in that block(end)
  BEGIN_TOKEN = 9 # Code, enclosed in { and }, to run before the program runs.
  ALIAS_TOKEN = 10 # Creates an alias: method, operator, or global variable.
  AND_TOKEN = 11 # Logical operator same as && except and has lower precedence.
  BEGIN_DOWN_TOKEN = 12 # Begins a code block or group of statements (end).
  BREAK_TOKEN = 13 # Terminates a while or until loop or a method.
  CASE_TOKEN = 14 # Compares an expression with a matching when clause (end).
# LOST NUMBER 15, ADD NEW KEYWORD HERE.
  DEFINED_TOKEN = 16 # Determines: variable, method, super method, or block.
  ELSE_TOKEN = 17 # Executes if previous conditional.
  ELSIF_TOKEN = 18 # Executes if previous conditional, in if or elsif.
  ENSURE_TOKEN = 19 # Always executes at block termination.
  FALSE_TOKEN = 20 # Logical or Boolean false, instance of FalseClass.
  TRUE_TOKEN = 21 # Instance of TrueClass.
  FOR_TOKEN = 22 # Begins a for loop; used with in.
  MODULE_TOKEN = 23 # Defines a module; closes with end.
  NEXT_TOKEN = 24 # Jumps before a loop's conditional.
  NIL_TOKEN = 25 # Object of NilClass.
  NOT_TOKEN = 26 # Logical operator, same as !.
  OR_TOKEN = 27 # Logical operator, same as ||.
  REDO_TOKEN = 28 # Jumps after a loop's conditional.
  RESCUE_TOKEN = 29 # Evaluates an expression after an exception is raised.
  RETRY_TOKEN = 30 # Repeats a method call outside of rescue.
  RETURN_TOKEN = 31 # Returns a value from a method or block.
  SELF_TOKEN = 32 # Current object (invoked by a method).
  SUPER_TOKEN = 33 # Calls method of the same name in the superclass.
  THEN_TOKEN = 34 # A continuation for if, unless, and when.
  UNDEF_TOKEN = 35 # Makes a method in current class undefined.
  UNLESS_TOKEN = 36 # Executes code block if conditional statement is false.
  UNTIL_TOKEN = 37 # Executes code block while conditional statement is false.
  WHEN_TOKEN = 38 # Starts a clause (one or more) under case.
  YIELD_TOKEN = 39 # Executes the block passed to the method.

  # This class implements useful methods for help to handling Ruby syntax.
  class RubySyntaxSupport

    def apply_regex(line, regex)
      result = line.scan(regex)[0]
      if not result
        return nil
      end
      return result
    end

    # Verify if line has the keyword and return it, otherwise return nil
    # @param line String to apply the regex
    # @return Return nil if anything is find, or the class name
    def get_class_name(line)
      regexExpression = /^\s*class\b[ |\t]+\s*(.*)\b/
      return apply_regex(line, regexExpression)
    end

    def handle_string(line)
      return line.gsub!(/\s+|:|@|=/,"")
    end

    # Verify if a line has an attribute. If it has attribute, first the 
    # function capture all lines and remove "@" or ":" and whitespace, finally
    # it splits the string by "," and return an array. Otherwise it returns 
    # nil.
    # @param line to inpect for find attribute.
    # @return Return nil if not find attribute or an array with the attribute.
    def get_attribute(line)
      regexExp = /^\s*(?:@|attr_(?:accessor|read|write))(.*)$/
      result = apply_regex(line, regexExp)
      unless result
        return nil
      end
      listOfAttributes = []
      # Has comma? Split string by comma
      result = result.join("")
      result.gsub!(/\(.*\)/,"")
      puts result
      if result.scan(/,/).count >= 1
        result = result.split(",")
        result.each do |variable|
          if variable.scan(/=/).count > 1
            return nil
          end
          variable = variable.scan(/.*=/)
          if variable =~ /\./
            return nil
          end
          variable = handle_string(variable.join(""))
          listOfAttributes.push(variable)
        end
      # Has sequence of "="?
      elsif result.scan(/=/).count > 1
        result = result.split("=")
        result.each do |variable|
          if variable =~ /\./
            return nil
          end
          variable = handle_string(variable.join(""))
          listOfAttributes.push(variable)
        end
      # Normal case
      else
        if result =~ /=/
          result = result.scan(/.*=/)
          result = result.join("")
          if result =~ /\./
            return nil
          end
        end
        if result =~ /\./
          return nil
        end
        if result =~ /\s+|:|@|=/
          result = handle_string(result)
        end
        listOfAttributes.push(result)
      end
      return listOfAttributes
    end

    def get_method(line)
      regexExpression = /^\s*def\b\s*(\w*)\b/
{'fraction_non_alphanumeric': 0.08754484879548949, 'fraction_numerical': 0.014966683751922092, 'mean_word_length': 2.6133333333333333, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5610413', 'n_tokens_mistral': 3058, 'n_tokens_neox': 2875, 'n_words': 1110}