<commit_before>require 'parslet'
require 'pathname'

require 'rip/keywords'

module Rip
  class Parser < Parslet::Parser
    root(:statements)

    # statement
    #   comment
    #   expression (literal, invocation or reference eg: 2 + 2, full_name())
    #   assignment (reference = expression eg: answer = 2 + 2, name = full_name())
    #   block (if, unless, switch, case, try, class, lambda et cetera)
    #   (expression | assignment | block) comment

    # in rip everything looks like one of the following
    #   comment
    #   string, number, regular expression etc
    #   list, map
    #   block
    #   reference
    #   reference followed by parameter list

    rule(:statement) { whitespaces? >> (comment | (expression >> spaces? >> comment.maybe)) >> whitespaces? }
    rule(:statements) { thing_list(statement, whitespaces?) }

    rule(:comment) { (pound >> (eol.absent? >> any).repeat.as(:comment)) >> eol.maybe }

    rule(:expression) { block_expression | simple_expression }

    #---------------------------------------------

    # NOTE anything that might be followed by an expression terminator
    rule(:simple_expression) do
      (((postfix | keyword | phrase) >> (spaces >> postfix).maybe) | postfix) >> spaces? >> expression_terminator?
    end

    # TODO allow parenthesis around phrase to arbitrary levels
    rule(:phrase) { (keyword | postfix).absent? >> (assignment | invocation | object) }

    rule(:postfix) do
      postfix_tail = spaces >> maybe_parens(phrase.as(:postfix_argument))
      (if_keyword >> postfix_tail).as(:if_postfix) | (unless_keyword >> postfix_tail).as(:unless_postfix) | (keyword >> postfix_tail).as(:postfix)
    end

    #---------------------------------------------

    rule(:parameters) { parens(comma_list(assignment | object).as(:parameters)) }

    # NOTE anything that should not be followed by an expression terminator
    rule(:block_expression) do
      block_body = surround_with(brace_open, statements.as(:body), brace_close)
      (keyword >> whitespaces? >> parameters.maybe >> whitespaces? >> block_body).as(:block)
    end

    #---------------------------------------------

    rule(:invocation) { regular_invocation | operator_invocation }
    rule(:regular_invocation) { ((block_expression | reference) >> parens(comma_list(object).as(:arguments))).as(:invocation) }
    rule(:operator_invocation) { (object.as(:operand) >> spaces >> reference.as(:operator) >> spaces >> object.as(:argument)).as(:invocation) }

    # TODO consider multiple assignment
    # TODO rules for visibility (public, protected, private)
    rule(:assignment) { (reference >> spaces >> equals >> spaces >> expression.as(:value)).as(:assignment) }

    #---------------------------------------------

    # FIXME invocation instead of regular_invocation
    rule(:object) { (block_expression | recursive_object | simple_object | regular_invocation | reference) >> property.repeat.as(:property_chain) }

    rule(:property) { dot >> (regular_invocation | reference) }

    # TODO rules for system, date, time, datetime, version, units?<commit_msg>Only phrases should be assigned<commit_after>require 'parslet'
require 'pathname'

require 'rip/keywords'

module Rip
  class Parser < Parslet::Parser
    root(:statements)

    # statement
    #   comment
    #   expression (literal, invocation or reference eg: 2 + 2, full_name())
    #   assignment (reference = expression eg: answer = 2 + 2, name = full_name())
    #   block (if, unless, switch, case, try, class, lambda et cetera)
    #   (expression | assignment | block) comment

    # in rip everything looks like one of the following
    #   comment
    #   string, number, regular expression etc
    #   list, map
    #   block
    #   reference
    #   reference followed by parameter list

    rule(:statement) { whitespaces? >> (comment | (expression >> spaces? >> comment.maybe)) >> whitespaces? }
    rule(:statements) { thing_list(statement, whitespaces?) }

    rule(:comment) { (pound >> (eol.absent? >> any).repeat.as(:comment)) >> eol.maybe }

    rule(:expression) { block_expression | simple_expression }

    #---------------------------------------------

    # NOTE anything that might be followed by an expression terminator
    rule(:simple_expression) do
      (((postfix | keyword | phrase) >> (spaces >> postfix).maybe) | postfix) >> spaces? >> expression_terminator?
    end

    # TODO allow parenthesis around phrase to arbitrary levels
    rule(:phrase) { (keyword | postfix).absent? >> (assignment | invocation | object) }

    rule(:postfix) do
      postfix_tail = spaces >> maybe_parens(phrase.as(:postfix_argument))
      (if_keyword >> postfix_tail).as(:if_postfix) | (unless_keyword >> postfix_tail).as(:unless_postfix) | (keyword >> postfix_tail).as(:postfix)
    end

    #---------------------------------------------

    rule(:parameters) { parens(comma_list(assignment | object).as(:parameters)) }

    # NOTE anything that should not be followed by an expression terminator
    rule(:block_expression) do
      block_body = surround_with(brace_open, statements.as(:body), brace_close)
      (keyword >> whitespaces? >> parameters.maybe >> whitespaces? >> block_body).as(:block)
    end

    #---------------------------------------------

    rule(:invocation) { regular_invocation | operator_invocation }
    rule(:regular_invocation) { ((block_expression | reference) >> parens(comma_list(object).as(:arguments))).as(:invocation) }
    rule(:operator_invocation) { (object.as(:operand) >> spaces >> reference.as(:operator) >> spaces >> object.as(:argument)).as(:invocation) }

    # TODO consider multiple assignment
    # TODO rules for visibility (public, protected, private)
    rule(:assignment) { (reference >> spaces >> equals >> spaces >> phrase.as(:value)).as(:assignment) }

    #---------------------------------------------

    # FIXME invocation instead of regular_invocation
    rule(:object) { (block_expression | recursive_object | simple_object | regular_invocation | reference) >> property.repeat.as(:property_chain) }

    rule(:property) { dot >> (regular_invocation | reference) }

    # TODO rules for system, date, time, datetime, version, units?
{'fraction_non_alphanumeric': 0.17892707998715066, 'fraction_numerical': 0.00128493414712496, 'mean_word_length': 3.004501607717042, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 127, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '184951', 'n_tokens_mistral': 1707, 'n_tokens_neox': 1640, 'n_words': 435}