<commit_before>require 'parslet'

require 'rip'
require 'rip/parsers/helpers'
require 'rip/parsers/keyword'
require 'rip/parsers/reference'

module Rip::Parsers
  module Object
    include Parslet
    include Rip::Parsers::Helpers
    include Rip::Parsers::Keyword
    include Rip::Parsers::Reference

    # FIXME invocation instead of regular_invocation
    rule(:object) { (recursive_object | simple_object | structural_object | regular_invocation | reference) >> property.repeat.as(:property_chain) }

    rule(:property) { str('.') >> (regular_invocation | reference) }

    # TODO rules for system, date, time, datetime, version, units?
    rule(:simple_object) { numeric | character | string | regular_expression }

    rule(:recursive_object) { key_value_pair | range | hash_literal | list }

    rule(:structural_object) { class_literal | lambda_literal }

    #---------------------------------------------

    # WARNING order is important here: decimal must be before integer or the integral part of a decimal could be interpreted as a integer followed by a decimal starting with a '.' (dot)
    rule(:numeric) { decimal | integer }

    rule(:decimal) { (sign.maybe >> digits.maybe >> str('.') >> digits).as(:decimal) }

    rule(:integer) { (sign.maybe >> digits).as(:integer) }

    rule(:sign) { match['+-'] }

    rule(:digit) { match['0-9'] }

    # allow _ to be used to group digits ( 3_423_752 )
    # _ may not come first or last
    rule(:digits) { digit.repeat(1) >> (str('_').maybe >> digit.repeat(1)).repeat }

    #---------------------------------------------

    # FIXME should match any single printable unicode character
    rule(:character) { str('`') >> match['0-9a-zA-Z_'].as(:character) }

    #---------------------------------------------

    # NOTE a string is just a list with characters allowed in it
    rule(:string) { symbol_string | single_quoted_string | double_quoted_string | here_doc }

    # FIXME should match most (all?) non-whitespace characters
    rule(:symbol_string) { str(':') >> match['a-zA-Z_'].repeat(1).as(:string) }

    rule(:single_quoted_string) { str('\'') >> (str('\'').absent? >> any).repeat.as(:string) >> str('\'') }

    rule(:double_quoted_string) { str('"') >> (str('"').absent? >> any).repeat.as(:string) >> str('"') }

    rule(:here_doc) do
      label = match['A-Z_'].repeat(1)
      start = str('<<') >> label.as(:here_doc_start) >> eol
      content = (label.absent? >> any).repeat.as(:string)
      finish = label.as(:here_doc_end) >> eol.maybe
      start >> content >> finish
    end

    #---------------------------------------------

    # TODO expand regular expression pattern
    rule(:regular_expression) { str('/') >> (str('/').absent? >> any).repeat.as(:regex) >> str('/') }

    #---------------------------------------------

    # TODO allow type restriction
    # FIXME allow more types to be used as the key
    rule(:key_value_pair) { (simple_object | reference).as(:key) >> spaces? >> str(':') >> spaces? >> object.as(:value) }

    rule(:range) do
      rangable_object = integer | character | reference
      rangable_object.as(:start) >> str('..') >> str('.').maybe.as(:exclusivity) >> rangable_object.as(:end)
    end

    # NOTE a hash is just a list with only key_value_pairs allowed in it
    # TODO allow type restriction (to be passed on to key value pairs and list)
    rule(:hash_literal) { surround_with('{', thing_list(key_value_pair | reference).as(:hash), '}') }

    # TODO allow type restriction
    rule(:list) { surround_with('[', thing_list(object).as(:list), ']') }

    #---------------------------------------------

    rule(:class_literal) do
      ancestors = surround_with('(', thing_list(class_literal | reference).as(:ancestors).maybe, ')')
      (class_keyword >> whitespaces? >> ancestors.maybe >> whitespaces? >> block).as(:class)
    end

    rule(:lambda_literal) do<commit_msg>Stub in parsing code for future object literals<commit_after>require 'parslet'

require 'rip'
require 'rip/parsers/helpers'
require 'rip/parsers/keyword'
require 'rip/parsers/reference'

module Rip::Parsers
  module Object
    include Parslet
    include Rip::Parsers::Helpers
    include Rip::Parsers::Keyword
    include Rip::Parsers::Reference

    # FIXME invocation instead of regular_invocation
    rule(:object) { (recursive_object | simple_object | structural_object | regular_invocation | reference) >> property.repeat.as(:property_chain) }

    rule(:property) { str('.') >> (regular_invocation | reference) }

    # TODO rules for system, date, time, datetime, version, units?
    rule(:simple_object) { numeric | character | string | regular_expression }

    rule(:recursive_object) { key_value_pair | range | hash_literal | list }

    rule(:structural_object) { class_literal | lambda_literal }

    #---------------------------------------------

    # WARNING order is important here: decimal must be before integer or the integral part of a decimal could be interpreted as a integer followed by a decimal starting with a '.' (dot)
    rule(:numeric) { decimal | integer }

    rule(:decimal) { (sign.maybe >> digits.maybe >> str('.') >> digits).as(:decimal) }

    rule(:integer) { (sign.maybe >> digits).as(:integer) }

    rule(:sign) { match['+-'] }

    rule(:digit) { match['0-9'] }

    # allow _ to be used to group digits ( 3_423_752 )
    # _ may not come first or last
    rule(:digits) { digit.repeat(1) >> (str('_').maybe >> digit.repeat(1)).repeat }

    #---------------------------------------------

    # FIXME should match any single printable unicode character
    rule(:character) { str('`') >> match['0-9a-zA-Z_'].as(:character) }

    #---------------------------------------------

    # NOTE a string is just a list with characters allowed in it
    rule(:string) { symbol_string | single_quoted_string | double_quoted_string | here_doc }

    # FIXME should match most (all?) non-whitespace characters
    rule(:symbol_string) { str(':') >> match['a-zA-Z_'].repeat(1).as(:string) }

    rule(:single_quoted_string) { str('\'') >> (str('\'').absent? >> any).repeat.as(:string) >> str('\'') }

    rule(:double_quoted_string) { str('"') >> (str('"').absent? >> any).repeat.as(:string) >> str('"') }

    rule(:here_doc) do
      label = match['A-Z_'].repeat(1)
      start = str('<<') >> label.as(:here_doc_start) >> eol
      content = (label.absent? >> any).repeat.as(:string)
      finish = label.as(:here_doc_end) >> eol.maybe
      start >> content >> finish
    end

    #---------------------------------------------

    # TODO expand regular expression pattern
    rule(:regular_expression) { str('/') >> (str('/').absent? >> any).repeat.as(:regex) >> str('/') }

    #rule(:system) { '`' ( !'`' . / '\`' )* '`' }

    #rule(:version) do
    #  dot = str('.')
    #  (str('v') >> integer.as(:major) >> dot >> integer.as(:minor) >> dot >> integer.as(:patch)).as(:version)
    #end

    #---------------------------------------------

    #rule(:datetime) { date >> str('T') >> time }

    #rule(:date) do
    #  dash = str('-')
    #  (digit.repeat(4, 4).as(:year) >> dash >> digit.repeat(2, 2).as(:month) >> dash >> digit.repeat(2, 2).as(:day)).as(:date)
    #end

    ## TODO make second optional
    ## TODO handle fractional seconds (optional) and time zone offset (optional)
    #rule(:time) do
    #  colon = str(':')
    #  (digit.repeat(2, 2).as(:hour) >> colon >> digit.repeat(2, 2).as(:minute) >> colon >> digit.repeat(2, 2).as(:second)).as(:time)
    #end

    #---------------------------------------------

    # TODO allow type restriction
    # FIXME allow more types to be used as the key
    rule(:key_value_pair) { (simple_object | reference).as(:key) >> spaces? >> str(':') >> spaces? >> object.as(:value) }

    rule(:range) do
      rangable_object = integer | character | reference
      rangable_object.as(:start) >> str('..') >> str('.').maybe.as(:exclusivity) >> rangable_object.as(:end)
    end

    # NOTE a hash is just a list with only key_value_pairs allowed in it
    # TODO allow type restriction (to be passed on to key value pairs and list)
    rule(:hash_literal) { surround_with('{', thing_list(key_value_pair | reference).as(:hash), '}') }

    # TODO allow type restriction
    rule(:list) { surround_with('[', thing_list(object).as(:list), ']') }

    #---------------------------------------------

    rule(:class_literal) do
      ancestors = surround_with('(', thing_list(class_literal | reference).as(:ancestors).maybe, ')')
      (class_keyword >> whitespaces? >> ancestors.maybe >> whitespaces? >> block).as(:class)
    end

    rule(:lambda_literal) do
{'fraction_non_alphanumeric': 0.22473726758286175, 'fraction_numerical': 0.004850444624090542, 'mean_word_length': 2.6992738146091413, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 177, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '484250', 'n_tokens_mistral': 2617, 'n_tokens_neox': 2525, 'n_words': 630}