<commit_before>#--
# Copyright (c) 2012-2013, <NAME>, <EMAIL>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# Made in Japan.
#++

require 'neg/input'
require 'neg/errors'
require 'neg/translator'


module Neg

  class Parser

    def self.`(s)     ; StringParser.new(s); end
    def self._(o=nil) ; RegexParser.new(o); end

    def self.parser(&block)

      self.instance_eval(&block)
    end

    def self.translator(&block)

      @translator = Class.new(Neg::Translator)
      @translator.instance_eval(&block)
    end

    def self.method_missing(m, *args)

      return super if args.any?
      return super if m.to_s == 'to_ary'

      @root ||= m
      pa = NonTerminalParser.new(m)

      (class << self; self; end).__send__(:define_method, m) { pa }

      pa
    end

    def self.reduce(result)

      if result[0] && result[2] && result[3]
        result[4] =
          []
      else
        result[4] =
          result[4].each_with_object([]) { |cr, a| a << reduce(cr) if cr[2] }
      end

      result
    end

    def self.parse(s, opts={})

      i = Neg::Input(s)

      result = __send__(@root).parse(i, opts)

      result[2] = false if result[2] && ( ! i.eoi?)

      if @translator && opts[:translate] != false
        if result[2]
          @translator.translate(reduce(result))
        else
          raise ParseError.new(i, result)
        end
      elsif result[2] == false || opts[:noreduce]
        result
      else
        reduce(result)
      end
    end

    def self.recursive?

      nts = __send__(@root).non_terminals

      nts.size != nts.uniq.size
    end

    def self.to_s

      [
        "#{name}:",
        __send__(@root).non_terminals.uniq.collect { |nt|
          "  #{__send__(nt).to_s}"
        }
      ].join("\n")
    end

    #--
    # sub parsers
    #++

    class SubParser

      def +(pa)     ; SequenceParser.new(self, pa); end
      def |(pa)     ; AlternativeParser.new(self, pa); end
      def [](name)  ; NonTerminalParser.new(name.to_s, self); end
      def *(range)  ; RepetitionParser.new(self, range); end
      def ~         ; LookaheadParser.new(self, true); end
      def -@        ; LookaheadParser.new(self, false); end

      def child; @children.first; end

      def non_terminals(result=[])

        @children.each { |c| c.non_terminals(result) } if @children

        result
      end

      def parse(input_or_string, opts)

        input = Neg::Input(input_or_string)
        #start = input.position
        start = input.offset

        success, result, children = do_parse(input, opts)

        input.rewind(start) unless success

        [ @name, start, success, result, children ]
      end

      def recursive?

        false
      end
    end

    class NonTerminalParser < SubParser

      def initialize(name, child=nil)

        @name = name
        @children = child ? [ child ] : []
      end

      def non_terminals(result=[])

        result << @name if @name.is_a?(Symbol)
        super(result) if result.select { |s| s == @name }.size < 2

        result
      end<commit_msg>move recursive? implementation to sub parsers<commit_after>#--
# Copyright (c) 2012-2013, <NAME>, <EMAIL>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# Made in Japan.
#++

require 'neg/input'
require 'neg/errors'
require 'neg/translator'


module Neg

  class Parser

    def self.`(s)     ; StringParser.new(s); end
    def self._(o=nil) ; RegexParser.new(o); end

    def self.parser(&block)

      self.instance_eval(&block)
    end

    def self.translator(&block)

      @translator = Class.new(Neg::Translator)
      @translator.instance_eval(&block)
    end

    def self.method_missing(m, *args)

      return super if args.any?
      return super if m.to_s == 'to_ary'

      @root ||= m
      pa = NonTerminalParser.new(m)

      (class << self; self; end).__send__(:define_method, m) { pa }

      pa
    end

    def self.reduce(result)

      if result[0] && result[2] && result[3]
        result[4] =
          []
      else
        result[4] =
          result[4].each_with_object([]) { |cr, a| a << reduce(cr) if cr[2] }
      end

      result
    end

    def self.parse(s, opts={})

      i = Neg::Input(s)

      result = __send__(@root).parse(i, opts)

      result[2] = false if result[2] && ( ! i.eoi?)

      if @translator && opts[:translate] != false
        if result[2]
          @translator.translate(reduce(result))
        else
          raise ParseError.new(i, result)
        end
      elsif result[2] == false || opts[:noreduce]
        result
      else
        reduce(result)
      end
    end

    def self.recursive?

      __send__(@root).recursive?
    end

    def self.to_s

      [
        "#{name}:",
        __send__(@root).non_terminals.uniq.collect { |nt|
          "  #{__send__(nt).to_s}"
        }
      ].join("\n")
    end

    #--
    # sub parsers
    #++

    class SubParser

      def +(pa)     ; SequenceParser.new(self, pa); end
      def |(pa)     ; AlternativeParser.new(self, pa); end
      def [](name)  ; NonTerminalParser.new(name.to_s, self); end
      def *(range)  ; RepetitionParser.new(self, range); end
      def ~         ; LookaheadParser.new(self, true); end
      def -@        ; LookaheadParser.new(self, false); end

      def child; @children.first; end

      def parse(input_or_string, opts)

        input = Neg::Input(input_or_string)
        #start = input.position
        start = input.offset

        success, result, children = do_parse(input, opts)

        input.rewind(start) unless success

        [ @name, start, success, result, children ]
      end

      def non_terminals(result=[])

        @children.each { |c| c.non_terminals(result) } if @children

        result
      end

      def recursive?

        nts = non_terminals

        nts.size != nts.uniq.size
      end
    end

    class NonTerminalParser < SubParser

      def initialize(name, child=nil)

        @name = name
        @children = child ? [ child ] : []
      end

      def non_terminals(result=[])

        result << @name if @name.is_a?(Symbol)
        super(result) if result.select { |s| s == @name }.size < 2

        result
      end
{'fraction_non_alphanumeric': 0.11082693947144075, 'fraction_numerical': 0.004871513822920472, 'mean_word_length': 2.4245204336947457, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '184927', 'n_tokens_mistral': 2607, 'n_tokens_neox': 2453, 'n_words': 788}