<commit_before>require 'opal/parser/sexp'
require 'opal/parser/lexer'
require 'opal/parser/grammar'
require 'opal/parser/parser_scope'


module Opal
  class Parser < Racc::Parser
    attr_reader :lexer, :scope
    def parse(source, file = '(string)')
      @file = file
      @scopes = []
      @lexer = Lexer.new(source, file)
      @lexer.parser = self

      @yydebug = true

      self.parse_to_sexp
    end

    def parse_to_sexp
      push_scope
      result = do_parse
      pop_scope

      result
    end

    def next_token
      @lexer.next_token
    end

    def s(*parts)
      Sexp.new(parts)
    end

    def push_scope(type = nil)
      top = @scopes.last
      scope = ParserScope.new type
      scope.parent = top
      @scopes << scope
      @scope = scope
    end

    def pop_scope
      @scopes.pop
      @scope = @scopes.last
    end

    def value(tok)
      tok[0]
    end

    def source(tok)
      tok ? tok[1] : nil
    end

    def s0(type, source)
      sexp = s(type)
      sexp.source = source
      sexp
    end

    def s1(type, first, source)
      sexp = s(type, first)
      sexp.source = source
      sexp
    end

    def op_to_setter(op)
      "#{value(op)}=".to_sym
    end

    def new_ident(tok)
      s1(:identifier, value(tok).to_sym, source(tok))
    end

    def new_paren(open, expr, close)
      if expr.nil? or expr == [:block]
        s1(:paren, s0(:nil, source(open)), source(open))
      else
        s1(:paren, expr, source(open))
      end
    end

    def new_class(start, path, sup, body, endt)
      sexp = s(:class, path, sup, body)
      sexp.source = source(start)
      sexp
    end

    def new_colon2(lhs, tok, name)<commit_msg>def_spec part1<commit_after>require 'opal/parser/sexp'
require 'opal/parser/lexer'
require 'opal/parser/grammar'
require 'opal/parser/parser_scope'


module Opal
  class Parser < Racc::Parser
    attr_reader :lexer, :scope
    def parse(source, file = '(string)')
      @file = file
      @scopes = []
      @lexer = Lexer.new(source, file)
      @lexer.parser = self

      @yydebug = true

      self.parse_to_sexp
    end

    def parse_to_sexp
      push_scope
      result = do_parse
      pop_scope

      result
    end

    def next_token
      @lexer.next_token
    end

    def s(*parts)
      Sexp.new(parts)
    end

    def push_scope(type = nil)
      top = @scopes.last
      scope = ParserScope.new type
      scope.parent = top
      @scopes << scope
      @scope = scope
    end

    def pop_scope
      @scopes.pop
      @scope = @scopes.last
    end

    def value(tok)
      tok[0]
    end

    def source(tok)
      tok ? tok[1] : nil
    end

    def s0(type, source)
      sexp = s(type)
      sexp.source = source
      sexp
    end

    def s1(type, first, source)
      sexp = s(type, first)
      sexp.source = source
      sexp
    end

    def op_to_setter(op)
      "#{value(op)}=".to_sym
    end

    def new_ident(tok)
      s1(:identifier, value(tok).to_sym, source(tok))
    end

    def new_paren(open, expr, close)
      if expr.nil? or expr == [:block]
        s1(:paren, s0(:nil, source(open)), source(open))
      else
        s1(:paren, expr, source(open))
      end
    end

    def new_class(start, path, sup, body, endt)
      sexp = s(:class, path, sup, body)
      sexp.source = source(start)
      sexp
    end

    def new_def(kw, recv, name, args, body, end_tok)
      body = s(:block, body) if body.type != :block
      body << s(:nil) if body.size == 1

      sexp = s(:def, recv, value(name).to_sym, args, body)
      sexp.source = source(kw)
      sexp
    end

    def new_args(norm, opt, rest, block)
      res = s(:args)

      if norm
        norm.each do |arg|
          scope.add_local arg
          res << arg
        end
      end

      if opt
        opt[1..-1].each do |_opt|
          res << _opt[1]
        end
      end

      if rest
        res << rest
        rest_str = rest.to_s[1..-1]
        scope.add_local rest_str.to_sym unless rest_str.empty?
      end

      if block
        res << block
        scope.add_local block.to_s[1..-1].to_sym
      end

      res << opt if opt

      res
    end

    def new_colon2(lhs, tok, name)
{'fraction_non_alphanumeric': 0.10933777989518818, 'fraction_numerical': 0.0064316341114816575, 'mean_word_length': 1.8681693989071038, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6568023', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1479, 'n_words': 373}