<commit_before>module Laser
  module SexpAnalysis
    # This is a *global* annotation, namely the one that determines the statically-known
    # scope for each node in the AST, at the time of that node's execution. For
    # example, every node should be able to say "hey scope, what's 'this' for this
    # statement?", and be able to return its type (*NOT* its class, they're different).
    #
    # Depends on: ExpandedIdentifierAnnotation
    # This is the annotator for the parent annotation.
    class ScopeAnnotation < BasicAnnotation
      add_property :scope
      depends_on :RuntimeAnnotation
      
      def annotate!(root)
        @current_scope = Scope::GlobalScope
        @visibility = :private
        super
      end
      
      # Replaces the general node visit method with one that assigns
      # the current scope to the visited node.
      def default_visit(node)
        node.scope = @current_scope
        visit_children(node)
      end
      
      # Load-time binding resolution. This should run *before* any method-matching, since
      # it directly affects method-matching!
      add :var_field do |node, ref|
        default_visit node
        node.binding = @current_scope.lookup(node.expanded_identifier)
      end
      
      # Here we handle Ruby's resolution of rvalues: if it looks like a local variable,
      # then we look it up, and if we fail, we assume it's a no-arg method call. If it
      # looks like a constant, then it's a constant!
      add :var_ref, :const_ref, :const_path_ref do |node, ref|
        default_visit node
        begin
          node.binding = @current_scope.lookup(node.expanded_identifier)
        rescue Scope::ScopeLookupFailure => err
          if err.query =~ /^[A-Z]/
            raise err
          end
        end
      end

      # Visits a module node and either creates or re-enters the corresponding scope, annotating the
      # body with that scope.
      add :module do |node, path_node, body|
        path_node, body = node.children
        default_visit(path_node)
        node.scope = @current_scope

        temp_cur_scope, new_mod_name = unpack_path(@current_scope, path_node)
        new_scope = lookup_or_create_module(temp_cur_scope, new_mod_name)
        if new_scope.self_ptr.klass == ClassRegistry['Class']
          node.errors << ReopenedClassAsModuleError.new("Opened class #{new_scope.self_ptr.name} as a module.", node)
        end
        with_visibility(:public) do
          visit_with_scope(body, new_scope)
        end
      end

      # Visits a class node and either creates or re-enters a corresponding scope, annotating the
      # body with that scope.
      # TODO(adgar): raise if this occurs within a method definition
      add :class do |node, path_node, superclass_node, body|
        # TODO(adgar): Make this do real lookup.
        visit superclass_node
        if superclass_node && superclass_node.is_constant
        then superclass = superclass_node.constant_value
        elsif superclass_node 
          raise DynamicSuperclassError.new(
              "Superclass of #{path_node.expanded_identifier}" +
              " is not a constant value. bad idea!", superclass_node)
        else superclass = ClassRegistry['Object']
        end
        default_visit(path_node)
        node.scope = @current_scope
        superclass_node.scope = @current_scope if superclass_node

        temp_cur_scope, new_class_name = unpack_path(@current_scope, path_node)
        new_scope = lookup_or_create_class(temp_cur_scope, new_class_name, superclass)
        if new_scope.self_ptr.klass != ClassRegistry['Class']
          node.errors << ReopenedModuleAsClassError.new("Opened module #{new_scope.self_ptr.name} as a class.", node)
        end
        with_visibility(:public) do
          visit_with_scope(body, new_scope)
        end
      end
      
      # Provides a general-purpose method for looking up a binding,
      # and yielding on failure.
      def lookup_or_create(scope, name)
        begin
          scope.lookup(name).value.scope
        rescue Scope::ScopeLookupFailure => err
          yield
        end
      end

      # Looks up a module, and creates it on failure.
      def lookup_or_create_module(scope, new_mod_name)
        lookup_or_create(scope, new_mod_name) do
          new_scope = ClosedScope.new(scope, nil)
          new_mod = LaserModule.new(ClassRegistry['Module'], new_scope, new_mod_name)
          new_scope
        end
      end

      # Looks up a class, and creates it on failure.
      def lookup_or_create_class(scope, new_class_name, superclass)
        lookup_or_create(scope, new_class_name) do
          new_scope = ClosedScope.new(scope, nil)
          new_class = LaserClass.new(ClassRegistry['Class'], new_scope, new_class_name) do |klass|
            klass.superclass = superclass
          end
          new_scope
        end
      end
      
      # Given a current scope and any possible way to describe a constant,
      # break it into two parts: the name of the final constant, and the
      # scope it will be in. The actual constant need not yet have an existing
      # object representing it yet – it will be lookup_or_created later.
      #
      # @param [Scope] current_scope the scope to look up the path in
      # @param [Sexp] path_node the node that describes the constant
      # @return [Array[Scope,String]] A tuple of the final scope and the
      #     name of the constant to use (as extracted from the AST)
      def unpack_path(current_scope, path_node)
        case path_node.type
        when :const_path_ref
          left, right = path_node.children
          new_class_name = right.expanded_identifier
          current_scope = current_scope.lookup(left.expanded_identifier).value.scope
        when :top_const_ref
          current_scope = Scope::GlobalScope
          new_class_name = path_node.expanded_identifier
        else
          new_class_name = path_node.expanded_identifier
        end
        [current_scope, new_class_name]
      end

      # Enter the singleton class.
      add :sclass do |node, singleton, body|
        visit singleton
        method_self = @current_scope.lookup(singleton.expanded_identifier).value
        receiver = method_self.singleton_class
        singleton.scope = @current_scope
        visit_with_scope(body, receiver.scope)
      end

      ######## Detecting includes - requires method call detection! ########
      match_method_call 'include' do |node, args|
        default_visit node
        if node.runtime == :load && @current_scope.self_ptr.klass.ancestors.include?(ClassRegistry['Module']) && args.is_constant?
          args.constant_values.reverse.each do |arg|
            @current_scope.self_ptr.include_module(arg)
          end
        end
      end
      
      match_method_call 'extend' do |node, args|
        default_visit node
        if node.runtime == :load && @current_scope.self_ptr.klass.ancestors.include?(ClassRegistry['Module']) && args.is_constant?
          args.constant_values.reverse.each do |arg|
            @current_scope.self_ptr.singleton_class.include_module(arg)
          end<commit_msg>include/exclude now use precise method detection. Beautiful.<commit_after>module Laser
  module SexpAnalysis
    # This is a *global* annotation, namely the one that determines the statically-known
    # scope for each node in the AST, at the time of that node's execution. For
    # example, every node should be able to say "hey scope, what's 'this' for this
    # statement?", and be able to return its type (*NOT* its class, they're different).
    #
    # Depends on: ExpandedIdentifierAnnotation
    # This is the annotator for the parent annotation.
    class ScopeAnnotation < BasicAnnotation
      add_property :scope
      depends_on :RuntimeAnnotation
      
      def annotate!(root)
        @current_scope = Scope::GlobalScope
        @visibility = :private
        super
      end
      
      # Replaces the general node visit method with one that assigns
      # the current scope to the visited node.
      def default_visit(node)
        node.scope = @current_scope
        visit_children(node)
      end
      
      # Load-time binding resolution. This should run *before* any method-matching, since
      # it directly affects method-matching!
      add :var_field do |node, ref|
        default_visit node
        node.binding = @current_scope.lookup(node.expanded_identifier)
      end
      
      # Here we handle Ruby's resolution of rvalues: if it looks like a local variable,
      # then we look it up, and if we fail, we assume it's a no-arg method call. If it
      # looks like a constant, then it's a constant!
      add :var_ref, :const_ref, :const_path_ref do |node, ref|
        default_visit node
        begin
          node.binding = @current_scope.lookup(node.expanded_identifier)
        rescue Scope::ScopeLookupFailure => err
          if err.query =~ /^[A-Z]/
            raise err
          end
        end
      end

      # Visits a module node and either creates or re-enters the corresponding scope, annotating the
      # body with that scope.
      add :module do |node, path_node, body|
        path_node, body = node.children
        default_visit(path_node)
        node.scope = @current_scope

        temp_cur_scope, new_mod_name = unpack_path(@current_scope, path_node)
        new_scope = lookup_or_create_module(temp_cur_scope, new_mod_name)
        if new_scope.self_ptr.klass == ClassRegistry['Class']
          node.errors << ReopenedClassAsModuleError.new("Opened class #{new_scope.self_ptr.name} as a module.", node)
        end
        with_visibility(:public) do
          visit_with_scope(body, new_scope)
        end
      end

      # Visits a class node and either creates or re-enters a corresponding scope, annotating the
      # body with that scope.
      # TODO(adgar): raise if this occurs within a method definition
      add :class do |node, path_node, superclass_node, body|
        # TODO(adgar): Make this do real lookup.
        visit superclass_node
        if superclass_node && superclass_node.is_constant
        then superclass = superclass_node.constant_value
        elsif superclass_node 
          raise DynamicSuperclassError.new(
              "Superclass of #{path_node.expanded_identifier}" +
              " is not a constant value. bad idea!", superclass_node)
        else superclass = ClassRegistry['Object']
        end
        default_visit(path_node)
        node.scope = @current_scope
        superclass_node.scope = @current_scope if superclass_node

        temp_cur_scope, new_class_name = unpack_path(@current_scope, path_node)
        new_scope = lookup_or_create_class(temp_cur_scope, new_class_name, superclass)
        if new_scope.self_ptr.klass != ClassRegistry['Class']
          node.errors << ReopenedModuleAsClassError.new("Opened module #{new_scope.self_ptr.name} as a class.", node)
        end
        with_visibility(:public) do
          visit_with_scope(body, new_scope)
        end
      end
      
      # Provides a general-purpose method for looking up a binding,
      # and yielding on failure.
      def lookup_or_create(scope, name)
        begin
          scope.lookup(name).value.scope
        rescue Scope::ScopeLookupFailure => err
          yield
        end
      end

      # Looks up a module, and creates it on failure.
      def lookup_or_create_module(scope, new_mod_name)
        lookup_or_create(scope, new_mod_name) do
          new_scope = ClosedScope.new(scope, nil)
          new_mod = LaserModule.new(ClassRegistry['Module'], new_scope, new_mod_name)
          new_scope
        end
      end

      # Looks up a class, and creates it on failure.
      def lookup_or_create_class(scope, new_class_name, superclass)
        lookup_or_create(scope, new_class_name) do
          new_scope = ClosedScope.new(scope, nil)
          new_class = LaserClass.new(ClassRegistry['Class'], new_scope, new_class_name) do |klass|
            klass.superclass = superclass
          end
          new_scope
        end
      end
      
      # Given a current scope and any possible way to describe a constant,
      # break it into two parts: the name of the final constant, and the
      # scope it will be in. The actual constant need not yet have an existing
      # object representing it yet – it will be lookup_or_created later.
      #
      # @param [Scope] current_scope the scope to look up the path in
      # @param [Sexp] path_node the node that describes the constant
      # @return [Array[Scope,String]] A tuple of the final scope and the
      #     name of the constant to use (as extracted from the AST)
      def unpack_path(current_scope, path_node)
        case path_node.type
        when :const_path_ref
          left, right = path_node.children
          new_class_name = right.expanded_identifier
          current_scope = current_scope.lookup(left.expanded_identifier).value.scope
        when :top_const_ref
          current_scope = Scope::GlobalScope
          new_class_name = path_node.expanded_identifier
        else
          new_class_name = path_node.expanded_identifier
        end
        [current_scope, new_class_name]
      end

      # Enter the singleton class.
      add :sclass do |node, singleton, body|
        visit singleton
        method_self = @current_scope.lookup(singleton.expanded_identifier).value
        receiver = method_self.singleton_class
        singleton.scope = @current_scope
        visit_with_scope(body, receiver.scope)
      end

      match_precise_loadtime_method(proc { 
            [ClassRegistry['Module'].instance_methods['include']]}) do |node, args|
        default_visit node
        if args.is_constant?
          args.constant_values.reverse.each do |arg|
            @current_scope.self_ptr.include_module(arg)
          end
        end
      end
      
      match_precise_loadtime_method(proc { 
            [ClassRegistry['Module'].instance_methods['extend']]}) do |node, args|
        default_visit node
        if args.is_constant?
          args.constant_values.reverse.each do |arg|
            @current_scope.self_ptr.singleton_class.include_module(arg)
          end
{'fraction_non_alphanumeric': 0.07001682085786375, 'fraction_numerical': 0.0, 'mean_word_length': 2.507620452310718, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '352424', 'n_tokens_mistral': 3998, 'n_tokens_neox': 3917, 'n_words': 1211}