<commit_before>module CoffeeScript

  # The abstract base class for all CoffeeScript nodes.
  class Node
    # Tabs are two spaces for pretty-printing.
    TAB = '  '

    # Tag this node as a statement, meaning that it can't be used directly as
    # the result of an expression.
    def self.statement
      class_eval "def statement?; true; end"
    end

    # Tag this node as a statement that cannot be transformed into an expression.
    # (break, continue, etc.) It doesn't make sense to try to transform it.
    def self.statement_only
      statement
      class_eval "def statement_only?; true; end"
    end

    def write(code)
      puts "#{self.class.to_s}:\n#{@options.inspect}\n#{code}\n\n" if ENV['VERBOSE']
      code
    end

    # This is extremely important -- we convert JS statements into expressions
    # by wrapping them in a closure, only if it's possible, and we're not at
    # the top level of a block (which would be unnecessary), and we haven't
    # already been asked to return the result.
    def compile(o={})
      @options = o.dup
      top = @options.delete(:top)
      closure = statement? && !statement_only? && !top && !@options[:return]
      closure ? compile_closure(@options) : compile_node(@options)
    end

    def compile_closure(o={})
      indent = o[:indent]
      o[:indent] += TAB
      "(function() {\n#{compile_node(o.merge(:return => true))}\n#{indent}})()"
    end

    # Default implementations of the common node methods.
    def unwrap;           self;   end
    def statement?;       false;  end
    def statement_only?;  false;  end
  end

  # A collection of nodes, each one representing an expression.
  class Expressions < Node
    statement
    attr_reader :expressions

    STRIP_TRAILING_WHITESPACE = /\s+$/

    # Wrap up a node as an Expressions, unless it already is.
    def self.wrap(*nodes)
      return nodes[0] if nodes.length == 1 && nodes[0].is_a?(Expressions)
      Expressions.new(*nodes)
    end

    def initialize(*nodes)
      @expressions = nodes.flatten
    end

    # Tack an expression onto the end of this node.
    def <<(node)
      @expressions << node
      self
    end

    def unshift(node)
      @expressions.unshift(node)
      self
    end

    # If this Expressions consists of a single node, pull it back out.
    def unwrap
      @expressions.length == 1 ? @expressions.first : self
    end

    # Is the node last in this block of expressions.
    def last?(node)
      @last_index ||= @expressions.last.is_a?(CommentNode) ? -2 : -1
      node == @expressions[@last_index]
    end

    def compile(o={})
      o[:scope] ? super(o) : compile_root(o)
    end

    # The extra fancy is to handle pushing down returns to the final lines of
    # inner statements. Variables first defined within the Expressions body
    # have their declarations pushed up top of the closest scope.
    def compile_node(options={})
      compiled = @expressions.map do |node|
        o = options.dup
        returns = o.delete(:return)
        code = node.compile(o)
        if last?(node) && returns && !node.statement_only?
          if node.statement?
            node.compile(o.merge(:return => true))
          else
            "#{o[:indent]}return #{node.compile(o)};"
          end
        else
          ending = node.statement? ? '' : ';'
          indent = node.statement? ? '' : o[:indent]
          "#{indent}#{node.compile(o.merge(:top => true))}#{ending}"
        end
      end
      write(compiled.join("\n"))
    end

    # If this is the top-level Expressions, wrap everything in a safety closure.
    def compile_root(o={})
      indent = o[:no_wrap] ? '' : TAB
      o.merge!(:indent => indent, :scope => Scope.new(nil, self))
      code = o[:no_wrap] ? compile_node(o) : compile_with_declarations(o)
      code.gsub!(STRIP_TRAILING_WHITESPACE, '')
      o[:no_wrap] ? code : "(function(){\n#{code}\n})();"
    end

    def compile_with_declarations(o={})
      code  = compile_node(o)
      decls = ''
      decls = "#{o[:indent]}var #{o[:scope].declared_variables.join(', ')};\n" if o[:scope].declarations?(self)<commit_msg>logic error in Expressions was causing over-compilation by a factor of the depth of the tree<commit_after>module CoffeeScript

  # The abstract base class for all CoffeeScript nodes.
  class Node
    # Tabs are two spaces for pretty-printing.
    TAB = '  '

    # Tag this node as a statement, meaning that it can't be used directly as
    # the result of an expression.
    def self.statement
      class_eval "def statement?; true; end"
    end

    # Tag this node as a statement that cannot be transformed into an expression.
    # (break, continue, etc.) It doesn't make sense to try to transform it.
    def self.statement_only
      statement
      class_eval "def statement_only?; true; end"
    end

    def write(code)
      puts "#{self.class.to_s}:\n#{@options.inspect}\n#{code}\n\n" if ENV['VERBOSE']
      code
    end

    # This is extremely important -- we convert JS statements into expressions
    # by wrapping them in a closure, only if it's possible, and we're not at
    # the top level of a block (which would be unnecessary), and we haven't
    # already been asked to return the result.
    def compile(o={})
      @options = o.dup
      top = @options.delete(:top)
      closure = statement? && !statement_only? && !top && !@options[:return]
      closure ? compile_closure(@options) : compile_node(@options)
    end

    def compile_closure(o={})
      indent = o[:indent]
      o[:indent] += TAB
      "(function() {\n#{compile_node(o.merge(:return => true))}\n#{indent}})()"
    end

    # Default implementations of the common node methods.
    def unwrap;           self;   end
    def statement?;       false;  end
    def statement_only?;  false;  end
  end

  # A collection of nodes, each one representing an expression.
  class Expressions < Node
    statement
    attr_reader :expressions

    STRIP_TRAILING_WHITESPACE = /\s+$/

    # Wrap up a node as an Expressions, unless it already is.
    def self.wrap(*nodes)
      return nodes[0] if nodes.length == 1 && nodes[0].is_a?(Expressions)
      Expressions.new(*nodes)
    end

    def initialize(*nodes)
      @expressions = nodes.flatten
    end

    # Tack an expression onto the end of this node.
    def <<(node)
      @expressions << node
      self
    end

    def unshift(node)
      @expressions.unshift(node)
      self
    end

    # If this Expressions consists of a single node, pull it back out.
    def unwrap
      @expressions.length == 1 ? @expressions.first : self
    end

    # Is the node last in this block of expressions.
    def last?(node)
      @last_index ||= @expressions.last.is_a?(CommentNode) ? -2 : -1
      node == @expressions[@last_index]
    end

    def compile(o={})
      o[:scope] ? super(o) : compile_root(o)
    end

    # The extra fancy is to handle pushing down returns to the final lines of
    # inner statements. Variables first defined within the Expressions body
    # have their declarations pushed up top of the closest scope.
    def compile_node(options={})
      compiled = @expressions.map do |node|
        o = options.dup
        returns = o.delete(:return)
        if last?(node) && returns && !node.statement_only?
          if node.statement?
            node.compile(o.merge(:return => true))
          else
            "#{o[:indent]}return #{node.compile(o)};"
          end
        else
          ending = node.statement? ? '' : ';'
          indent = node.statement? ? '' : o[:indent]
          "#{indent}#{node.compile(o.merge(:top => true))}#{ending}"
        end
      end
      write(compiled.join("\n"))
    end

    # If this is the top-level Expressions, wrap everything in a safety closure.
    def compile_root(o={})
      indent = o[:no_wrap] ? '' : TAB
      o.merge!(:indent => indent, :scope => Scope.new(nil, self))
      code = o[:no_wrap] ? compile_node(o) : compile_with_declarations(o)
      code.gsub!(STRIP_TRAILING_WHITESPACE, '')
      o[:no_wrap] ? code : "(function(){\n#{code}\n})();"
    end

    def compile_with_declarations(o={})
      code  = compile_node(o)
      decls = ''
      decls = "#{o[:indent]}var #{o[:scope].declared_variables.join(', ')};\n" if o[:scope].declarations?(self)
{'fraction_non_alphanumeric': 0.11885838150289017, 'fraction_numerical': 0.001445086705202312, 'mean_word_length': 2.593682388576374, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4196744', 'n_tokens_mistral': 2537, 'n_tokens_neox': 2492, 'n_words': 802}