<commit_before>#--
# Copyright protects this work.
# See LICENSE file for details.
#++

require 'yaml'
require 'erbook/template'
require 'digest/sha1'

module ERBook
  class Document
    # Data from the format specification file.
    attr_reader :format

    # All root nodes in the document.
    attr_reader :roots

    # All nodes in the document.
    attr_reader :nodes

    # All nodes in the document arranged by node type.
    attr_reader :nodes_by_type

    ##
    # ==== Parameters
    #
    # [format_name]
    #   Either the short-hand name of a built-in format
    #   or the path to a format specification file.
    #
    # [input_text]
    #   The body of the input document.
    #
    # [input_file]
    #   Name of the file from which the input document originated.
    #
    # ==== Options
    #
    # [:unindent]
    #     If true, all node content is unindented hierarchically.
    #
    def initialize format_name, input_text, input_file, options = {}
      # process format specification
        @format_file = format_name.to_s

        File.file? @format_file or
          @format_file = File.join(ERBook::FORMATS_DIR, @format_file + '.yaml')

        begin
          @format = YAML.load_file(@format_file)
          @format[:file] = File.expand_path(@format_file)
          @format[:name] = File.basename(@format_file).sub(/\..*?$/, '')

          if @format.key? 'code'
            eval @format['code'].to_s, TOPLEVEL_BINDING, "#{@format_file}:code"
          end

        rescue Exception
          error "Could not load format specification file #{@format_file.inspect}"
        end

        @node_defs = @format['nodes']

      # process input document
      begin
        # create sandbox for input evaluation
          template = Template.new(input_file, input_text, options[:unindent])
          sandbox = template.sandbox

          @template_vars = {
            :@format        => @format,
            :@roots         => @roots = [], # root nodes of all trees
            :@nodes         => @nodes = [], # all nodes in the forest
            :@nodes_by_type => @nodes_by_type = Hash.new {|h,k| h[k] = [] },
            :@stack         => [], # stack for all nodes
          }.each_pair {|k,v| sandbox.instance_variable_set(k, v) }

          #:stopdoc:

          ##
          # Handles the method call from a node
          # placeholder in the input document.
          #
          def sandbox.__node_impl__ node_type, *node_args, &node_content
            node = Node.new(
              :type => node_type,
              :defn => @format['nodes'][node_type],
              :args => node_args,
              :children => [],

              # omit erbook internals from the stack trace
              :trace => caller.reject {|t|
                [$0, ERBook::INSTALL].any? {|f| t.index(f) == 0 }
              }
            )
            @nodes << node
            @nodes_by_type[node.type] << node

            # calculate occurrence number for this node
            if node.defn['number']
              @count_by_type ||= Hash.new {|h,k| h[k] = 0 }
              node.number = (@count_by_type[node.type] += 1)
            end

            # assign node family
            if parent = @stack.last
              parent.children << node<commit_msg>do not omit ERBook internals from node stack trace<commit_after>#--
# Copyright protects this work.
# See LICENSE file for details.
#++

require 'yaml'
require 'erbook/template'
require 'digest/sha1'

module ERBook
  class Document
    # Data from the format specification file.
    attr_reader :format

    # All root nodes in the document.
    attr_reader :roots

    # All nodes in the document.
    attr_reader :nodes

    # All nodes in the document arranged by node type.
    attr_reader :nodes_by_type

    ##
    # ==== Parameters
    #
    # [format_name]
    #   Either the short-hand name of a built-in format
    #   or the path to a format specification file.
    #
    # [input_text]
    #   The body of the input document.
    #
    # [input_file]
    #   Name of the file from which the input document originated.
    #
    # ==== Options
    #
    # [:unindent]
    #     If true, all node content is unindented hierarchically.
    #
    def initialize format_name, input_text, input_file, options = {}
      # process format specification
        @format_file = format_name.to_s

        File.file? @format_file or
          @format_file = File.join(ERBook::FORMATS_DIR, @format_file + '.yaml')

        begin
          @format = YAML.load_file(@format_file)
          @format[:file] = File.expand_path(@format_file)
          @format[:name] = File.basename(@format_file).sub(/\..*?$/, '')

          if @format.key? 'code'
            eval @format['code'].to_s, TOPLEVEL_BINDING, "#{@format_file}:code"
          end

        rescue Exception
          error "Could not load format specification file #{@format_file.inspect}"
        end

        @node_defs = @format['nodes']

      # process input document
      begin
        # create sandbox for input evaluation
          template = Template.new(input_file, input_text, options[:unindent])
          sandbox = template.sandbox

          @template_vars = {
            :@format        => @format,
            :@roots         => @roots = [], # root nodes of all trees
            :@nodes         => @nodes = [], # all nodes in the forest
            :@nodes_by_type => @nodes_by_type = Hash.new {|h,k| h[k] = [] },
            :@stack         => [], # stack for all nodes
          }.each_pair {|k,v| sandbox.instance_variable_set(k, v) }

          #:stopdoc:

          ##
          # Handles the method call from a node
          # placeholder in the input document.
          #
          def sandbox.__node_impl__ node_type, *node_args, &node_content
            node = Node.new(
              :type => node_type,
              :defn => @format['nodes'][node_type],
              :args => node_args,
              :children => [],
              :trace => caller
            )
            @nodes << node
            @nodes_by_type[node.type] << node

            # calculate occurrence number for this node
            if node.defn['number']
              @count_by_type ||= Hash.new {|h,k| h[k] = 0 }
              node.number = (@count_by_type[node.type] += 1)
            end

            # assign node family
            if parent = @stack.last
              parent.children << node
{'fraction_non_alphanumeric': 0.1103865859338612, 'fraction_numerical': 0.0012420431609998447, 'mean_word_length': 1.8656583629893237, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1166948', 'n_tokens_mistral': 1902, 'n_tokens_neox': 1830, 'n_words': 520}