<commit_before>require 'ostruct'

module Elasticsearch
  module Yaml
    class Parser
      NODE_NAME = 'inherit'
      TAG_NAME  = '!file'

      def load_file(file)
        parse_file(file).to_ruby
      end

      def parse_file(file)
        document = parse_yaml_content(File.read(file))
        replace_inherit_node(document)
        document
      rescue StandardError => e
        # STDOUT.puts "Error while reading file #{file}: #{e}"
        raise e
      end

      def parse_yaml_content(content)
        YAML.parse(content).root
      end

      def replace_inherit_node(document)
        document.grep(Psych::Nodes::Mapping).each do |node|
          nodes = node.children
          find_file_include_nodes(nodes).reverse.each do |pair|
            index = nodes.index(pair.item)

            nodes.delete_if{ |n| n == pair.item || n == pair.file }

            content = parse_file(pair.file.value)
            content.children.each do |c|
              nodes.insert(index, c)
              index += 1
            end
          end
        end
      end

      private

      def insertion_node?(n, next_node)
        n.class == Psych::Nodes::Scalar &&
        n.value == NODE_NAME &&
        next_node &&
        next_node.respond_to?(:tag) &&
        next_node.tag == TAG_NAME
      end

      def find_file_include_nodes(nodes)
        nodes.each_index.each_with_object([]) do |index, result|
          m, n = nodes.slice(index, 2)
          if insertion_node?(m, n)
            result << OpenStruct.new(item: m, file: n)
          end
        end
      end
    end
  end
end<commit_msg>Refactored replacement of nodes in the AST tree<commit_after>require 'ostruct'

module Elasticsearch
  module Yaml
    class Parser
      NODE_NAME = 'inherit'
      TAG_NAME  = '!file'

      def load_file(file)
        parse_file(file).to_ruby
      end

      def parse_file(file)
        document = parse_yaml_content(File.read(file))
        replace_file_include_nodes(document)
        document
      rescue StandardError => e
        # STDOUT.puts "Error while reading file #{file}: #{e}"
        raise e
      end

      def parse_yaml_content(content)
        YAML.parse(content).root
      end

      def replace_file_include_nodes(document)
        document.grep(Psych::Nodes::Mapping).each do |n|
          n.children.tap do |nodes|
            find_file_include_nodes(nodes).reverse.each do |node|
              # remove the pair of Scalar nodes
              nodes.delete(node.item)
              nodes.delete(node.file)

              # and replace it with the content from the external yaml
              content = parse_file(node.file.value)
              content.children.each_with_index do |item, index|
                nodes.insert(node.index + index, item)
              end
            end
          end
        end
      end

      private

      def insertion_node?(n, next_node)
        n.class == Psych::Nodes::Scalar &&
        n.value == NODE_NAME &&
        next_node &&
        next_node.respond_to?(:tag) &&
        next_node.tag == TAG_NAME
      end

      def find_file_include_nodes(nodes)
        nodes.each_index.each_with_object([]) do |index, result|
          m, n = nodes.slice(index, 2)
          if insertion_node?(m, n)
            result << OpenStruct.new(item: m, file: n, index: index)
          end
        end
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.09233038348082596, 'fraction_numerical': 0.0008849557522123894, 'mean_word_length': 1.8640202702702702, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3941637', 'n_tokens_mistral': 1030, 'n_tokens_neox': 995, 'n_words': 234}