<commit_before>module Elasticsearch
  module Helpers 
    def Helpers.curl_request(method, url, parameters = nil)
      method.upcase!
      if parameters == nil
        `curl -X#{method} #{url}`
      else
        `curl -X#{method} #{url} #{parameters}`
      end
    end

    class Reader
      def initialize(templates_path)
        @templates_path = templates_path
      end
      def read_and_parse_json_file(file)
        JSON.parse File.read file
      rescue Exception => e
        puts "error while reading file #{file}"
        raise e
      end

      def read_and_parse_yaml_file(file)
        YAML.load File.read file
      rescue Exception => e
        puts "error while reading file #{file}"
        raise e
      end

      def read_and_parse_file(file)
        case File.extname(file)
        when ".json"
          read_and_parse_json_file(file)
        when ".yaml", ".yml"
          read_and_parse_yaml_file(file)
        end
      end

      def read_settings(mapping)
        filename = "#{@templates_path}#{mapping}/settings.yaml"
        if File.exist?(filename)
          read_and_parse_file(filename)
        else
          {}
        end
      end

      def read_mappings(mapping)
        mappings = {}

        Dir.chdir("#{@templates_path}#{mapping}/mappings/") do
          paths = Dir['*.{json,yml,yaml}']
          if default_file = paths.find { |f| f =~ /_default\.*/ }
            default = read_and_parse_file(default_file)
            paths.delete default_file
          end

          default ||= {}

          paths.each do |p|
            name, _ = p.split(".")
            content = read_and_parse_file p
            mappings[name] = default.deep_merge(content)
          end
        end

        mappings
      end

      # reads the template pattern from the file
      # see http://www.elasticsearch.org/guide/reference/api/admin-indices-templates/
      # for usage and format of the template pattern
      def read_template_pattern(mapping)
        filename = "#{@templates_path}#{mapping}/template_pattern"
        if File.exist?(filename)
          File.read(filename).chomp
        else
          nil
        end
      end

      def compile_template(name)
        require 'active_support/core_ext/hash/deep_merge'
        mappings = read_mappings(name)
        settings = read_settings(name)
        template_pattern = read_template_pattern(name)
        output = { "settings" => settings, "mappings" => mappings }
        output['template'] = template_pattern if template_pattern
        JSON.dump output
      end
    end
  end<commit_msg>Allows to disable getting default attributes for a type<commit_after>module Elasticsearch
  module Helpers 
    def Helpers.curl_request(method, url, parameters = nil)
      method.upcase!
      if parameters == nil
        `curl -X#{method} #{url}`
      else
        `curl -X#{method} #{url} #{parameters}`
      end
    end

    class Reader
      def initialize(templates_path)
        @templates_path = templates_path
      end
      def read_and_parse_json_file(file)
        JSON.parse File.read file
      rescue Exception => e
        puts "error while reading file #{file}"
        raise e
      end

      def read_and_parse_yaml_file(file)
        YAML.load File.read file
      rescue Exception => e
        puts "error while reading file #{file}"
        raise e
      end

      def read_and_parse_file(file)
        case File.extname(file)
        when ".json"
          read_and_parse_json_file(file)
        when ".yaml", ".yml"
          read_and_parse_yaml_file(file)
        end
      end

      def read_settings(mapping)
        filename = "#{@templates_path}#{mapping}/settings.yaml"
        if File.exist?(filename)
          read_and_parse_file(filename)
        else
          {}
        end
      end

      def read_mappings(mapping)
        mappings = {}

        Dir.chdir("#{@templates_path}#{mapping}/mappings/") do
          paths = Dir['*.{json,yml,yaml}']
          if default_file = paths.find { |f| f =~ /_default\.*/ }
            default = read_and_parse_file(default_file)
            paths.delete default_file
          end

          default ||= {}

          paths.each do |p|
            name, _ = p.split(".")
            content = read_and_parse_file p
            if content.fetch('inherit', true)
              mappings[name] = default.deep_merge(content)
            else
              mappings[name] = content
            end
          end
        end

        mappings
      end

      # reads the template pattern from the file
      # see http://www.elasticsearch.org/guide/reference/api/admin-indices-templates/
      # for usage and format of the template pattern
      def read_template_pattern(mapping)
        filename = "#{@templates_path}#{mapping}/template_pattern"
        if File.exist?(filename)
          File.read(filename).chomp
        else
          nil
        end
      end

      def compile_template(name)
        require 'active_support/core_ext/hash/deep_merge'
        mappings = read_mappings(name)
        settings = read_settings(name)
        template_pattern = read_template_pattern(name)
        output = { "settings" => settings, "mappings" => mappings }
        output['template'] = template_pattern if template_pattern
        JSON.dump output
      end
    end
  end
{'fraction_non_alphanumeric': 0.09187874251497007, 'fraction_numerical': 0.0, 'mean_word_length': 1.9497792494481236, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5297032', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1511, 'n_words': 358}