<commit_before>class Configuratron

    attr_reader :keys, :overwrite, :files_updated

    def initialize(opts={})
        if opts[:secrets_file]
            begin
                @secrets = YAML::load_file(opts[:secrets_file])
            rescue Exception => e
                raise RuntimeError, "Yaml config file not found"
            end
        else
            @secrets = find_secrets
        end

        @files_updated = []

        @overwrite = opts.has_key?(:overwrite) ? opts[:overwrite] : false

        replace_config(opts[:json_dir])

    end

    def replace_config(dir)

        search_folder = File.expand_path(dir)
        files =  Dir.glob(search_folder + "/*.json")

        if files.empty?
            puts "Was unable to find any JSON files [#{search_folder}]"
        else
            Dir.glob(File.expand_path(dir) + "/*.json") do |json_file|

                next if json_file =~ /.new./

                node_data =  JSON.parse(File.read(json_file))

                modified = merge_hashes(@secrets, node_data)

                if modified
                  file_to_write = get_file_name(json_file)
                  files_updated << file_to_write
                  File.open(file_to_write, 'w') do |fh|
                     fh.puts JSON.pretty_generate(node_data)
                     fh.close
                  end
                end

            end
        end
    end

    private

    # Merge source into destination in-place
    # Only where they have keys in common
    # Taking care not to lose keys in destination but not source
    # Returns whether or not destination was modified
    def merge_hashes(source, destination, depth = 0)

      modified = false

      source.each_key do |key|

        if destination.has_key?(key)
          case source[key]
          when Hash
           modified = merge_hashes( source[key], destination[key], depth + 1 )
          when String
            destination[key] = source[key]
            modified = true
          else
            puts "Searching through source hash for a string but found #{source[key].class}"
            exit 1
          end
        else
          # intent is that if destination has a top-level key
          # and there is a subkey in source but not destination
          # it is added to destination
          destination[key] = source[key] if depth > 0
        end

      end

      if modified
        return true
      else
        return false
      end

    end

    def get_file_name(json_file)
        f = File.split(json_file)
        path = f[0]
        file = f[1]

        if @overwrite
            json_file
        else<commit_msg>Mark as modified if merged in value that did not exist<commit_after>class Configuratron

    attr_reader :keys, :overwrite, :files_updated

    def initialize(opts={})
        if opts[:secrets_file]
            begin
                @secrets = YAML::load_file(opts[:secrets_file])
            rescue Exception => e
                raise RuntimeError, "Yaml config file not found"
            end
        else
            @secrets = find_secrets
        end

        @files_updated = []

        @overwrite = opts.has_key?(:overwrite) ? opts[:overwrite] : false

        replace_config(opts[:json_dir])

    end

    def replace_config(dir)

        search_folder = File.expand_path(dir)
        files =  Dir.glob(search_folder + "/*.json")

        if files.empty?
            puts "Was unable to find any JSON files [#{search_folder}]"
        else
            Dir.glob(File.expand_path(dir) + "/*.json") do |json_file|

                next if json_file =~ /.new./

                node_data =  JSON.parse(File.read(json_file))

                modified = merge_hashes(@secrets, node_data)

                if modified
                  file_to_write = get_file_name(json_file)
                  files_updated << file_to_write
                  File.open(file_to_write, 'w') do |fh|
                     fh.puts JSON.pretty_generate(node_data)
                     fh.close
                  end
                end

            end
        end
    end

    private

    # merge source hash into destination hash
    # but skip any top-level hashes in source that are not in destination
    # return true if destination was modified, false otherwise

    def merge_hashes(source, destination, depth = 0)

      modified = false

      source.each_key do |key|

        if destination.has_key?(key)

          case source[key]
          when Hash
           modified = merge_hashes( source[key], destination[key], depth + 1 )
          when String
            destination[key] = source[key]
            modified = true
          else
            puts "Searching through source hash for a string but found #{source[key].class}"
            exit 1
          end

        else

          if depth > 0
            destination[key] = source[key]
            modified = true
          end

        end

      end

      if modified
        return true
      else
        return false
      end

    end

    def get_file_name(json_file)
        f = File.split(json_file)
        path = f[0]
        file = f[1]

        if @overwrite
            json_file
        else
{'fraction_non_alphanumeric': 0.07422324510932106, 'fraction_numerical': 0.0023014959723820483, 'mean_word_length': 1.5576262873957822, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '470679', 'n_tokens_mistral': 1482, 'n_tokens_neox': 1380, 'n_words': 408}