<commit_before>module Flex
  class Tasks


    attr_reader :options

    def initialize(overrides={})
      options = Flex::Utils.env2options *default_options.keys

      options[:timeout]    = options[:timeout].to_i      if options[:timeout]
      options[:batch_size] = options[:batch_size].to_i   if options[:batch_size]
      options[:index]      = options[:index].split(',')  if options[:index]
      options[:models]     = options[:models].split(',') if options[:models]

      if options[:import_options]
        import_options = {}
        options[:import_options].split('&').each do |pair|
          k, v  = pair.split('=')
          import_options[k.to_sym] = v
        end
        options[:import_options] = import_options
      end

      @options = default_options.merge(options).merge(overrides)
    end

    def default_options
      @default_options ||= { :force          => false,
                             :timeout        => 20,
                             :batch_size     => 1000,
                             :import_options => { },
                             :index          => Conf.variables[:index],
                             :models         => Conf.flex_models,
                             :config_file    => Conf.config_file,
                             :verbose        => true }
    end

    def create_indices
      indices.each do |index|
        delete_index(index) if options[:force]
        raise ExistingIndexError, "#{index.inspect} already exists. Please use FORCE=1 if you want to delete it first." \
              if exist?(index)
        create(index)
      end
    end

    def delete_indices
      indices.each { |index| delete_index(index) }
    end

    def import_models
      Conf.http_client_options[:timeout] = options[:timeout]
      deleted = []
      models.each do |klass|
        index = klass.flex.index
        next unless options[:index].include?(index)

        if options[:force]
          unless deleted.include?(index)
            delete_index(index)
            deleted << index
            puts "#{index} index deleted"
          end
        end

        unless exist?(index)
          create(index)
          puts "#{index} index created"
        end

        if defined?(Mongoid::Document) && klass.ancestors.include?(Mongoid::Document)
          def klass.find_in_batches(options={})
            0.step(count, options[:batch_size]) do |offset|
              yield limit(options[:batch_size]).skip(offset).to_a
            end
          end
        end

        unless klass.respond_to?(:find_in_batches)
          Conf.logger.error "Class #{klass} does not respond to :find_in_batches. Skipped."
          next
        end

        pbar = ProgBar.new(klass.count, options[:batch_size], "Class #{klass}: ")

        klass.find_in_batches(:batch_size => options[:batch_size]) do |array|
          opts   = {:index => index}.merge(options[:import_options])
          result = Flex.import_collection(array, opts) || next
          pbar.process_result(result, array.size)
        end

        pbar.finish
      end
    end

    private

    def indices
      options[:index] || struct.keys
    end

    def exist?(index)
      Flex.exist?(:index => index)
    end

    def struct
      @struct ||= ( @indices_yaml = options[:config_file] || Conf.config_file
                    raise Errno::ENOENT, "no such file or directory #{@indices_yaml.inspect}. " +<commit_msg>fix for indices method<commit_after>module Flex
  class Tasks


    attr_reader :options

    def initialize(overrides={})
      options = Flex::Utils.env2options *default_options.keys

      options[:timeout]    = options[:timeout].to_i      if options[:timeout]
      options[:batch_size] = options[:batch_size].to_i   if options[:batch_size]
      options[:index]      = options[:index].split(',')  if options[:index]
      options[:models]     = options[:models].split(',') if options[:models]

      if options[:import_options]
        import_options = {}
        options[:import_options].split('&').each do |pair|
          k, v  = pair.split('=')
          import_options[k.to_sym] = v
        end
        options[:import_options] = import_options
      end

      @options = default_options.merge(options).merge(overrides)
    end

    def default_options
      @default_options ||= { :force          => false,
                             :timeout        => 20,
                             :batch_size     => 1000,
                             :import_options => { },
                             :index          => Conf.variables[:index],
                             :models         => Conf.flex_models,
                             :config_file    => Conf.config_file,
                             :verbose        => true }
    end

    def create_indices
      indices.each do |index|
        delete_index(index) if options[:force]
        raise ExistingIndexError, "#{index.inspect} already exists. Please use FORCE=1 if you want to delete it first." \
              if exist?(index)
        create(index)
      end
    end

    def delete_indices
      indices.each { |index| delete_index(index) }
    end

    def import_models
      Conf.http_client_options[:timeout] = options[:timeout]
      deleted = []
      models.each do |klass|
        index = klass.flex.index
        next unless options[:index].include?(index)

        if options[:force]
          unless deleted.include?(index)
            delete_index(index)
            deleted << index
            puts "#{index} index deleted"
          end
        end

        unless exist?(index)
          create(index)
          puts "#{index} index created"
        end

        if defined?(Mongoid::Document) && klass.ancestors.include?(Mongoid::Document)
          def klass.find_in_batches(options={})
            0.step(count, options[:batch_size]) do |offset|
              yield limit(options[:batch_size]).skip(offset).to_a
            end
          end
        end

        unless klass.respond_to?(:find_in_batches)
          Conf.logger.error "Class #{klass} does not respond to :find_in_batches. Skipped."
          next
        end

        pbar = ProgBar.new(klass.count, options[:batch_size], "Class #{klass}: ")

        klass.find_in_batches(:batch_size => options[:batch_size]) do |array|
          opts   = {:index => index}.merge(options[:import_options])
          result = Flex.import_collection(array, opts) || next
          pbar.process_result(result, array.size)
        end

        pbar.finish
      end
    end

    private

    def indices
      i = options[:index] || struct.keys
      i.is_a?(Array) ? i : [i]
    end

    def exist?(index)
      Flex.exist?(:index => index)
    end

    def struct
      @struct ||= ( @indices_yaml = options[:config_file] || Conf.config_file
                    raise Errno::ENOENT, "no such file or directory #{@indices_yaml.inspect}. " +
{'fraction_non_alphanumeric': 0.11209140873589818, 'fraction_numerical': 0.0026034133641886026, 'mean_word_length': 1.8884711779448622, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5151439', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1910, 'n_words': 419}