<commit_before>require 'thor'

def load_dot_file!
  load(File.join(Dir.pwd, ".colonel"))
end

module Colonel
  class CLI < Thor

    desc "backup", "Backup content to an archive"
    def backup
      load_dot_file!

      index = DocumentIndex.new(Colonel.config.storage_path)
      docs = index.documents.map { |doc| Document.open(doc[:name]) }
      Serializer.generate(docs, STDOUT)
    end

    desc "restore", "Restore from a backup"
    method_option :input, type: :string, aliases: '-i'
    def restore
      load_dot_file!

      if options[:input]
        raise ArgumentError, "File input not implemented yet, consider using '< file'"
      else
        Serializer.load(STDIN)
      end
    end

    desc "index", "Index documents in document index into elasticsearch"
    method_option :index_name, type: :string, aliases: '-i'
    method_option :content_items, type: :array, aliases: '-c', required: true
    def index
      load_dot_file!

      Colonel.config.index_name = options[:input] if options[:input]

      index = DocumentIndex.new(Colonel.config.storage_path)
      docs = index.documents.map { |doc| Document.open(doc[:name]) }

      mapping = options[:content_items].map do |klass|
        klass = eval(klass)
        type_name = klass.item_type_name

        [type_name, klass]
      end

      Indexer.index(docs, Hash[mapping])
    end
  end
end<commit_msg>Make restore reindex as well<commit_after>require 'thor'

def load_dot_file!
  load(File.join(Dir.pwd, ".colonel"))
end

module Colonel
  class CLI < Thor

    desc "backup", "Backup content to an archive"
    def backup
      load_dot_file!

      index = DocumentIndex.new(Colonel.config.storage_path)
      docs = index.documents.map { |doc| Document.open(doc[:name]) }
      Serializer.generate(docs, STDOUT)
    end

    desc "restore", "Restore from a backup"
    method_option :input_file, type: :string, aliases: '-f'
    method_option :index_name, type: :string, aliases: '-i'
    method_option :content_items, type: :array, aliases: '-c', required: true
    def restore
      load_dot_file!

      if options[:input_file]
        raise ArgumentError, "File input not implemented yet, consider using '< file'"
      else
        Serializer.load(STDIN)
      end

      perform_indexing(options[:index_name], options[:content_items])
    end

    desc "index", "Index documents in document index into elasticsearch"
    method_option :index_name, type: :string, aliases: '-i'
    method_option :content_items, type: :array, aliases: '-c', required: true
    def index
      load_dot_file!

      perform_indexing(options[:index_name], options[:content_items])
    end

    private

    def perform_indexing(index_name, content_items)
      Colonel.config.index_name = index_name if index_name

      index = DocumentIndex.new(Colonel.config.storage_path)
      docs = index.documents.map { |doc| Document.open(doc[:name]) }

      mapping = content_items.map do |klass|
        klass = eval(klass)
        type_name = klass.item_type_name

        [type_name, klass]
      end

      Indexer.index(docs, Hash[mapping])
    end
  end
end
{'fraction_non_alphanumeric': 0.10543130990415335, 'fraction_numerical': 0.0, 'mean_word_length': 3.114323258869908, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5279321', 'n_tokens_mistral': 988, 'n_tokens_neox': 954, 'n_words': 254}