<commit_before>namespace :db do
  namespace :articles do
    desc "Bulk-load works from standard input"
    task :load => :environment do
      begin
        input = []
        $stdin.each_line { |line| input << ActiveSupport::Multibyte::Unicode.tidy_bytes(line) } unless $stdin.tty?
        number = input.length
        fail Errno::ENOENT, "No works to import." if input.empty?
      rescue Errno::ENOENT => e
        puts e.message
        exit
      end

      member = ENV['MEMBER']
      if member.blank? && Publisher.pluck(:member_id).length == 1
        # if we have only configured a single publisher
        member = Publisher.pluck(:member_id).join(",")
      end

      # import in batches of 1,000 works
      input.each_slice(1000) do |batch|
        import = FileImport.new(file: batch, member: member)
        import.queue_work_import
      end
      puts "Started import of #{number} works in the background..."
    end
  end

  namespace :works do
    namespace :import do
      desc "Import works from DataCite API"
      task :datacite => :environment do
        case ENV['IMPORT'].to_s.downcase
        when "member"
          member = ENV['MEMBER'].presence || Publisher.pluck(:name).join(",")
        else
          member = ENV['MEMBER'].presence
        end

        import = DataciteImport.new(
          from_update_date: ENV['FROM_UPDATE_DATE'],
          until_update_date: ENV['UNTIL_UPDATE_DATE'],
          from_pub_date: ENV['FROM_PUB_DATE'],
          until_pub_date: ENV['UNTIL_PUB_DATE'],
          type: ENV['TYPE'],
          member: member)
        number = import.total_results
        if number > 0
          import.queue_work_import
          puts "Started import of #{number} works in the background..."
        else
          puts "No works to import."
        end
      end

      desc "Import works with RelatedIdentifiers from DataCite API"
      task :datacite_related => :environment do
        import = DataciteRelated.new(
          from_date: ENV['FROM_UPDATE_DATE'],
          until_date: ENV['UNTIL_UPDATE_DATE'])
        number = import.total_results
        if number > 0
          import.queue_work_import
          puts "Started import of #{number} works in the background..."
        else
          puts "No works to import."
        end
      end

      desc "Import works from DataONE Solr API"
      task :dataone => :environment do
        import = DataoneImport.new(
          from_pub_date: ENV['FROM_PUB_DATE'],
          until_pub_date: ENV['UNTIL_PUB_DATE'],
          from_update_date: ENV['FROM_UPDATE_DATE'],
          until_update_date: ENV['UNTIL_UPDATE_DATE'])
        number = import.total_results
        if number > 0
          import.queue_work_import
          puts "Started import of #{number} works in the background..."
        else<commit_msg>pass import options to rake task. #414<commit_after>namespace :db do
  namespace :articles do
    desc "Bulk-load works from standard input"
    task :load => :environment do
      begin
        input = []
        $stdin.each_line { |line| input << ActiveSupport::Multibyte::Unicode.tidy_bytes(line) } unless $stdin.tty?
        number = input.length
        fail Errno::ENOENT, "No works to import." if input.empty?
      rescue Errno::ENOENT => e
        puts e.message
        exit
      end

      member = ENV['MEMBER']
      if member.blank? && Publisher.pluck(:member_id).length == 1
        # if we have only configured a single publisher
        member = Publisher.pluck(:member_id).join(",")
      end

      # import in batches of 1,000 works
      input.each_slice(1000) do |batch|
        import = FileImport.new(file: batch, member: member)
        import.queue_work_import
      end
      puts "Started import of #{number} works in the background..."
    end
  end

  namespace :works do
    namespace :import do
      desc "Import works from DataCite API"
      task :datacite => :environment do
        case ENV['IMPORT'].to_s.downcase
        when "member"
          member = ENV['MEMBER'].presence || Publisher.pluck(:name).join(",")
        else
          member = ENV['MEMBER'].presence
        end

        import = DataciteImport.new(
          from_update_date: ENV['FROM_UPDATE_DATE'],
          until_update_date: ENV['UNTIL_UPDATE_DATE'],
          from_pub_date: ENV['FROM_PUB_DATE'],
          until_pub_date: ENV['UNTIL_PUB_DATE'],
          type: ENV['TYPE'],
          member: member)
        number = import.total_results
        if number > 0
          import.queue_work_import
          puts "Started import of #{number} works in the background..."
        else
          puts "No works to import."
        end
      end

      desc "Import works with RelatedIdentifiers from DataCite API"
      task :datacite_related => :environment do
        import = DataciteRelated.where(name: 'datacite_related').first
        options = { from_date: ENV['FROM_UPDATE_DATE'],
                    until_date: ENV['UNTIL_UPDATE_DATE'] }
        number = import.total_results(options)
        if number > 0
          import.queue_work_import(options)
          puts "Started import of #{number} works in the background..."
        else
          puts "No works to import."
        end
      end

      desc "Import works from DataONE Solr API"
      task :dataone => :environment do
        import = DataoneImport.new(
          from_pub_date: ENV['FROM_PUB_DATE'],
          until_pub_date: ENV['UNTIL_PUB_DATE'],
          from_update_date: ENV['FROM_UPDATE_DATE'],
          until_update_date: ENV['UNTIL_UPDATE_DATE'])
        number = import.total_results
        if number > 0
          import.queue_work_import
          puts "Started import of #{number} works in the background..."
        else
{'fraction_non_alphanumeric': 0.09619064185075665, 'fraction_numerical': 0.004696468951121934, 'mean_word_length': 2.2230941704035874, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '870156', 'n_tokens_mistral': 1681, 'n_tokens_neox': 1673, 'n_words': 461}