<commit_before># encoding: UTF-8

namespace :db do
  namespace :articles do
    desc "Bulk-load articles from Crossref API"
    task :import => :environment do
      # only run if configuration option :import
      case ENV['IMPORT']
      when "member", "member_sample"
        member = ENV['MEMBER'] || Publisher.pluck(:crossref_id).join(",")
        sample = ENV['SAMPLE']
      when "all", "sample"
        member = ENV['MEMBER']
        sample = ENV['SAMPLE']
      when "sample", "member_sample"
        sample ||= 20
      else
        member = ENV['MEMBER']
        sample = ENV['SAMPLE'] || 20
      end

      options = { from_update_date: ENV['FROM_UPDATE_DATE'],
                  until_update_date: ENV['UNTIL_UPDATE_DATE'],
                  from_pub_date: ENV['FROM_PUB_DATE'],
                  until_pub_date: ENV['UNTIL_PUB_DATE'],
                  type: ENV['TYPE'],
                  member: member,
                  issn: ENV['ISSN'],
                  sample: sample }
      import = Import.new(options)
      number = ENV['SAMPLE'] || import.total_results
      import.queue_article_import if number.to_i > 0
      puts "Started import of #{number} articles in the background..."
    end

    desc "Bulk-load articles from standard input"
    task :load => :environment do
      input = []
      $stdin.each_line { |line| input << ActiveSupport::Multibyte::Unicode.tidy_bytes(line) } unless $stdin.tty?

      number = input.length
      member = ENV['MEMBER']
      if member.nil? && Publisher.pluck(:crossref_id).length == 1
        # if we have only configured a single publisher
        member = Publisher.pluck(:crossref_id).first
      end

      if number > 0
        # import in batches of 1,000 articles
        input.each_slice(1000) do |batch|
          import = Import.new(file: batch, member: member)
          import.queue_article_import
        end
        puts "Started import of #{number} articles in the background..."
      else
        puts "No articles to import."
      end
    end

    desc "Delete articles"
    task :delete => :environment do
      if ENV['MEMBER'].blank?
        puts "Please use MEMBER environment variable. No article deleted."
        exit
      end

      Article.queue_article_delete(ENV['MEMBER'])
      if ENV['MEMBER'] == "all"
        puts "Started deleting all articles in the background..."
      else
        puts "Started deleting all articles from MEMBER #{ENV['MEMBER']} in the background..."
      end
    end

    desc "Add missing sources"
    task :add_sources, [:date] => :environment do |_, args|
      if args.date.nil?
        puts "Date in format YYYY-MM-DD required"
        exit
      end

      articles = Article.where("published_on >= ?", args.date)

      if args.extras.empty?
        sources = Source.all
      else
        sources = Source.where("name in (?)", args.extras)
      end

      retrieval_statuses = []
      articles.each do |article|
        sources.each do |source|
          retrieval_status = RetrievalStatus.where(article_id: article.id, source_id: source.id).find_or_initialize
          if retrieval_status.new_record?
            retrieval_status.save!
            retrieval_statuses << retrieval_status
          end
        end
      end

      puts "#{retrieval_statuses.count} retrieval status(es) added for #{sources.count} source(s) and #{articles.count} articles"
    end

    desc "Remove all HTML and XML tags from article titles"
    task :sanitize_title => :environment do
      Article.all.each { |article| article.save }
      puts "#{Article.count} article titles sanitized"
    end

    desc "Add publication year, month and day"
    task :date_parts => :environment do
      begin
        start_date = Date.parse(ENV['START_DATE']) if ENV['START_DATE']
      rescue => e
        # raises error if invalid date supplied
        puts "Error: #{e.message}"
        exit
      end

      if start_date
        puts "Adding date parts for all articles published since #{start_date}."
        articles = Article.where("published_on >= ?", start_date)
      else
        articles = Article.all
      end

      articles.each do |article|
        article.update_date_parts
        article.save
      end
      puts "Date parts for #{articles.count} articles added"
    end
  end

  namespace :alerts do
    desc "Resolve all alerts with level INFO and WARN"
    task :resolve => :environment do
      Alert.unscoped do
        before = Alert.count
        Alert.where("level < 3").update_all(resolved: true)
        after = Alert.count
        puts "Deleted #{before - after} resolved alerts, #{after} unresolved alerts remaining"
      end
    end

    desc "Delete all resolved alerts"
    task :delete => :environment do
      Alert.unscoped do
        before = Alert.count
        Alert.where(:unresolved => false).delete_all
        after = Alert.count
        puts "Deleted #{before - after} resolved alerts, #{after} unresolved alerts remaining"
      end
    end
  end

  namespace :api_requests do
<commit_msg>automatically update all unresolved alerts of level INFO and WARN<commit_after># encoding: UTF-8

namespace :db do
  namespace :articles do
    desc "Bulk-load articles from Crossref API"
    task :import => :environment do
      # only run if configuration option :import
      case ENV['IMPORT']
      when "member", "member_sample"
        member = ENV['MEMBER'] || Publisher.pluck(:crossref_id).join(",")
        sample = ENV['SAMPLE']
      when "all", "sample"
        member = ENV['MEMBER']
        sample = ENV['SAMPLE']
      when "sample", "member_sample"
        sample ||= 20
      else
        member = ENV['MEMBER']
        sample = ENV['SAMPLE'] || 20
      end

      options = { from_update_date: ENV['FROM_UPDATE_DATE'],
                  until_update_date: ENV['UNTIL_UPDATE_DATE'],
                  from_pub_date: ENV['FROM_PUB_DATE'],
                  until_pub_date: ENV['UNTIL_PUB_DATE'],
                  type: ENV['TYPE'],
                  member: member,
                  issn: ENV['ISSN'],
                  sample: sample }
      import = Import.new(options)
      number = ENV['SAMPLE'] || import.total_results
      import.queue_article_import if number.to_i > 0
      puts "Started import of #{number} articles in the background..."
    end

    desc "Bulk-load articles from standard input"
    task :load => :environment do
      input = []
      $stdin.each_line { |line| input << ActiveSupport::Multibyte::Unicode.tidy_bytes(line) } unless $stdin.tty?

      number = input.length
      member = ENV['MEMBER']
      if member.nil? && Publisher.pluck(:crossref_id).length == 1
        # if we have only configured a single publisher
        member = Publisher.pluck(:crossref_id).first
      end

      if number > 0
        # import in batches of 1,000 articles
        input.each_slice(1000) do |batch|
          import = Import.new(file: batch, member: member)
          import.queue_article_import
        end
        puts "Started import of #{number} articles in the background..."
      else
        puts "No articles to import."
      end
    end

    desc "Delete articles"
    task :delete => :environment do
      if ENV['MEMBER'].blank?
        puts "Please use MEMBER environment variable. No article deleted."
        exit
      end

      Article.queue_article_delete(ENV['MEMBER'])
      if ENV['MEMBER'] == "all"
        puts "Started deleting all articles in the background..."
      else
        puts "Started deleting all articles from MEMBER #{ENV['MEMBER']} in the background..."
      end
    end

    desc "Add missing sources"
    task :add_sources, [:date] => :environment do |_, args|
      if args.date.nil?
        puts "Date in format YYYY-MM-DD required"
        exit
      end

      articles = Article.where("published_on >= ?", args.date)

      if args.extras.empty?
        sources = Source.all
      else
        sources = Source.where("name in (?)", args.extras)
      end

      retrieval_statuses = []
      articles.each do |article|
        sources.each do |source|
          retrieval_status = RetrievalStatus.where(article_id: article.id, source_id: source.id).find_or_initialize
          if retrieval_status.new_record?
            retrieval_status.save!
            retrieval_statuses << retrieval_status
          end
        end
      end

      puts "#{retrieval_statuses.count} retrieval status(es) added for #{sources.count} source(s) and #{articles.count} articles"
    end

    desc "Remove all HTML and XML tags from article titles"
    task :sanitize_title => :environment do
      Article.all.each { |article| article.save }
      puts "#{Article.count} article titles sanitized"
    end

    desc "Add publication year, month and day"
    task :date_parts => :environment do
      begin
        start_date = Date.parse(ENV['START_DATE']) if ENV['START_DATE']
      rescue => e
        # raises error if invalid date supplied
        puts "Error: #{e.message}"
        exit
      end

      if start_date
        puts "Adding date parts for all articles published since #{start_date}."
        articles = Article.where("published_on >= ?", start_date)
      else
        articles = Article.all
      end

      articles.each do |article|
        article.update_date_parts
        article.save
      end
      puts "Date parts for #{articles.count} articles added"
    end
  end

  namespace :alerts do
    desc "Resolve all alerts with level INFO and WARN"
    task :resolve => :environment do
      Alert.unscoped do
        before = Alert.count
        Alert.where("level < 3").update_all(unresolved: false)
        after = Alert.count
        puts "Deleted #{before - after} resolved alerts, #{after} unresolved alerts remaining"
      end
    end

    desc "Delete all resolved alerts"
    task :delete => :environment do
      Alert.unscoped do
        before = Alert.count
        Alert.where(:unresolved => false).delete_all
        after = Alert.count
        puts "Deleted #{before - after} resolved alerts, #{after} unresolved alerts remaining"
      end
    end
  end

  namespace :api_requests do

{'fraction_non_alphanumeric': 0.09928100068945139, 'fraction_numerical': 0.0033487639121441938, 'mean_word_length': 2.2712628865979383, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7415361', 'n_tokens_mistral': 2945, 'n_tokens_neox': 2857, 'n_words': 867}