<commit_before>require_relative '../statistics.rb'

namespace :statistics do
  desc '月次のイベント履歴を集計します'
  task :aggregation, [:from, :to] => :environment do |tasks, args|
    date_from_str = -> (str) {
      formats = %w(%Y%m %Y/%m %Y-%m)
      d = formats.map { |fmt|
        begin
          Time.zone.strptime(str, fmt)
        rescue ArgumentError
          Time.zone.local(str) if str.length == 4
        end
      }.compact.first
      raise ArgumentError, "Invalid format: `#{str}`, allow format is #{formats.push('%Y').join(' or ')}" if d.nil?
      d
    }

    from = if args[:from]
             if args[:from].length == 4
               date_from_str.call(args[:from]).beginning_of_year
             else
               date_from_str.call(args[:from]).beginning_of_month
             end
           else
             Time.current.prev_month.beginning_of_month
           end
    to = if args[:to]
           if args[:to].length == 4
             date_from_str.call(args[:to]).end_of_year
           else
             date_from_str.call(args[:to]).end_of_month
           end
         else
           Time.current.prev_month.end_of_month
         end

    EventHistory.where(evented_at: from..to).delete_all

    loop.with_object([from]) { |_, list|
      nm = list.last.next_month
      raise StopIteration if nm > to
      list << nm
    }.each { |date|
      Statistics::Aggregation.run(date: date)
    }
  end

  desc 'キーワードからイベント情報を検索します'
  task :search, [:keyword] => :environment do |tasks, args|
    raise ArgumentError, 'Require the keyword' if args[:keyword].nil?

    require 'pp'

    puts 'Searching Connpass'
    pp Statistics::Client::Connpass.new.search(keyword: args[:keyword])

    puts 'Searching Doorkeeper'
    pp Statistics::Client::Doorkeeper.new.search(keyword: args[:keyword])
  end
end<commit_msg>Sleep 60 seconds and retry if raise Statistics::Client::APIRateLimitError<commit_after>require_relative '../statistics.rb'

namespace :statistics do
  desc '月次のイベント履歴を集計します'
  task :aggregation, [:from, :to] => :environment do |tasks, args|
    date_from_str = -> (str) {
      formats = %w(%Y%m %Y/%m %Y-%m)
      d = formats.map { |fmt|
        begin
          Time.zone.strptime(str, fmt)
        rescue ArgumentError
          Time.zone.local(str) if str.length == 4
        end
      }.compact.first
      raise ArgumentError, "Invalid format: `#{str}`, allow format is #{formats.push('%Y').join(' or ')}" if d.nil?
      d
    }

    from = if args[:from]
             if args[:from].length == 4
               date_from_str.call(args[:from]).beginning_of_year
             else
               date_from_str.call(args[:from]).beginning_of_month
             end
           else
             Time.current.prev_month.beginning_of_month
           end
    to = if args[:to]
           if args[:to].length == 4
             date_from_str.call(args[:to]).end_of_year
           else
             date_from_str.call(args[:to]).end_of_month
           end
         else
           Time.current.prev_month.end_of_month
         end

    EventHistory.where(evented_at: from..to).delete_all

    loop.with_object([from]) { |_, list|
      nm = list.last.next_month
      raise StopIteration if nm > to
      list << nm
    }.each { |date|
      begin
        puts "Aggregate for #{date.strftime('%Y/%m')}"
        Statistics::Aggregation.run(date: date)
      rescue Statistics::Client::APIRateLimitError
        puts 'API rate limit exceeded.'
        puts "This task will retry in 60 seconds from now(#{Time.zone.now})."
        sleep 60
        retry
      end
    }
  end

  desc 'キーワードからイベント情報を検索します'
  task :search, [:keyword] => :environment do |tasks, args|
    raise ArgumentError, 'Require the keyword' if args[:keyword].nil?

    require 'pp'

    puts 'Searching Connpass'
    pp Statistics::Client::Connpass.new.search(keyword: args[:keyword])

    puts 'Searching Doorkeeper'
    pp Statistics::Client::Doorkeeper.new.search(keyword: args[:keyword])
  end
end
{'fraction_non_alphanumeric': 0.12303075768942236, 'fraction_numerical': 0.003000750187546887, 'mean_word_length': 2.284072249589491, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6312079', 'n_tokens_mistral': 1322, 'n_tokens_neox': 1287, 'n_words': 280}