<commit_before>module Sufia
  class UserStatImporter

    def initialize(options={})
      @verbose = options[:verbose]
      @logging = options[:logging]
    end

    def import
      log_message('Begin import of User stats.')
      ::User.find_each do |user|
        log_message("Working on user #{user.user_key}")
        start_date = date_since_last_cache(user)
        next if start_date.to_date == Date.today

        stats = {}
        file_ids_for_user(user).each do |file_id|
          retry_count = 0
          begin
            view_stats = FileViewStat.statistics(file_id, start_date, user.id)
            stats = tally_results(view_stats, :views, stats)
            sleep(1.0) # adding sleep so we do not access GA to fast

            dl_stats = FileDownloadStat.statistics(file_id, start_date, user.id)
            stats = tally_results(dl_stats, :downloads, stats)
            sleep(1.0) # adding sleep so we do not access GA to fast
          rescue
            sleep(1.0)
            retry_count++
            retry unless retry_count > 5
          end

        end

        create_or_update_user_stats(stats, user)
      end
    end


private

    def date_since_last_cache(user)
      last_cached_stat = UserStat.where(user_id: user.id).order(date: :asc).last

      if last_cached_stat
        last_cached_stat.date + 1.day
      else
        Sufia.config.analytic_start_date
      end
    end

    def file_ids_for_user(user)
      ids = []
      ::GenericFile.find_in_batches("#{Solrizer.solr_name('depositor', :symbol)}:\"#{user.user_key}\"", fl:"id") do |group|
        ids.concat group.map { |doc| doc["id"] }
      end
      ids
    end

    # For each date, add the view and download counts for this
    # file to the view & download sub-totals for that day.
    # The resulting hash will look something like this:
    # {"2014-11-30 00:00:00 UTC" => {:views=>2, :downloads=>5},
    #  "2014-12-01 00:00:00 UTC" => {:views=>4, :downloads=>4}}
    def tally_results(file_stats, stat_name, total_stats)
      file_stats.each do |stats|
        # Exclude the stats from today since it will only be a partial day's worth of data
        break if stats.date == Date.today

        date_key = stats.date.to_s
        old_count = total_stats[date_key] ? total_stats[date_key].fetch(stat_name) { 0 } : 0
        new_count = old_count + stats.method(stat_name).call

        old_values = total_stats[date_key] || {}
        total_stats.store(date_key, old_values)
        total_stats[date_key].store(stat_name, new_count)
      end
      total_stats
    end

    def create_or_update_user_stats(stats, user)
      stats.each do |date_string, data|
        date = Time.zone.parse(date_string)

        user_stat = UserStat.where(user_id: user.id).where(date: date).first
        user_stat ||= UserStat.new(user_id: user.id, date: date)

        user_stat.file_views = data[:views] || 0
        user_stat.file_downloads = data[:downloads] || 0
        user_stat.save!
      end
    end

    def log_message(message)
      puts message if @verbose
      Rails.logger.info "#{self.class}: #{message}" if @logging
    end

  end
end
<commit_msg>fixing rescue errors and adding more output.  refs #139<commit_after>module Sufia
  class UserStatImporter

    def initialize(options={})
      @verbose = options[:verbose]
      @logging = options[:logging]
    end

    def import
      log_message('Begin import of User stats.')
      ::User.find_each do |user|
        log_message("Working on user #{user.user_key}")
        start_date = date_since_last_cache(user)
        next if start_date.to_date == Date.today

        stats = {}
        file_ids_for_user(user).each do |file_id|
          retry_count = 0
          begin
            view_stats = FileViewStat.statistics(file_id, start_date, user.id)
            stats = tally_results(view_stats, :views, stats)
            sleep(1.0) # adding sleep so we do not access GA to fast

            dl_stats = FileDownloadStat.statistics(file_id, start_date, user.id)
            stats = tally_results(dl_stats, :downloads, stats)
            sleep(1.0) # adding sleep so we do not access GA to fast
          rescue Exception => e
            sleep(1.0)
            retry_count += 1
            if retry_count <= 5
              retry
            else
              logger.warn "Retried error on #{user} for file #{file_id} too many times.  Continuing... \n#{e}"
            end
          end

        end

        create_or_update_user_stats(stats, user)
      end
    end


private

    def date_since_last_cache(user)
      last_cached_stat = UserStat.where(user_id: user.id).order(date: :asc).last

      if last_cached_stat
        last_cached_stat.date + 1.day
      else
        Sufia.config.analytic_start_date
      end
    end

    def file_ids_for_user(user)
      ids = []
      ::GenericFile.find_in_batches("#{Solrizer.solr_name('depositor', :symbol)}:\"#{user.user_key}\"", fl:"id") do |group|
        ids.concat group.map { |doc| doc["id"] }
      end
      ids
    end

    # For each date, add the view and download counts for this
    # file to the view & download sub-totals for that day.
    # The resulting hash will look something like this:
    # {"2014-11-30 00:00:00 UTC" => {:views=>2, :downloads=>5},
    #  "2014-12-01 00:00:00 UTC" => {:views=>4, :downloads=>4}}
    def tally_results(file_stats, stat_name, total_stats)
      file_stats.each do |stats|
        # Exclude the stats from today since it will only be a partial day's worth of data
        break if stats.date == Date.today

        date_key = stats.date.to_s
        old_count = total_stats[date_key] ? total_stats[date_key].fetch(stat_name) { 0 } : 0
        new_count = old_count + stats.method(stat_name).call

        old_values = total_stats[date_key] || {}
        total_stats.store(date_key, old_values)
        total_stats[date_key].store(stat_name, new_count)
      end
      total_stats
    end

    def create_or_update_user_stats(stats, user)
      stats.each do |date_string, data|
        date = Time.zone.parse(date_string)

        user_stat = UserStat.where(user_id: user.id).where(date: date).first
        user_stat ||= UserStat.new(user_id: user.id, date: date)

        user_stat.file_views = data[:views] || 0
        user_stat.file_downloads = data[:downloads] || 0
        user_stat.save!
      end
    end

    def log_message(message)
      puts message if @verbose
      Rails.logger.info "#{self.class}: #{message}" if @logging
    end

  end
end

{'fraction_non_alphanumeric': 0.10255625287004438, 'fraction_numerical': 0.014388489208633094, 'mean_word_length': 2.4173640167364017, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1191', 'n_tokens_mistral': 2162, 'n_tokens_neox': 2062, 'n_words': 514}