<commit_before># https://developers.google.com/maps/documentation/javascript/tutorial
# http://stackoverflow.com/questions/19304574/center-set-zoom-of-map-to-cover-all-markers-visible-markers

module Stats
  def self.cached_daily_use(user_id, options)
    user = User.find_by_global_id(user_id)
    if !user || WeeklyStatsSummary.where(:user_id => user.id).count == 0
      sanitize_find_options!(options, user) if user.eval_account?
      return daily_use(user_id, options)
    end
    sanitize_find_options!(options, user)
    week_start = options[:start_at].utc.beginning_of_week(:sunday)
    week_end = options[:end_at].utc.end_of_week(:sunday)
    start_weekyear = (week_start.to_date.cwyear * 100) + week_start.to_date.cweek
    end_weekyear = (week_end.to_date.cwyear * 100) + week_end.to_date.cweek
    summaries = WeeklyStatsSummary.where(['user_id = ? AND weekyear >= ? AND weekyear <= ?', user.id, start_weekyear, end_weekyear])
    summary_lookups = {}
    summaries.each{|s| summary_lookups[s.weekyear] = s }

    days = {}
    all_stats = []
    weekyear_dates = {}
    word_development = {}
    options[:start_at].to_date.upto(options[:end_at].to_date) do |date|
      weekyear = (date.beginning_of_week(:sunday).cwyear * 100) + date.beginning_of_week(:sunday).cweek
      weekyear_dates[weekyear] ||= []
      weekyear_dates[weekyear] << date
    end
    date_distance = (week_end.to_date - week_start.to_date).to_f
    summaries.find_in_batches(batch_size: 5) do |batch|
      batch.each do |summary|
        (weekyear_dates[summary.weekyear] || []).each do |date|
          day = summary && summary.data && summary.data['stats']['days'][date.to_s]
          filtered_day_stats = nil
          if day
            filtered_day_stats = [day['total']]
            if options[:device_ids] || options[:location_ids]
              groups = day['group_counts'].select do |group|
                (!options[:device_ids] || options[:device_ids].include?(group['device_id'])) && 
                (!options[:location_ids] || options[:location_ids].include?(group['geo_cluster_id']) || options[:location_ids].include?(group['ip_cluster_id']))
              end
              filtered_day_stats = groups
            end
          else
            filtered_day_stats = [Stats.stats_counts([])]
          end
          all_stats += filtered_day_stats
          days[date.to_s] = usage_stats(filtered_day_stats)
        end
        track_word_development(word_development, summary, options[:start_at], date_distance)
        weekyear_dates.delete(summary.weekyear)
      end
    end
    weekyear_dates.each do |weekyear, dates|
      dates.each do |date|
        filtered_day_stats = [Stats.stats_counts([])]
        all_stats += filtered_day_stats
        days[date.to_s] = usage_stats(filtered_day_stats)
      end
    end
    
    res = usage_stats(all_stats)
    res[:days] = days
    word_development.each do |key, list|
      # if a word is used more than 7 times in the last few weeks, go ahead
      # and call it an emergent word
      # if a word has been dwindling
      development_cutoff = key == 'emergent_words' ? 7 : 0.75
      res[key.to_sym] = {}
      list.each do |word, val|
        if val > development_cutoff
          res[key.to_sym][word] = val
        end
      end
    end
    
    res[:start_at] = options[:start_at].to_time.utc.iso8601
    res[:end_at] = options[:end_at].to_time.utc.iso8601
    res[:cached] = true
    res
  end
  
  # TODO: this doesn't account for timezones at all. wah waaaaah.
  def self.daily_use(user_id, options)
    sessions = find_sessions(user_id, options)<commit_msg>nicer cutoff for emergent words<commit_after># https://developers.google.com/maps/documentation/javascript/tutorial
# http://stackoverflow.com/questions/19304574/center-set-zoom-of-map-to-cover-all-markers-visible-markers

module Stats
  def self.cached_daily_use(user_id, options)
    user = User.find_by_global_id(user_id)
    if !user || WeeklyStatsSummary.where(:user_id => user.id).count == 0
      sanitize_find_options!(options, user) if user.eval_account?
      return daily_use(user_id, options)
    end
    sanitize_find_options!(options, user)
    week_start = options[:start_at].utc.beginning_of_week(:sunday)
    week_end = options[:end_at].utc.end_of_week(:sunday)
    start_weekyear = (week_start.to_date.cwyear * 100) + week_start.to_date.cweek
    end_weekyear = (week_end.to_date.cwyear * 100) + week_end.to_date.cweek
    summaries = WeeklyStatsSummary.where(['user_id = ? AND weekyear >= ? AND weekyear <= ?', user.id, start_weekyear, end_weekyear])
    summary_lookups = {}
    summaries.each{|s| summary_lookups[s.weekyear] = s }

    days = {}
    all_stats = []
    weekyear_dates = {}
    word_development = {}
    options[:start_at].to_date.upto(options[:end_at].to_date) do |date|
      weekyear = (date.beginning_of_week(:sunday).cwyear * 100) + date.beginning_of_week(:sunday).cweek
      weekyear_dates[weekyear] ||= []
      weekyear_dates[weekyear] << date
    end
    date_distance = (week_end.to_date - week_start.to_date).to_f
    summaries.find_in_batches(batch_size: 5) do |batch|
      batch.each do |summary|
        (weekyear_dates[summary.weekyear] || []).each do |date|
          day = summary && summary.data && summary.data['stats']['days'][date.to_s]
          filtered_day_stats = nil
          if day
            filtered_day_stats = [day['total']]
            if options[:device_ids] || options[:location_ids]
              groups = day['group_counts'].select do |group|
                (!options[:device_ids] || options[:device_ids].include?(group['device_id'])) && 
                (!options[:location_ids] || options[:location_ids].include?(group['geo_cluster_id']) || options[:location_ids].include?(group['ip_cluster_id']))
              end
              filtered_day_stats = groups
            end
          else
            filtered_day_stats = [Stats.stats_counts([])]
          end
          all_stats += filtered_day_stats
          days[date.to_s] = usage_stats(filtered_day_stats)
        end
        track_word_development(word_development, summary, options[:start_at], date_distance)
        weekyear_dates.delete(summary.weekyear)
      end
    end
    weekyear_dates.each do |weekyear, dates|
      dates.each do |date|
        filtered_day_stats = [Stats.stats_counts([])]
        all_stats += filtered_day_stats
        days[date.to_s] = usage_stats(filtered_day_stats)
      end
    end
    
    res = usage_stats(all_stats)
    res[:days] = days
    word_development.each do |key, list|
      # if a word is used more than 7 times in the last few weeks, go ahead
      # and call it an emergent word
      # if a word has been dwindling
      development_cutoff = key == 'emergent_words' ? 1 : 0.75
      res[key.to_sym] = {}
      list.each do |word, val|
        if val > development_cutoff
          res[key.to_sym][word] = val
        end
      end
    end
    
    res[:start_at] = options[:start_at].to_time.utc.iso8601
    res[:end_at] = options[:end_at].to_time.utc.iso8601
    res[:cached] = true
    res
  end
  
  # TODO: this doesn't account for timezones at all. wah waaaaah.
  def self.daily_use(user_id, options)
    sessions = find_sessions(user_id, options)
{'fraction_non_alphanumeric': 0.11997798569069895, 'fraction_numerical': 0.008805723720418272, 'mean_word_length': 3.0360910605219322, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5643475', 'n_tokens_mistral': 2468, 'n_tokens_neox': 2388, 'n_words': 431}