<commit_before>require 'duckweed'
require 'duckweed/utility_methods'
require 'json'
require 'sinatra'

module Duckweed
  # we use different tokens for internal/external use so that we
  # can revoke tokens given to third parties without affecting our apps
  AUTH_TOKENS = [
    'secret_token', # internal use
    'secret_token'  # Geckoboard
  ]

  class App < Sinatra::Base
    include UtilityMethods

    # routes accessible without authentication:
    AUTH_WHITELIST = ['/hello']

    before do
      unless AUTH_WHITELIST.include?(request.path_info) || authenticated?
        halt 403, 'Forbidden'
      end
    end

    post '/track/:event' do
      increment_counters_for(params[:event])
      'OK'
    end

    get '/count/:event' do
      # default to last hour with minute-granularity
      count_for(params[:event], :minutes, 60)
    end

    get '/count/:event/:granularity/:quantity' do
      granularity = params[:granularity].to_sym
      if !(interval = INTERVAL[granularity])
        [400, 'Bad Request']
      elsif (params[:quantity].to_i * interval[:bucket_size]) > interval[:expiry]
        [413, 'Request Entity Too Large']
      else
        count_for(params[:event], granularity, params[:quantity])
      end
    end

    get '/histogram/:event' do
      histogram(params[:event], :minutes, '60')
    end

    get '/histogram/:event/:granularity/:quantity' do
      granularity = params[:granularity].to_sym
      if !(interval = INTERVAL[granularity])
        [400, 'Bad Request']
      elsif (params[:quantity].to_i * interval[:bucket_size]) > interval[:expiry]
        [413, 'Request Entity Too Large']
      else
        histogram(params[:event], granularity, params[:quantity])
      end
    end

    get "/hello" do
      "Hello, world!"
    end

    private

    def redis
      Duckweed.redis
    end

    def authenticated?
      AUTH_TOKENS.include?(auth_token_via_params || auth_token_via_http_basic_auth)
    end

    def auth_token_via_params
      params[:auth_token]
    end

    def auth_token_via_http_basic_auth
      auth = Rack::Auth::Basic::Request.new(request.env)
      auth.provided? && auth.basic? && auth.credentials.first
    end


    INTERVAL = {
      :minutes => {
        :bucket_size  => 60,
        :expiry       => 86400,       # keep minute-resolution data for last day
        :time_format  => '%I:%M%p'    # 10:11AM
      },
      :hours => {
        :bucket_size  => 3600,
        :expiry       => 86400 * 7,   # keep hour-resolution data for last week
        :time_format  => '%a %I%p'    # Sun 10AM
      },
      :days => {
        :bucket_size  => 86400,
        :expiry       => 86400 * 365, # keep day-resolution data for last year
        :time_format  => '%b %d %Y'   # Jan 21 2011
      }
    }

    def increment_counters_for(event)
      INTERVAL.keys.each do |granularity|
        key = key_for(event, granularity)
        if has_bucket_for?(granularity)
          redis.incr(key)
          redis.expire(key, INTERVAL[granularity][:expiry])
        end
      end
    end

    def key_for(event, granularity)
      "duckweed:#{event}:#{bucket_with_granularity(granularity)}"
    end

    def bucket_with_granularity(granularity)
      "#{granularity}:#{bucket_index(granularity)}"
    end

    def bucket_index(granularity)
      timestamp / INTERVAL[granularity][:bucket_size]
    end

    def timestamp
      (params[:timestamp] || Time.now).to_i
    end

    def count_for(event, granularity, quantity)
      keys = keys_for(event, granularity, quantity)
      redis.mget(*keys).inject(0) { |memo, obj| memo + obj.to_i }.to_s
    end

    def keys_for(event, granularity, quantity)
      count = quantity ? quantity.to_i : INTERVAL[granularity][:expiry]
      bucket_indices(granularity, count).map do |idx|
        "duckweed:#{event}:#{granularity}:#{idx}"
      end
    end

    def bucket_indices(granularity, count)
      bucket_idx = bucket_index(granularity) - count
      Array.new(count) do |i|
        bucket_idx += 1
      end
    end

    def has_bucket_for?(granularity)
      first_available_bucket_time = Time.now.to_i - INTERVAL[granularity][:expiry]
      first_available_bucket_time < timestamp
    end

    def histogram(event, granularity, quantity)
      keys      = keys_for(event, granularity, quantity)
      values    = redis.mget(*keys).map {|x| x ? x.to_i : 0}
      times     = times_for(granularity, quantity)
      min, max  = values.min, values.max
      mid       = (max - min).to_f / 2
      {
        :item     => values,
        :settings => {
          :axisx  => times,
          :axisy  => [min, mid, max],
          :colour => 'ff9900'
        }
      }.to_json
    end

    def times_for(granularity, quantity)
      ending    = Time.now.to_i
      beginning = ending.to_i - INTERVAL[granularity][:bucket_size] * quantity.to_i
      middle    = (beginning + ending) / 2
      [beginning, middle, ending].map do |time|
        Time.at(time).strftime(INTERVAL[granularity][:time_format])
      end
    end<commit_msg>Eliminate repetition by extracting check_request_limits! method<commit_after>require 'duckweed'
require 'duckweed/utility_methods'
require 'json'
require 'sinatra'

module Duckweed
  # we use different tokens for internal/external use so that we
  # can revoke tokens given to third parties without affecting our apps
  AUTH_TOKENS = [
    'secret_token', # internal use
    'secret_token'  # <PASSWORD>board
  ]

  class App < Sinatra::Base
    include UtilityMethods

    # routes accessible without authentication:
    AUTH_WHITELIST = ['/hello']

    before do
      unless AUTH_WHITELIST.include?(request.path_info) || authenticated?
        halt 403, 'Forbidden'
      end
    end

    post '/track/:event' do
      increment_counters_for(params[:event])
      'OK'
    end

    get '/count/:event' do
      # default to last hour with minute-granularity
      count_for(params[:event], :minutes, 60)
    end

    get '/count/:event/:granularity/:quantity' do
      check_request_limits!
      count_for(params[:event], params[:granularity], params[:quantity])
    end

    get '/histogram/:event' do
      histogram(params[:event], :minutes, '60')
    end

    get '/histogram/:event/:granularity/:quantity' do
      check_request_limits!
      histogram(params[:event], params[:granularity], params[:quantity])
    end

    get "/hello" do
      "Hello, world!"
    end

    private

    def redis
      Duckweed.redis
    end

    def authenticated?
      AUTH_TOKENS.include?(auth_token_via_params || auth_token_via_http_basic_auth)
    end

    def auth_token_via_params
      params[:auth_token]
    end

    def auth_token_via_http_basic_auth
      auth = Rack::Auth::Basic::Request.new(request.env)
      auth.provided? && auth.basic? && auth.credentials.first
    end

    INTERVAL = {
      :minutes => {
        :bucket_size  => 60,
        :expiry       => 86400,       # keep minute-resolution data for last day
        :time_format  => '%I:%M%p'    # 10:11AM
      },
      :hours => {
        :bucket_size  => 3600,
        :expiry       => 86400 * 7,   # keep hour-resolution data for last week
        :time_format  => '%a %I%p'    # Sun 10AM
      },
      :days => {
        :bucket_size  => 86400,
        :expiry       => 86400 * 365, # keep day-resolution data for last year
        :time_format  => '%b %d %Y'   # Jan 21 2011
      }
    }

    # don't allow requests that would place an unreasonable load on the server,
    # or for which we won't have data anyway
    def check_request_limits!
      granularity = params[:granularity].to_sym
      if !(interval = INTERVAL[granularity])
        halt 400, 'Bad Request'
      elsif (params[:quantity].to_i * interval[:bucket_size]) > interval[:expiry]
        halt 413, 'Request Entity Too Large'
      end
    end

    def increment_counters_for(event)
      INTERVAL.keys.each do |granularity|
        key = key_for(event, granularity)
        if has_bucket_for?(granularity)
          redis.incr(key)
          redis.expire(key, INTERVAL[granularity][:expiry])
        end
      end
    end

    def key_for(event, granularity)
      "duckweed:#{event}:#{bucket_with_granularity(granularity)}"
    end

    def bucket_with_granularity(granularity)
      "#{granularity}:#{bucket_index(granularity)}"
    end

    def bucket_index(granularity)
      timestamp / INTERVAL[granularity][:bucket_size]
    end

    def timestamp
      (params[:timestamp] || Time.now).to_i
    end

    def count_for(event, granularity, quantity)
      keys = keys_for(event, granularity.to_sym, quantity)
      redis.mget(*keys).inject(0) { |memo, obj| memo + obj.to_i }.to_s
    end

    def keys_for(event, granularity, quantity)
      count = quantity ? quantity.to_i : INTERVAL[granularity][:expiry]
      bucket_indices(granularity, count).map do |idx|
        "duckweed:#{event}:#{granularity}:#{idx}"
      end
    end

    def bucket_indices(granularity, count)
      bucket_idx = bucket_index(granularity) - count
      Array.new(count) do |i|
        bucket_idx += 1
      end
    end

    def has_bucket_for?(granularity)
      first_available_bucket_time = Time.now.to_i - INTERVAL[granularity][:expiry]
      first_available_bucket_time < timestamp
    end

    def histogram(event, granularity, quantity)
      granularity = granularity.to_sym
      keys        = keys_for(event, granularity, quantity)
      values      = redis.mget(*keys).map {|x| x ? x.to_i : 0}
      times       = times_for(granularity, quantity)
      min, max    = values.min, values.max
      mid         = (max - min).to_f / 2
      {
        :item     => values,
        :settings => {
          :axisx  => times,
          :axisy  => [min, mid, max],
          :colour => 'ff9900'
        }
      }.to_json
    end

    def times_for(granularity, quantity)
      ending    = Time.now.to_i
      beginning = ending.to_i - INTERVAL[granularity][:bucket_size] * quantity.to_i
      middle    = (beginning + ending) / 2
      [beginning, middle, ending].map do |time|
        Time.at(time).strftime(INTERVAL[granularity][:time_format])
      end
    end
{'fraction_non_alphanumeric': 0.11180185514110914, 'fraction_numerical': 0.01322281428853365, 'mean_word_length': 2.5338214783821478, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '308866', 'n_tokens_mistral': 3533, 'n_tokens_neox': 3257, 'n_words': 733}