<commit_before>require 'sinatra'
require 'sinatra/multi_route'
require 'redis'
require 'json'

REDIS_DB = '3'
REDIS_LIST_NAME = 'sweatpants_test'

redis = Redis.new#(:host => "10.0.1.1", :port => 6380)
redis.select REDIS_DB

set :port, 9203

class RedisSweatStore
  def initialize redis
    @list_name = 'sweatpants'
    @redis = redis
  end

  def queue request
    @redis.rpush @list_name, request
  end

  def dequeue count=nil
    list_length = @redis.llen(REDIS_LIST_NAME)
    multi_pop (count.nil? ? list_length : count)
  end

  private
  def multi_pop count
    @redis.multi {
      peek count
      @redis.ltrim(@list_name, count, -1)
    }[0]
  end

  def peek count
    @redis.lrange(@list_name, 0, count - 1)
  end
end

class SweatStore
  def initialize
    @store = []
  end

  def queue request
    @store << request
  end

  def dequeue count=nil
    count.nil? ? @store.pop(@store.size) : @store.pop(count)
  end
end

store = SweatStore.new

route :get, :post, :put, '*' do
  # p request
  request_to_store = "#{es_metadata}\n#{es_body}"
  store.queue_request request_to_store
end

helpers do
  def es_metadata
    # POST /users/user/68782
    path = request.env['REQUEST_URI'].split('/').reject(&:empty?)
    if path.size == 3
      { index: { _index: path[0], _type: path[1], _id: path[2] } }
    else
      nil
    end
  end

  def es_body
    JSON.parse(request.env['rack.request.form_vars'])
  end
end
<commit_msg>refactor classes<commit_after>require 'sinatra'
require 'sinatra/multi_route'
require 'redis'
require 'json'

REDIS_DB = '3'
REDIS_LIST_NAME = 'sweatpants_test'

redis = Redis.new#(:host => "10.0.1.1", :port => 6380)
redis.select REDIS_DB

set :port, 9203

class RedisSweatStore
  def initialize redis
    @list_name = 'sweatpants'
    @redis = redis
  end

  def queue request
    @redis.rpush @list_name, request
  end

  def dequeue count=nil
    list_length = @redis.llen(REDIS_LIST_NAME)
    multi_pop (count.nil? ? list_length : count)
  end

  private
  def multi_pop count
    @redis.multi {
      peek count
      @redis.ltrim(@list_name, count, -1)
    }[0]
  end

  def peek count
    @redis.lrange(@list_name, 0, count - 1)
  end
end

class SweatStore
  def initialize
    @store = []
  end

  def queue request
    @store << request
  end

  def dequeue count=nil
    count.nil? ? @store.pop(@store.size) : @store.pop(count)
  end
end

store = SweatStore.new

route :get, :post, :put, '*' do
  # p request
  # store.queue_request request_to_store
  request.fullpath
end


class SweatJob
  def path
    request.fullpath
  end

  def body
    JSON.parse(request.body.read)
  end
end

class ESHandler
  def initialize
    @es_client = nil
  end

  def process jobs
    content = jobs.map{|job| parse_job(job.path, job.body)}.join("\n")
    # send content to @es_client
  end

  private
  def parse_job path, body
    components = path.split('/').reject(&:empty?)
    return false if path.size != 3
    header = { index: { _index: path[0], _type: path[1], _id: path[2] } }
    "#{header}\n#{body}"
  end
end
{'fraction_non_alphanumeric': 0.1202884300229433, 'fraction_numerical': 0.016060308095706327, 'mean_word_length': 3.107671601615074, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6385254', 'n_tokens_mistral': 1207, 'n_tokens_neox': 1152, 'n_words': 288}