<commit_before>require 'bundler/setup'
require 'json'
require 'sinatra'
require 'em-redis'
require 'sinatra/async'
require 'uuidtools'


class Cloudq < Sinatra::Base
  register Sinatra::Async

  aget '/' do
    body "Welcome to Cloudq"
  end

  # Post Job to the Queue
  apost "/:queue" do |q|
    # parse data from json
    data = request.body.read.to_s
    params['job'] = JSON.parse(data)['job']
    
    # push into redis
    redis = EM::Protocols::Redis.connect
    redis.push_head [q,'-',:queued].join(''), params['job'].to_json

    body "Success"
  end


  # Get Job from the Queue
  aget "/:queue" do |q|
    redis = EM::Protocols::Redis.connect
    redis.rpop [q,'-',:queued].join('') do |response|
      if response 
        id = UUIDTools::UUID.random_create.to_s 
        job = JSON.parse(response).merge(:id => id)
        redis.set id, job
        body job.to_json
      else
        body "empty"
      end
    end
  end

  # Remove Job from the Queue

  adelete "/:queue/:id" do |q, id|
    redis = EM::Protocols::Redis.connect
    redis.delete id
    body "Success"
  end
end
<commit_msg>connected to redis to go<commit_after>require 'bundler/setup'
require 'json'
require 'sinatra'
require 'em-redis'
require 'sinatra/async'
require 'uuidtools'


class Cloudq < Sinatra::Base
  register Sinatra::Async

  aget '/' do
    body "Welcome to Cloudq"
  end

  # Post Job to the Queue
  apost "/:queue" do |q|
    # parse data from json
    data = request.body.read.to_s
    params['job'] = JSON.parse(data)['job']
    
    # push into redis
    redis = EM::Protocols::Redis.connect(ENV['REDISTOGO_URL'])
    redis.push_head [q,'-',:queued].join(''), params['job'].to_json

    body "Success"
  end


  # Get Job from the Queue
  aget "/:queue" do |q|
    redis = EM::Protocols::Redis.connect(ENV['REDISTOGO_URL'])
    redis.rpop [q,'-',:queued].join('') do |response|
      if response 
        id = UUIDTools::UUID.random_create.to_s 
        job = JSON.parse(response).merge(:id => id)
        redis.set id, job
        body job.to_json
      else
        body "empty"
      end
    end
  end

  # Remove Job from the Queue

  adelete "/:queue/:id" do |q, id|
    redis = EM::Protocols::Redis.connect(ENV['REDISTOGO_URL'])
    redis.delete id
    body "Success"
  end
end

{'fraction_non_alphanumeric': 0.13817227809357235, 'fraction_numerical': 0.0, 'mean_word_length': 2.769357495881384, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4236965', 'n_tokens_mistral': 832, 'n_tokens_neox': 815, 'n_words': 210}