<commit_before>require 'bundler/setup'

require_relative '../helpers/Logging'
require_relative '../helpers/Params'
require_relative '../helpers/Properties'
require_relative '../helpers/Rates'

require 'json'
require 'simple_oauth'
require 'typhoeus'

class TwitterRequest

  include TwitterLog
  include TwitterParams
  include TwitterProperties
  include TwitterRates

  attr_reader :data, :log, :params, :props

  def initialize(args)
    @params   = args.fetch(:params)
    @data     = args.fetch(:data)
    @log      = args[:log] || default_logger
    @props    = load_props(@data[:props])
  end

  def url
    raise NotImplementedError, "No URL Specified for the Request"
  end

  def request_name
    raise NotImplementedError, "Request Name Not Specified"
  end

  def twitter_endpoint
    raise NotImplementedError, "Twitter Endpoint Not Specified"
  end

  def success(response)
    raise NotImplementedError, "Success Handler Not Specified"
  end

  def error(response)
    puts "FAILURE      : #{Time.now}"
    puts "Response Code: #{response.code}"
    puts "Response Info: #{response.status_message}"
    raise RuntimeError, "TwitterRequest Failed: #{response.code}"
  end

  def authorization
    header = SimpleOAuth::Header.new("GET", url, escaped_params, props)
    { 'Authorization' => header.to_s }
  end

  def options
    options = {}
    options[:method]  = :get
    options[:headers] = authorization

    request_params = {}
    escaped_params.keys.each do |key|
      request_params[key] = escaped_params[key] if include_param?(key)
    end

    options[:params]  = request_params
    options
  end

  def make_request
    check_rates
    request = Typhoeus::Request.new(url, options)
    log.info("REQUESTING: #{request.base_url}?#{display_params}")
    response = request.run
    @rate_count = @rate_count - 1
    response
  end

  def collect
    response = make_request
    if response.code == 200
      success(response) do |data|
        yield data
      end
    else
      error(response)
    end
  end<commit_msg>Modified TwitterRequest to ignore 500 errors.<commit_after>require 'bundler/setup'

require_relative '../helpers/Logging'
require_relative '../helpers/Params'
require_relative '../helpers/Properties'
require_relative '../helpers/Rates'

require 'json'
require 'simple_oauth'
require 'typhoeus'

class TwitterRequest

  include TwitterLog
  include TwitterParams
  include TwitterProperties
  include TwitterRates

  attr_reader :data, :log, :params, :props

  def initialize(args)
    @params   = args.fetch(:params)
    @data     = args.fetch(:data)
    @log      = args[:log] || default_logger
    @props    = load_props(@data[:props])
  end

  def url
    raise NotImplementedError, "No URL Specified for the Request"
  end

  def request_name
    raise NotImplementedError, "Request Name Not Specified"
  end

  def twitter_endpoint
    raise NotImplementedError, "Twitter Endpoint Not Specified"
  end

  def success(response)
    raise NotImplementedError, "Success Handler Not Specified"
  end

  def error(response)
    puts "FAILURE      : #{Time.now}"
    puts "Response Code: #{response.code}"
    puts "Response Info: #{response.status_message}"
    raise RuntimeError, "TwitterRequest Failed: #{response.code}"
  end

  def authorization
    header = SimpleOAuth::Header.new("GET", url, escaped_params, props)
    { 'Authorization' => header.to_s }
  end

  def options
    options = {}
    options[:method]  = :get
    options[:headers] = authorization

    request_params = {}
    escaped_params.keys.each do |key|
      request_params[key] = escaped_params[key] if include_param?(key)
    end

    options[:params]  = request_params
    options
  end

  def make_request
    check_rates
    request = Typhoeus::Request.new(url, options)
    log.info("REQUESTING: #{request.base_url}?#{display_params}")
    response = request.run
    @rate_count = @rate_count - 1
    response
  end

  def collect
    while true do
      response = make_request
      if response.code == 200
        success(response) do |data|
          yield data
        end
	break
      elsif response.code == 500
        puts "FAILURE      : #{Time.now}"
        puts "Response Code: #{response.code}"
        puts "Response Info: #{response.status_message}"
        puts "RETRYING!"
      else
        error(response)
      end
    end
  end
{'fraction_non_alphanumeric': 0.0992, 'fraction_numerical': 0.0032, 'mean_word_length': 3.2321083172147, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '317081', 'n_tokens_mistral': 1331, 'n_tokens_neox': 1301, 'n_words': 349}