<commit_before>require "httparty"

module RedditApi
  class Client

    def self.sleep_time
      if ENV["ENVIRONMENT"] == "TEST" || ENV["RAILS_ENV"] == "test"
        0
      else
        3
      end
    end

    SLEEP_TIME = self.sleep_time
    MAX_FAILURES = 5

    attr_reader :failures

    def initialize(args = {})
      @client = args.fetch(:client, HTTParty)
      @requestor = args.fetch(:requestor, RedditApi::Requestor.new(client: client))
      @parser = args.fetch(:parser, RedditApi::ResponseParser)
      @null_response_factory = RedditApi:: NullResponse
      @failures = args.fetch(:failures, 0)
    end

    def get(query)
      while query.capture_count < query.count && failures < MAX_FAILURES
        response = send_request(query)
        response = parser.parse_response(response, query.count)
        update_progress(query, response)
      end
      reset_failures
    end

    protected
    attr_writer :failures
    private
    attr_reader :client, :null_response_factory, :requestor, :parser

    def send_request(query)
      request = requestor.build(query)
      response = client.get(*request)
      response || null_response_factory.new
    end

    def update_progress(query, response)
      captured_before = query.capture_count
      update_query(query, response)
      captured_after = query.capture_count<commit_msg>Add sleep to Client to prevent API issues<commit_after>require "httparty"

module RedditApi
  class Client

    def self.sleep_time
      if ENV["ENVIRONMENT"] == "TEST" || ENV["RAILS_ENV"] == "test"
        0
      else
        3
      end
    end

    SLEEP_TIME = self.sleep_time
    MAX_FAILURES = 5

    attr_reader :failures

    def initialize(args = {})
      @client = args.fetch(:client, HTTParty)
      @requestor = args.fetch(:requestor, RedditApi::Requestor.new(client: client))
      @parser = args.fetch(:parser, RedditApi::ResponseParser)
      @null_response_factory = RedditApi:: NullResponse
      @failures = args.fetch(:failures, 0)
    end

    def get(query)
      while query.capture_count < query.count && failures < MAX_FAILURES
        response = send_request(query)
        response = parser.parse_response(response, query.count)
        update_progress(query, response)
      end
      reset_failures
    end

    protected
    attr_writer :failures
    private
    attr_reader :client, :null_response_factory, :requestor, :parser

    def send_request(query)
      sleep(SLEEP_TIME)
      request = requestor.build(query)
      response = client.get(*request)
      response || null_response_factory.new
    end

    def update_progress(query, response)
      captured_before = query.capture_count
      update_query(query, response)
      captured_after = query.capture_count
{'fraction_non_alphanumeric': 0.09130434782608696, 'fraction_numerical': 0.002898550724637681, 'mean_word_length': 2.9612625538020088, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1040172', 'n_tokens_mistral': 830, 'n_tokens_neox': 815, 'n_words': 181}