<commit_before>#
# Extra classes needed to operate with Chimp
#
module Chimp
  #
  # This class contains all the necessary to make calls to api1.5 via the right_api_client gem
  # or obtain a list of instances via api1.6 calls.
  #
  class Connection

    include Singleton
    attr_accessor :client, :all_instances, :retry

    def initialize
    end

    def self.connect_and_cache

      self.start_right_api_client

      Log.debug "Making initial Api1.6 call to cache entries."

      result = self.all_instances
      if result.empty? || result.nil?
        Log.error "Couldnt contact API1.6 correctly, will now exit."
        exit -1
      else
        Log.debug "API lists #{result.count} operational instances in the account"
      end
    end

    def self.connect
      self.start_right_api_client
    end

    def self.start_right_api_client
      require 'yaml'
      require 'right_api_client'
      begin
        creds = YAML.load_file("#{ENV['HOME']}/.rest_connection/rest_api_config.yaml")

        # Extract the account
        creds[:account] = File.basename(creds[:api_url])

        # Figure out url to hit:
        creds[:api_url] = "https://"+URI.parse(creds[:api_url]).host

        @endpoint = URI.parse(creds[:api_url]).host

        Log.debug "Logging into Api 1.5 right_api_client"

        @client = RightApi::Client.new(:email => creds[:user], :password => <PASSWORD>],
                                        :account_id => creds[:account], :api_url => creds[:api_url],
                                        :timeout => nil )
      rescue
        puts "##############################################################################"
        puts "Error: "
        puts " - credentials file could not be loaded correctly"
        puts "or                           "
        puts " - connection couldnt be establishhed"
        puts "##############################################################################"
        exit -1
      end
    end

    def self.client
      @client
    end

    def self.endpoint
      @endpoint
    end

    #
    # Returns every single operational instance in the account
    #
    def self.all_instances()
      begin
        filters_list = "state=operational"
        filters = CGI::escape(filters_list)

        query="/api/instances?view=full&filter="+filters

        @all_instances = Connection.api16_call(query)

      rescue Exception => e
        Log.error "[#{Chimp.get_job_uuid}] self.all_instaces"
        Log.error "[#{Chimp.get_job_uuid}] #{e.message}"
      end

      return @all_instances
    end

    #
    # Returns every single operational instance in the account, matching the filters passed.
    #
    def self.instances(extra_filters)
      begin
        filters_list = "state=operational&"+extra_filters
        filters = CGI::escape(filters_list)

        query="/api/instances?view=full&filter="+filters

        instances = Connection.api16_call(query)

      rescue Exception => e
        Log.error "[#{Chimp.get_job_uuid}] self.instances"
        Log.error "[#{Chimp.get_job_uuid}] #{e.message}"
      end

      return instances
    end

    #
    # Provides a way to make an api1.6 call directly
    #
    def Connection.api16_call(query)

      @retry = true
      retries = 5
      attempts = 0
      sleep_for = 20

      begin
        get  = Net::HTTP::Get.new(query)
        get['Cookie']        = @client.cookies.map { |key, value| "%s=%s" % [key, value] }.join(';')
        get['X-Api_Version'] = '1.6'
        get['X-Account']     = @client.account_id

        http = Net::HTTP.new(@endpoint, 443)
        http.use_ssl = true

        Log.debug "[#{Chimp.get_job_uuid}] Querying API for: #{query}"


        while attempts < retries
          if @retry
            if attempts > 0
              Log.debug "[#{Chimp.get_job_uuid}] Retrying..."
              sleep_time = sleep_for * attempts
              # Add a random amount to avoid staggering calls
              sleep_time += rand(15)

              Log.debug "[#{Chimp.get_job_uuid}] Sleeping between retries for #{sleep_time}"
              sleep(sleep_time)
            end

            Log.debug "[#{Chimp.get_job_uuid}] Attempt # #{attempts+1} at querying the API" unless attempts == 0

            time = Benchmark.measure do
              @response = http.request(get)
              attempts += 1
            end

            Log.debug "[#{Chimp.get_job_uuid}] API Request time: #{time.real} seconds"
            Log.debug "[#{Chimp.get_job_uuid}] API Query was: #{query}"

            # Validate API response
            instances = validate_response(@response, query)
          else
            # We dont retry, exit the loop.
            break
          end
        end

        if attempts == retries

          Log.error "[#{Chimp.get_job_uuid}] Api call failed more than #{retries} times."

          Chimp.set_failure(true)
          instances = []
          raise "[#{Chimp.get_job_uuid}] Api call failed more than #{retries} times."
        end

      rescue Exception => e
        Log.error "[#{Chimp.get_job_uuid}] #{e.message}"
        Log.error "[#{Chimp.get_job_uuid}] Catched exception on http request to the api, retrying"<commit_msg>OPS-821 extra debug<commit_after>#
# Extra classes needed to operate with Chimp
#
module Chimp
  #
  # This class contains all the necessary to make calls to api1.5 via the right_api_client gem
  # or obtain a list of instances via api1.6 calls.
  #
  class Connection

    include Singleton
    attr_accessor :client, :all_instances, :retry

    def initialize
    end

    def self.connect_and_cache

      self.start_right_api_client

      Log.debug "Making initial Api1.6 call to cache entries."

      result = self.all_instances
      if result.empty? || result.nil?
        Log.error "Couldnt contact API1.6 correctly, will now exit."
        exit -1
      else
        Log.debug "API lists #{result.count} operational instances in the account"
      end
    end

    def self.connect
      self.start_right_api_client
    end

    def self.start_right_api_client
      require 'yaml'
      require 'right_api_client'
      begin
        creds = YAML.load_file("#{ENV['HOME']}/.rest_connection/rest_api_config.yaml")

        # Extract the account
        creds[:account] = File.basename(creds[:api_url])

        # Figure out url to hit:
        creds[:api_url] = "https://"+URI.parse(creds[:api_url]).host

        @endpoint = URI.parse(creds[:api_url]).host

        Log.debug "Logging into Api 1.5 right_api_client"

        @client = RightApi::Client.new(:email => creds[:user], :password => creds[:<PASSWORD>],
                                        :account_id => creds[:account], :api_url => creds[:api_url],
                                        :timeout => nil )
      rescue
        puts "##############################################################################"
        puts "Error: "
        puts " - credentials file could not be loaded correctly"
        puts "or                           "
        puts " - connection couldnt be establishhed"
        puts "##############################################################################"
        exit -1
      end
    end

    def self.client
      @client
    end

    def self.endpoint
      @endpoint
    end

    #
    # Returns every single operational instance in the account
    #
    def self.all_instances()
      begin
        filters_list = "state=operational"
        filters = CGI::escape(filters_list)

        query="/api/instances?view=full&filter="+filters

        @all_instances = Connection.api16_call(query)

      rescue Exception => e
        Log.error "[#{Chimp.get_job_uuid}] self.all_instaces"
        Log.error "[#{Chimp.get_job_uuid}] #{e.message}"
      end

      return @all_instances
    end

    #
    # Returns every single operational instance in the account, matching the filters passed.
    #
    def self.instances(extra_filters)
      begin
        filters_list = "state=operational&"+extra_filters
        filters = CGI::escape(filters_list)

        query="/api/instances?view=full&filter="+filters

        instances = Connection.api16_call(query)

      rescue Exception => e
        Log.error "[#{Chimp.get_job_uuid}] self.instances"
        Log.error "[#{Chimp.get_job_uuid}] #{e.message}"
      end

      return instances
    end

    #
    # Provides a way to make an api1.6 call directly
    #
    def Connection.api16_call(query)

      @retry = true
      retries = 5
      attempts = 0
      sleep_for = 20

      begin
        get  = Net::HTTP::Get.new(query)
        get['Cookie']        = @client.cookies.map { |key, value| "%s=%s" % [key, value] }.join(';')
        get['X-Api_Version'] = '1.6'
        get['X-Account']     = @client.account_id

        http = Net::HTTP.new(@endpoint, 443)
        http.use_ssl = true

        Log.debug "[#{Chimp.get_job_uuid}] Querying API for: #{query}"


        while attempts < retries
          if @retry
            if attempts > 0
              Log.debug "[#{Chimp.get_job_uuid}] Retrying..."
              sleep_time = sleep_for * attempts
              # Add a random amount to avoid staggering calls
              sleep_time += rand(15)

              Log.debug "[#{Chimp.get_job_uuid}] Sleeping between retries for #{sleep_time}"
              sleep(sleep_time)
            end

            Log.debug "[#{Chimp.get_job_uuid}] Attempt # #{attempts+1} at querying the API" unless attempts == 0

            time = Benchmark.measure do
              begin
                @response = http.request(get)
              rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError => e
                Log.error "################################################"
                Log.error "################################################"
                Log.error "FAILED on HTTP REQUEST"
                Log.error "################################################"
                Log.error "################################################"
                Log.error "################################################"
                exit -1
              end
              attempts += 1
            end

            Log.debug "[#{Chimp.get_job_uuid}] API Request time: #{time.real} seconds"
            Log.debug "[#{Chimp.get_job_uuid}] API Query was: #{query}"

            # Validate API response
            instances = validate_response(@response, query)
          else
            # We dont retry, exit the loop.
            break
          end
        end

        if attempts == retries

          Log.error "[#{Chimp.get_job_uuid}] Api call failed more than #{retries} times."

          Chimp.set_failure(true)
          instances = []
          raise "[#{Chimp.get_job_uuid}] Api call failed more than #{retries} times."
        end

      rescue Exception => e
        Log.error "[#{Chimp.get_job_uuid}] #{e.message}"
        Log.error "[#{Chimp.get_job_uuid}] Catched exception on http request to the api, retrying"
{'fraction_non_alphanumeric': 0.14874842427516657, 'fraction_numerical': 0.0066630650099045565, 'mean_word_length': 2.1287323943661973, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 24, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1353789', 'n_tokens_mistral': 3161, 'n_tokens_neox': 3089, 'n_words': 808}