<commit_before>#
# Copyright (C) 2014 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License.  You
# may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.  See the License for the specific language governing
# permissions and limitations under the License.
#

require 'rest-client'
require 'json'
require 'hashie'

module RubyYarn

  class ApiMash < Hashie::Mash
    attr_reader :client
    def initialize(hash, client=nil)
      @client = client
      super(hash)
    end
  end

  class YarnApiObject
    attr_reader :client
    def initialize(base_url)
      @client = YarnRestApiClient.new(base_url)
    end
  end

  class YarnRestApiClient
    attr_reader :base_url
    attr_reader :connections

    def initialize(base_urls)
      @connections = base_urls.split(",")
      if @connections.length == 1 
        @base_url = @connections[0]

        ### assume connected when only one server given
        @connected = true
      else
        @base_url = nil
      end
    end

    def connected?
      @connected || false
    end

    def base_url
      connect! unless connected?
      @base_url 
    end
    
    def connect!
      return if connected?
   
      zz = @connections.map do |cx|
        begin
          RestClient.get(cx + "/cluster", accept: :json) do |response,request,result| 
            # raises exception if response is not 200-206. Also follows redirects
            response.return!
            raise StandbyRM unless response.headers[:refresh].nil? 
          end

          cx
        rescue StandbyRM, Errno::ECONNREFUSED, RestClient::Exception
          nil
        end
      end

      ### should only be one after compact, but just in case...
      @base_url = zz.compact.sample

      raise Errno::ECONNREFUSED if @base_url.nil? || @base_url.empty?
      @connected = true
    end


    class StandbyRM < Exception
    end

    def get(url, klass, key=nil)
      retried = false
      connect! unless connected?
      begin
        RestClient.get(url, :accept=>:json) do |response, request, result, &blk|
          if response.headers[:refresh]
            # TODO We detect that this is a Standy RM by catching
            # the refresh header it sends, feels like a bit of a hack -
            # is there a better way to do this?
            refresh_url = URI(response.headers[:refresh].split(';').last.split('=',2).last)
            new_base_uri = URI(base_url)
            new_base_uri.host = refresh_url.host
            new_base_uri.port = refresh_url.port
            @base_url = new_base_uri.to_s
            url = refresh_url.to_s
            raise StandbyRM
          end
          case response.code
          when 200
            j = JSON.parse(response.body)<commit_msg>fix up some variable names<commit_after>#
# Copyright (C) 2014 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License.  You
# may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.  See the License for the specific language governing
# permissions and limitations under the License.
#

require 'rest-client'
require 'json'
require 'hashie'

module RubyYarn

  class ApiMash < Hashie::Mash
    attr_reader :client
    def initialize(hash, client=nil)
      @client = client
      super(hash)
    end
  end

  class YarnApiObject
    attr_reader :client
    def initialize(base_url)
      @client = YarnRestApiClient.new(base_url)
    end
  end

  class YarnRestApiClient
    attr_reader :base_url
    attr_reader :resource_managers

    def initialize(base_urls)
      @resource_managers = base_urls.split(",")
      if @resource_managers.length == 1 
        @base_url = @resource_managers[0]

        ### assume rm_alive when only one server given
        @rm_alive = true
      else
        @base_url = nil
      end
    end

    def rm_alive?
      @rm_alive || false
    end

    def base_url
      attach_to_rm! unless rm_alive?
      @base_url 
    end
    
    def attach_to_rm!
      return if rm_alive?
   
      zz = @resource_managers.map do |cx|
        begin
          RestClient.get(cx + "/cluster", accept: :json) do |response,request,result| 
            # raises exception if response is not 200-206. Also follows redirects
            response.return!
            raise StandbyRM unless response.headers[:refresh].nil? 
          end

          cx
        rescue StandbyRM, Errno::ECONNREFUSED, RestClient::Exception
          nil
        end
      end

      ### should only be one after compact, but just in case...
      @base_url = zz.compact.sample

      raise Errno::ECONNREFUSED if @base_url.nil? || @base_url.empty?
      @rm_alive = true
    end


    class StandbyRM < Exception
    end

    def get(url, klass, key=nil)
      retried = false
      attach_to_rm! unless rm_alive?
      begin
        RestClient.get(url, :accept=>:json) do |response, request, result, &blk|
          if response.headers[:refresh]
            # TODO We detect that this is a Standy RM by catching
            # the refresh header it sends, feels like a bit of a hack -
            # is there a better way to do this?
            refresh_url = URI(response.headers[:refresh].split(';').last.split('=',2).last)
            new_base_uri = URI(base_url)
            new_base_uri.host = refresh_url.host
            new_base_uri.port = refresh_url.port
            @base_url = new_base_uri.to_s
            url = refresh_url.to_s
            raise StandbyRM
          end
          case response.code
          when 200
            j = JSON.parse(response.body)
{'fraction_non_alphanumeric': 0.07934959349593496, 'fraction_numerical': 0.0065040650406504065, 'mean_word_length': 2.2945902517407606, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2943241', 'n_tokens_mistral': 1827, 'n_tokens_neox': 1764, 'n_words': 583}