<commit_before>require 'rubygems'
require 'json'
require 'time'
require 'curb'

$LOAD_PATH << './lib'
$LOAD_PATH << '..'
require 'query'
require 'compat'
require 'KibanaConfig'

=begin
= Class: Kelastic
	- Performs an elastic search request
	- Adds Kibana specific object for JS communication
== Parameters:
  query::   The query object to send to ES
  index::   The index to query
=end
class Kelastic
	attr_accessor :response,:url
  def initialize(query,index)

    @url = "http://#{Kelastic.server}/#{index}/_search"
    # TODO: This badly needs error handling for missing indices
  	@response = Kelastic.run(@url,query)

  	@response['kibana'] = {
  		"index"	=> index,
  	}

  end

  def to_s
    JSON.pretty_generate(@response)
  end

  class << self
    def all_indices
      url = "http://#{Kelastic.server}/_status"
      c = Curl::Easy.new(url)
      c.perform
      @status = JSON.parse(c.body_str)
      @status['indices'].keys.sort
    end

    def date_range(from,to)
      (Date.parse(from.getutc.to_s)..Date.parse(to.getutc.to_s)).to_a
    end

    def index_range(from,to)
      if KibanaConfig::Smart_index == true
        requested = date_range(from,to).map{ |date| date.strftime("logstash-%Y.%m.%d") }
        intersection = requested & all_indices
        if intersection.length <= KibanaConfig::Smart_index_limit
          intersection.sort.reverse
        else
          KibanaConfig::Default_index
        end
      else
        KibanaConfig::Default_index
      end
    end

    def server
      list = KibanaConfig::Elasticsearch
      if list.kind_of?(Array)
        $eslb ||= 0
        $eslb = $eslb < list.length ? $eslb : 0
        server = list[$eslb]
        $eslb += 1
        server
      else
        list
      end
    end

    # TODO: Verify this index exists?
    def current_index
      (Time.now.utc).strftime("logstash-%Y.%m.%d")
    end

    def mapping(index)
      url = "http://#{Kelastic.server}/#{index}/_mapping"
      c = Curl::Easy.new(url)
      c.perform
      JSON.parse(c.body_str)
    end

    # It would be nice to handle different types here, but we don't do that
    # anywhere else.
    def field_type(index,field)
      attributes = collect_item_attributes(mapping(index),field)
      attributes['type']
    end

    def collect_item_attributes(h,field)
      result = {}
      h.each do |k, v|
        if k == field
          h[k].each {|k, v| result[k] = v }
        elsif v.is_a? Hash
          collect_item_attributes(h[k],field).each do |k, v|
            result[k] = v
          end
        end<commit_msg>Added smart_index check to current_index function<commit_after>require 'rubygems'
require 'json'
require 'time'
require 'curb'

$LOAD_PATH << './lib'
$LOAD_PATH << '..'
require 'query'
require 'compat'
require 'KibanaConfig'

=begin
= Class: Kelastic
	- Performs an elastic search request
	- Adds Kibana specific object for JS communication
== Parameters:
  query::   The query object to send to ES
  index::   The index to query
=end
class Kelastic
	attr_accessor :response,:url
  def initialize(query,index)

    @url = "http://#{Kelastic.server}/#{index}/_search"
    # TODO: This badly needs error handling for missing indices
  	@response = Kelastic.run(@url,query)

  	@response['kibana'] = {
  		"index"	=> index,
  	}

  end

  def to_s
    JSON.pretty_generate(@response)
  end

  class << self
    def all_indices
      url = "http://#{Kelastic.server}/_status"
      c = Curl::Easy.new(url)
      c.perform
      @status = JSON.parse(c.body_str)
      @status['indices'].keys.sort
    end

    def date_range(from,to)
      (Date.parse(from.getutc.to_s)..Date.parse(to.getutc.to_s)).to_a
    end

    def index_range(from,to)
      if KibanaConfig::Smart_index == true
        requested = date_range(from,to).map{ |date| date.strftime("logstash-%Y.%m.%d") }
        intersection = requested & all_indices
        if intersection.length <= KibanaConfig::Smart_index_limit
          intersection.sort.reverse
        else
          KibanaConfig::Default_index
        end
      else
        KibanaConfig::Default_index
      end
    end

    def server
      list = KibanaConfig::Elasticsearch
      if list.kind_of?(Array)
        $eslb ||= 0
        $eslb = $eslb < list.length ? $eslb : 0
        server = list[$eslb]
        $eslb += 1
        server
      else
        list
      end
    end

    # TODO: Verify this index exists?
    def current_index
      if KibanaConfig::Smart_index == true
        (Time.now.utc).strftime("logstash-%Y.%m.%d")
      else
        KibanaConfig::Default_index
      end
    end

    def mapping(index)
      url = "http://#{Kelastic.server}/#{index}/_mapping"
      c = Curl::Easy.new(url)
      c.perform
      JSON.parse(c.body_str)
    end

    # It would be nice to handle different types here, but we don't do that
    # anywhere else.
    def field_type(index,field)
      attributes = collect_item_attributes(mapping(index),field)
      attributes['type']
    end

    def collect_item_attributes(h,field)
      result = {}
      h.each do |k, v|
        if k == field
          h[k].each {|k, v| result[k] = v }
        elsif v.is_a? Hash
          collect_item_attributes(h[k],field).each do |k, v|
            result[k] = v
          end
        end
{'fraction_non_alphanumeric': 0.11856060606060606, 'fraction_numerical': 0.0011363636363636363, 'mean_word_length': 2.4137039431157077, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4961566', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1744, 'n_words': 414}