<commit_before>class MetricPoller
  include Celluloid
  include CloudWatch

  attr_accessor :namespace, :metric_name
  attr_reader :poll_interval
  attr_reader :poll_timer

  #TODO optional dimensions
  def initialize(namespace, metric_name, options={})
    @namespace     = namespace
    @metric_name   = metric_name
    @poll_interval = options[:poll_interval] || 60
    @poll_timer    = every(@poll_interval) { async.poll }
    async.poll
  end

  def poll
    Logger.debug "polling"
    #TODO poll all metrics
    # if the time scale is longer than 60 seconds, split that if cloudwatch returns an error
    # 
    # if cloudwatch returns a 'too many metrics' error, split metrics into multiple arrays by dimension
    # spin up a metric poller for each dimension slice
    # stop the poll timer (let the subpollers handle it)
    # 
    # is the cloudwatch 'too meny metrics' error separate from transient retryable errors?
  end

  def poll_interval=(interval)
    @poll_interval = interval
    @poll_timer.cancel
    @poll_timer = every(@poll_interval) { async.poll }
  end
end<commit_msg>get datapoints when polling<commit_after>class MetricPoller
  include Celluloid
  include CloudWatch

  attr_accessor :namespace, :metric_name
  attr_reader :poll_interval
  attr_reader :poll_timer

  #TODO optional dimensions
  def initialize(namespace, metric_name, options={})
    @namespace     = namespace
    @metric_name   = metric_name
    @poll_interval = options[:poll_interval] || 60
    @poll_timer    = every(@poll_interval) { async.poll }
    async.poll
  end

  def poll
    Logger.debug "polling"
    datapoints = metric.statistics(start_time: Time.now - 3600, end_time: Time.now, statistics: ['SampleCount']).datapoints

    puts "datapoints: #{datapoints}"
    #TODO poll all metrics
    # if the time scale is longer than 60 seconds, split that if cloudwatch returns an error
    # 
    # if cloudwatch returns a 'too many metrics' error, split metrics into multiple arrays by dimension
    # spin up a metric poller for each dimension slice
    # stop the poll timer (let the subpollers handle it)
    # 
    # is the cloudwatch 'too meny metrics' error separate from transient retryable errors?
  end

  def metric
    @metric ||= AWS::CloudWatch::Metric.new(@namespace, @metric_name)
  end

  def poll_interval=(interval)
    @poll_interval = interval
    @poll_timer.cancel
    @poll_timer = every(@poll_interval) { async.poll }
  end
end
{'fraction_non_alphanumeric': 0.0728827361563518, 'fraction_numerical': 0.004885993485342019, 'mean_word_length': 3.6358490566037736, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6096066', 'n_tokens_mistral': 737, 'n_tokens_neox': 707, 'n_words': 243}