<commit_before>#!/usr/bin/ruby

# json_stats_fetcher.rb - Publish Ostrich stats to Ganglia.
#
# The latest version is always available at:
# http://github.com/twitter/ostrich/blob/master/src/scripts/json_stats_fetcher.rb
#

require 'rubygems'
require 'getoptlong'
require 'socket'
require 'json'
require 'timeout'
require 'open-uri'

def valid_gmetric_name?(name)
  # Determines if a gmetric name is valid.
  #
  # Ganglia is very intolerant of metric named with non-standard characters,
  # where non-standard contains most everything other than letters, numbers and
  # some common symbols.
  #
  # Returns true if the metric is a valid gmetric name, otherwise false.
  if name =~ /^[A-Za-z0-9_\-\.]+$/
    return true
  else
    $stderr.puts "Metric <#{name}> contains invalid characters."
    return false
  end
end

def report_metric(name, value, units)
  if not valid_gmetric_name?(name)
    return
  end

  if $report_to_ganglia
    system("gmetric -t float -n \"#{$ganglia_prefix}#{name}\" -v \"#{value}\" -u \"#{units}\" -d #{$stat_timeout}")
  else
    puts "#{$ganglia_prefix}#{name}=#{value}"
  end
end

$ostrich3 = false # guessed ostrich version
$report_to_ganglia = true
$ganglia_prefix = ''
$stat_timeout = 86400
$pattern = /^x-/

hostname = "localhost"
port = 9989
use_web = false

def usage(port)
  puts
  puts "usage: json_stats_fetcher.rb [options]"
  puts "options:"
  puts "    -n              say what I would report, but don't report it"
  puts "    -w              use web interface"<commit_msg>from george wu: better to just fix the metric name than silently do the wrong thing<commit_after>#!/usr/bin/ruby

# json_stats_fetcher.rb - Publish Ostrich stats to Ganglia.
#
# The latest version is always available at:
# http://github.com/twitter/ostrich/blob/master/src/scripts/json_stats_fetcher.rb
#

require 'rubygems'
require 'getoptlong'
require 'socket'
require 'json'
require 'timeout'
require 'open-uri'

def report_metric(name, value, units)
  name = name.gsub(/[^A-Za-z0-9_\-\.]/, "_")

  if $report_to_ganglia
    system("gmetric -t float -n \"#{$ganglia_prefix}#{name}\" -v \"#{value}\" -u \"#{units}\" -d #{$stat_timeout}")
  else
    puts "#{$ganglia_prefix}#{name}=#{value}"
  end
end

$ostrich3 = false # guessed ostrich version
$report_to_ganglia = true
$ganglia_prefix = ''
$stat_timeout = 86400
$pattern = /^x-/

hostname = "localhost"
port = 9989
use_web = false

def usage(port)
  puts
  puts "usage: json_stats_fetcher.rb [options]"
  puts "options:"
  puts "    -n              say what I would report, but don't report it"
  puts "    -w              use web interface"
{'fraction_non_alphanumeric': 0.13188073394495411, 'fraction_numerical': 0.009174311926605505, 'mean_word_length': 3.5434027777777777, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3847959', 'n_tokens_mistral': 920, 'n_tokens_neox': 916, 'n_words': 265}