<commit_before>#!/usr/bin/env ruby
$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
CONNERRHELP = <<EOC

---------------------------------------------------------------------------
Apparently we could not connect to the redis instance.

By default the watcher will attempt to connect to "redis://localhost:6379/0"
If your redis instance is listening elsewhere, please start like so:

REDIS_URL="redis://hostname:port/dbnum" noah-watcher.rb

This will be rectified in a future release. Sorry about that.
----------------------------------------------------------------------------
EOC

  HELP = <<-EOH
  ---------------------------------------------------------------------------------------
  Unfortunately, the agent script has some difficult requirements right now.
  Please rerun with the '--depinstall' option to have them installed for you and try again
  ---------------------------------------------------------------------------------------
  EOH
require 'rubygems'
require 'slop'
require 'logger'
require 'json'

opts = Slop.parse do
  banner "Usage: noah-watcher.rb [options]"
  on '--depinstall', "Installs additional dependencies" do
    puts "Installing dependencies"
    puts "em-hiredis..."
    `gem install em-hiredis`
    puts "em-http-request prerelease..."
    `gem install em-http-request --pre`
    puts "cookiejar...."
    `gem install cookiejar`
    exit
  end
  on :h, :help, 'Print help', :tail => true do
    puts help
    exit
  end
end

begin
  require 'em-hiredis'
  require 'eventmachine'
  require 'em-http-request'
  require 'cookiejar'
rescue LoadError => e
  puts e.message
  puts HELP
  exit
end

begin
  require 'noah'
  require 'noah/agent'
rescue Errno::ECONNREFUSED
  puts CONNERRHELP
  exit
end

Noah::Log.logger = Logger.new(STDOUT)
LOGGER = Noah::Log.logger
LOGGER.progname = __FILE__

EventMachine.run do
  EM.error_handler do |e|
    LOGGER.warn(e)
  end
  trap("INT") { LOGGER.debug("Shutting down. Watches will not be fired");EM.stop }
  noah = Noah::Agent.new<commit_msg>adding logging options and fixing the redis url mess<commit_after>#!/usr/bin/env ruby
$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
CONNERRHELP = <<EOC

---------------------------------------------------------------------------
Apparently we could not connect to the redis instance.

By default the watcher will attempt to connect to "redis://localhost:6379/0"
If your redis instance is listening elsewhere, please start like so:

REDIS_URL="redis://hostname:port/dbnum" noah-watcher.rb

This will be rectified in a future release. Sorry about that.
----------------------------------------------------------------------------
EOC

  HELP = <<-EOH
  ---------------------------------------------------------------------------------------
  Unfortunately, the agent script has some difficult requirements right now.
  Please rerun with the '--depinstall' option to have them installed for you and try again
  ---------------------------------------------------------------------------------------
  EOH
require 'rubygems'
require 'slop'
require 'logger'
require 'json'

opts = Slop.parse do
  banner "Usage: noah-watcher.rb [options]"
  on :r, :redis_url, 'Redis URL to use. This MUST match what the Noah server is using for now', :default => 'redis://localhost:6379/0', :argument => true
  on :o, :output, 'Log destination - full file path. Default is STDOUT', :default => STDOUT, :argument => true
  on '--depinstall', "Installs additional dependencies" do
    puts "Installing dependencies"
    puts "em-hiredis..."
    `gem install em-hiredis`
    puts "em-http-request prerelease..."
    `gem install em-http-request --pre`
    puts "cookiejar...."
    `gem install cookiejar`
    exit
  end
  on :h, :help, 'Print help', :tail => true do
    puts help
    exit
  end
end

begin
  require 'em-hiredis'
  require 'eventmachine'
  require 'em-http-request'
  require 'cookiejar'
rescue LoadError => e
  puts e.message
  puts HELP
  exit
end

ENV['REDIS_URL'] = opts[:redis_url]

begin
  require 'noah'
  require 'noah/agent'
rescue Errno::ECONNREFUSED
  puts CONNERRHELP
  exit
end

Noah::Log.logger = Logger.new(opts[:output])
LOGGER = Noah::Log.logger
LOGGER.progname = __FILE__

EventMachine.run do
  EM.error_handler do |e|
    LOGGER.warn(e)
  end
  trap("INT") { LOGGER.debug("Shutting down. Watches will not be fired");EM.stop }
  noah = Noah::Agent.new
{'fraction_non_alphanumeric': 0.243018018018018, 'fraction_numerical': 0.0033783783783783786, 'mean_word_length': 2.052233676975945, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4076276', 'n_tokens_mistral': 1317, 'n_tokens_neox': 1302, 'n_words': 418}