<commit_before>#!/usr/bin/env ruby

require 'pathname'
require 'yaml'
require 'erb'

$environment = 'production'

$root = Pathname.new(__FILE__).dirname.dirname.realpath
$backgroundrb_config_file = "#{$root}/config/backgroundrb.yml"
$sphinx_config_file = "#{$root}/config/#{$environment}.sphinx.conf"
$sphinx_db_file = "#{$root}/db/sphinx/production/page_terms_core.sph"
$ts_config_file = "#{$root}/config/sphinx.yml"
$backgroundrb_port = 0

Dir.chdir $root

def process_command
  case ARGV[0]
  when 'start'
    start_sphinx
    ensure_stop_backgroundrb    # make sure it is dead before we try to start it
    sleep 1                     # otherwise it might not start and might not give us an error
    start_backgroundrb
  when 'stop'
    stop_sphinx
    stop_backgroundrb
  when 'restart'
    restart_sphinx
    stop_backgroundrb
    sleep 1
    start_backgroundrb
  when 'status'
    status_sphinx
    status_backgroundrb
  when 'start-sphinx'
    start_sphinx
  when 'stop-sphinx'
    stop_sphinx
  when 'start-bgrb'
    start_backgroundrb
  when 'stop-bgrb'
    stop_backgroundrb
  end
end

##
## SPHINX
##

def ensure_sphinx_config
  assert_file_exists($ts_config_file)
  unless File.exists? $sphinx_config_file
    puts 'No sphinx configuration exists, creating one (%s)' % $sphinx_config_file
    system("rake thinking_sphinx:configure RAILS_ENV=#{$environment}")
    assert_file_exists($sphinx_config_file)
  end
  unless File.exists? $sphinx_db_file
    puts "No sphinx db exists... creating index now"
    system("rake thinking_sphinx:index RAILS_ENV=#{$environment}")
  end
end

def restart_sphinx
  ensure_sphinx_config
  system("rake thinking_sphinx:running_start RAILS_ENV=#{$environment}")
end

def start_sphinx
  ensure_sphinx_config
  puts "Starting sphinx searchd..."
  system("rake thinking_sphinx:start RAILS_ENV=#{$environment}")
end

def stop_sphinx
  puts "Stopping sphinx searchd..."
  system("rake thinking_sphinx:stop RAILS_ENV=#{$environment}")
  pid = `pgrep searchd`.chomp
  if pid.any?
    puts "  ERROR: searchd is still running with process id %s" % pid
  end
end

def status_sphinx
  pid = `pgrep searchd`.chomp
  if pid.any?
    puts "Sphinx searchd running"
    puts "  process id %s" % pid
    puts "  running for %s" % process_elapsed_time(pid)
  else
    puts "Sphinx searchd NOT running."
  end
end

##
## BACKGROUNDRB
##

def bgrb_pid
  `exec pgrep -f backgroundrb`.chomp
end

def load_backgroundrb_config
  $brb_config ||= begin
    assert_file_exists($backgroundrb_config_file)
    conf = YAML::load(ERB.new(IO.read($backgroundrb_config_file)).result)
    $backgroundrb_port = conf[:backgroundrb][:port]
    $backgroundrb_pid_file = "#{$root}/tmp/pids/backgroundrb_#{$backgroundrb_port}.pid"
    conf
  end
end
<commit_msg>removed weird 'exec'<commit_after>#!/usr/bin/env ruby

require 'pathname'
require 'yaml'
require 'erb'

$environment = 'production'

$root = Pathname.new(__FILE__).dirname.dirname.realpath
$backgroundrb_config_file = "#{$root}/config/backgroundrb.yml"
$sphinx_config_file = "#{$root}/config/#{$environment}.sphinx.conf"
$sphinx_db_file = "#{$root}/db/sphinx/production/page_terms_core.sph"
$ts_config_file = "#{$root}/config/sphinx.yml"
$backgroundrb_port = 0

Dir.chdir $root

def process_command
  case ARGV[0]
  when 'start'
    start_sphinx
    ensure_stop_backgroundrb    # make sure it is dead before we try to start it
    sleep 1                     # otherwise it might not start and might not give us an error
    start_backgroundrb
  when 'stop'
    stop_sphinx
    stop_backgroundrb
  when 'restart'
    restart_sphinx
    stop_backgroundrb
    sleep 1
    start_backgroundrb
  when 'status'
    status_sphinx
    status_backgroundrb
  when 'start-sphinx'
    start_sphinx
  when 'stop-sphinx'
    stop_sphinx
  when 'start-bgrb'
    start_backgroundrb
  when 'stop-bgrb'
    stop_backgroundrb
  end
end

##
## SPHINX
##

def ensure_sphinx_config
  assert_file_exists($ts_config_file)
  unless File.exists? $sphinx_config_file
    puts 'No sphinx configuration exists, creating one (%s)' % $sphinx_config_file
    system("rake thinking_sphinx:configure RAILS_ENV=#{$environment}")
    assert_file_exists($sphinx_config_file)
  end
  unless File.exists? $sphinx_db_file
    puts "No sphinx db exists... creating index now"
    system("rake thinking_sphinx:index RAILS_ENV=#{$environment}")
  end
end

def restart_sphinx
  ensure_sphinx_config
  system("rake thinking_sphinx:running_start RAILS_ENV=#{$environment}")
end

def start_sphinx
  ensure_sphinx_config
  puts "Starting sphinx searchd..."
  system("rake thinking_sphinx:start RAILS_ENV=#{$environment}")
end

def stop_sphinx
  puts "Stopping sphinx searchd..."
  system("rake thinking_sphinx:stop RAILS_ENV=#{$environment}")
  pid = `pgrep searchd`.chomp
  if pid.any?
    puts "  ERROR: searchd is still running with process id %s" % pid
  end
end

def status_sphinx
  pid = `pgrep searchd`.chomp
  if pid.any?
    puts "Sphinx searchd running"
    puts "  process id %s" % pid
    puts "  running for %s" % process_elapsed_time(pid)
  else
    puts "Sphinx searchd NOT running."
  end
end

##
## BACKGROUNDRB
##

def bgrb_pid
  `pgrep -f backgroundrb`.chomp
end

def load_backgroundrb_config
  $brb_config ||= begin
    assert_file_exists($backgroundrb_config_file)
    conf = YAML::load(ERB.new(IO.read($backgroundrb_config_file)).result)
    $backgroundrb_port = conf[:backgroundrb][:port]
    $backgroundrb_pid_file = "#{$root}/tmp/pids/backgroundrb_#{$backgroundrb_port}.pid"
    conf
  end
end

{'fraction_non_alphanumeric': 0.10220296135789093, 'fraction_numerical': 0.001444564824846515, 'mean_word_length': 4.235349716446125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7472110', 'n_tokens_mistral': 1990, 'n_tokens_neox': 1974, 'n_words': 441}