<commit_before>#!/usr/bin/env ruby

unless RUBY_VERSION >= '1.9'
  require 'rubygems'
end

if File.exist?(File.join(Dir.pwd, 'earth.gemspec'))
  require 'bundler'
  Bundler.setup
  $LOAD_PATH.unshift(File.dirname(__FILE__))
  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
end

require 'active_support/all'
require 'active_record'

case ENV['EARTH_DB_ADAPTER']
when 'mysql'
  adapter = 'mysql'
  database = 'test_earth'
  username = 'root'
  password = 'password'
  
  system %{mysql -u #{username} -p#{password} -e "DROP DATABASE #{database}"}
  system %{mysql -u #{username} -p#{password} -e "CREATE DATABASE #{database}"}
when 'sqlite'
  adapter = 'sqlite3'
  database = ':memory:'
  username = nil
  password = <PASSWORD>
else
  adapter = 'postgresql'
  username = ENV['EARTH_POSTGRES_USERNAME'] || `whoami`.chomp
  password = ENV['<PASSWORD>']
  database = ENV['EARTH_POSTGRES_DATABASE'] || 'test_earth'

  createdb_bin = ENV['EARTH_CREATEDB_BIN'] || 'createdb'
  dropdb_bin = ENV['EARTH_DROPDB_BIN'] || 'dropdb'
  system %{#{dropdb_bin} #{database}}
  system %{#{createdb_bin} #{database}}
end<commit_msg>don't always drop/recreate dbs on earth_tester<commit_after>#!/usr/bin/env ruby

unless RUBY_VERSION >= '1.9'
  require 'rubygems'
end

if File.exist?(File.join(Dir.pwd, 'earth.gemspec'))
  require 'bundler'
  Bundler.setup
  $LOAD_PATH.unshift(File.dirname(__FILE__))
  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
end

require 'active_support/all'
require 'active_record'

case ENV['EARTH_DB_ADAPTER']
when 'mysql'
  adapter = 'mysql'
  database = 'test_earth'
  username = 'root'
  password = 'password'
  
  # system %{mysql -u #{username} -p#{password} -e "DROP DATABASE #{database}"}
  # system %{mysql -u #{username} -p#{password} -e "CREATE DATABASE #{database}"}
when 'sqlite'
  adapter = 'sqlite3'
  database = ':memory:'
  username = nil
  password = nil
else
  adapter = 'postgresql'
  username = ENV['EARTH_POSTGRES_USERNAME'] || `whoami`.chomp
  password = ENV['EAR<PASSWORD>']
  database = ENV['EARTH_POSTGRES_DATABASE'] || 'test_earth'

  createdb_bin = ENV['EARTH_CREATEDB_BIN'] || 'createdb'
  dropdb_bin = ENV['EARTH_DROPDB_BIN'] || 'dropdb'
  # system %{#{dropdb_bin} #{database}}
  # system %{#{createdb_bin} #{database}}
end
{'fraction_non_alphanumeric': 0.16789908655937363, 'fraction_numerical': 0.0026098303610265334, 'mean_word_length': 5.10079575596817, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2441507', 'n_tokens_mistral': 893, 'n_tokens_neox': 884, 'n_words': 176}