<commit_before>#!/usr/bin/env ruby

require 'tempfile'
require 'time'

# Cross-platform way of finding an executable in the $PATH.
#
#   which('ruby') #=> /usr/bin/ruby
#
# Function based on code found at http://stackoverflow.com/a/5471032/221689
def which!(cmd)
  found = which(cmd)
  throw "I could not find #{cmd}" if found.nil?
  found
end

def which(cmd)
  exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
  ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
    exts.each { |ext|
      exe = File.join(path, "#{cmd}#{ext}")
      return exe if File.executable? exe
    }
  end
  nil
end

mysqldump = which!('mysqldump')
mysql = which!('mysql')

puts "mysql found: #{mysql}"
puts "mysqldump found: #{mysqldump}"

exclude = [
  'mysql',                # system database treated specially
  '#mysql50#lost+found',  # horrible non-actual database
  'lost+found',           # see above
  'information_schema',   # we don't ever need to back this up, right?
]

# Get a list of databases other than the ones in the exclude array
query = "select schema_name from schemata where NOT (schema_name IN (#{exclude.map {|x| "'#{x}'"}.join(',')}))"

# This will fail horribly if our excluded databases have shell metacharacters.
# Obviously we should not shell out to mysql at all.
databases = %x{#{mysql} -BN -e "#{query}" information_schema}.split(' ')
<commit_msg>Added performance_schema to list of excluded databases<commit_after>#!/usr/bin/env ruby

require 'tempfile'
require 'time'

# Cross-platform way of finding an executable in the $PATH.
#
#   which('ruby') #=> /usr/bin/ruby
#
# Function based on code found at http://stackoverflow.com/a/5471032/221689
def which!(cmd)
  found = which(cmd)
  throw "I could not find #{cmd}" if found.nil?
  found
end

def which(cmd)
  exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
  ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
    exts.each { |ext|
      exe = File.join(path, "#{cmd}#{ext}")
      return exe if File.executable? exe
    }
  end
  nil
end

mysqldump = which!('mysqldump')
mysql = which!('mysql')

puts "mysql found: #{mysql}"
puts "mysqldump found: #{mysqldump}"

exclude = [
  'mysql',                # system database treated specially
  '#mysql50#lost+found',  # horrible non-actual database
  'lost+found',           # see above
  'information_schema',   # we don't ever need to back this up, right?
  'performance_schema',   # we don't ever need to back this up, right?
]

# Get a list of databases other than the ones in the exclude array
query = "select schema_name from schemata where NOT (schema_name IN (#{exclude.map {|x| "'#{x}'"}.join(',')}))"

# This will fail horribly if our excluded databases have shell metacharacters.
# Obviously we should not shell out to mysql at all.
databases = %x{#{mysql} -BN -e "#{query}" information_schema}.split(' ')

{'fraction_non_alphanumeric': 0.14831697054698456, 'fraction_numerical': 0.010518934081346423, 'mean_word_length': 3.835593220338983, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7403282', 'n_tokens_mistral': 1003, 'n_tokens_neox': 955, 'n_words': 308}