<commit_before>#! /usr/bin/env ruby

# Usage:
# slurp.rb dump_path db_name

# dump path is on mostfit.in
dump_path = ARGV[0]

# db_name is on the local machine
db_name   = ARGV[1] || File.basename(Dir.pwd)

# scp, and uncompress the database dump
sql_filename = File.basename(dump_path,".bz2")
bz2_filename = dump_path.split("/")[-1]
unless (File.exists?(sql_filename) or File.exists?(bz2_filename))
  fail "Failed to copy." unless system("scp mostfit.in:#{dump_path} .")
end
unless File.exists?(sql_filename)
  puts "Uncompressinging #{bz2_filename} ..."
  fail "Failed to uncompress" unless system("bunzip2 #{bz2_filename}")
end
# backup - defaults to no
print "Backup the current database? [y/N]: "
choice = $stdin.gets.chomp.downcase
system('./bin/dump.rb') if choice == 'y'

# now load the db
fail "Failed to load database dump." unless system("mysql -p -u root #{db_name} < #{sql_filename}")
<commit_msg>improved slurp.rb to use pv is it exists. You now get a progress bar while importing mysql databases. handy for large files<commit_after>#! /usr/bin/env ruby

# Usage:
# slurp.rb dump_path db_name

# dump path is on mostfit.in
dump_path = ARGV[0]

# db_name is on the local machine
db_name   = ARGV[1] || File.basename(Dir.pwd)

# scp, and uncompress the database dump
sql_filename = File.basename(dump_path,".bz2")
bz2_filename = dump_path.split("/")[-1]
unless (File.exists?(sql_filename) or File.exists?(bz2_filename))
  fail "Failed to copy." unless system("scp mostfit.in:#{dump_path} .")
end
unless File.exists?(sql_filename)
  puts "Uncompressinging #{bz2_filename} ..."
  fail "Failed to uncompress" unless system("bunzip2 #{bz2_filename}")
end
# backup - defaults to no
print "Backup the current database? [y/N]: "
choice = $stdin.gets.chomp.downcase
system('./bin/dump.rb') if choice == 'y'

# now load the db
if `pv -V`.empty?
  puts "no pv...sorry. continuing without progress bar"
  fail "Failed to load database dump." unless system("mysql -p -u root #{db_name} < #{sql_filename}")
else
  cmd = "pv #{sql_filename} | mysql -phatstars -u root #{db_name}"
  puts "using #{cmd}"
  fail "Failed to load database dump." unless system(cmd)
end

{'fraction_non_alphanumeric': 0.14061054579093432, 'fraction_numerical': 0.008325624421831638, 'mean_word_length': 4.560411311053985, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1941206', 'n_tokens_mistral': 772, 'n_tokens_neox': 761, 'n_words': 234}