<commit_before># Rake tasks for Core DB
#
# include in Rakefile via:
#
# require 'vault-tools/core_db_tasks'

desc "Pull db/structure.sql from core HEAD"
task :pull_core do
  steps = []
  steps << 'cd contrib/'
  if File.exists?('contrib/core')
    steps << 'rm -rf core'
  end
  steps << 'git clone -n <EMAIL>@github.<EMAIL>:heroku/core --depth 1'
  steps << 'cd core'
  steps << 'git checkout HEAD db/structure.sql'
  sh steps.join(' && ')
end

desc "Drop and create vault-usage-test, core-test and shushu-test databases"
task :create_core_db => [:drop_core_db, :pull_core] do
  sh 'createdb core-test'
  sh 'psql core-test -f contrib/core/db/structure.sql'
end

desc "Drop the vault-usage-test, core-test and shushu-test databases"
task :drop_core_db do
  sh 'dropdb core-test || true'
end<commit_msg>fix to core rake task<commit_after># Rake tasks for Core DB
#
# include in Rakefile via:
#
# require 'vault-tools/core_db_tasks'

desc "Pull db/structure.sql from core HEAD"
task :pull_core do
  steps = []
  steps << 'cd contrib/'
  if File.exists?('contrib/core')
    steps << 'rm -rf core'
  end
  steps << 'git clone -n git@github.com:heroku/core --depth 1'
  steps << 'cd core'
  steps << 'git checkout HEAD db/structure.sql'
  # make sure we don't submodule it
  steps << 'rm -rf .git'
  sh steps.join(' && ')
end

desc "Drop and create vault-usage-test, core-test and shushu-test databases"
task :create_core_db => [:drop_core_db, :pull_core] do
  sh 'createdb core-test'
  sh 'psql core-test -f contrib/core/db/structure.sql'
end

desc "Drop the vault-usage-test, core-test and shushu-test databases"
task :drop_core_db do
  sh 'dropdb core-test || true'
end
{'fraction_non_alphanumeric': 0.1330137807070102, 'fraction_numerical': 0.0011983223487118035, 'mean_word_length': 3.757834757834758, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4418332', 'n_tokens_mistral': 618, 'n_tokens_neox': 599, 'n_words': 191}