<commit_before>require 'yaml'

namespace :db do
  desc 'Upload a scrubbed copy of the database as specified by config/scrub.yml to S3'
  task :scrub => :environment do
    scrub_config = YAML.load_file("#{Rails.root.to_s}/config/brillo.yml")
    Brillo.new(scrub_config, logger: Logger.new(STDOUT)).scrub_to_s3
  end
<commit_msg>[rake tasks] make scrubber quiet by default<commit_after>require 'yaml'

namespace :db do
  desc 'Upload a scrubbed copy of the database as specified by config/scrub.yml to S3'
  task :scrub => :environment do
    scrub_config = YAML.load_file("#{Rails.root.to_s}/config/brillo.yml")
    logger = ENV["VERBOSE"] ? Logger.new(STDOUT) : Rails.logger
    Brillo.new(scrub_config, logger: logger).scrub_to_s3
  end

{'fraction_non_alphanumeric': 0.11351351351351352, 'fraction_numerical': 0.005405405405405406, 'mean_word_length': 4.571428571428571, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1837721', 'n_tokens_mistral': 267, 'n_tokens_neox': 258, 'n_words': 68}