<commit_before>require 'file_gsub'

namespace :loba do
  desc 'recursive search for Loba lines (ignores comments or quoted occurances unless [:any] argument supplied)'
  task :grep, :options do |t, args|
    args.with_defaults(options: nil)

    options = args[:options]
    if options && options.downcase == ':any'
      Loba::TaskSupport.grep({any: true})
    else
      Loba::TaskSupport.grep
    end
  end
end<commit_msg>add rake tasks for commenting, uncommenting, and deleting loba lines from code<commit_after>require 'file_gsub'

namespace :loba do
  desc 'recursive search for Loba lines (ignores comments or quoted occurances unless [:any] argument supplied)'
  task :grep, :options do |t, args|
    args.with_defaults(options: nil)

    options = args[:options]
    if options && options.downcase == ':any'
      Loba::TaskSupport.grep({any: true})
    else
      Loba::TaskSupport.grep
    end
  end

  desc '[EXPERIMENTAL] comments out any active Loba lines (recursive file search)'
  task :comment do
    STDOUT.puts 'This task will put "#LOBA_COMMENTED" in front of Loba lines that are not commented out already or that do not appear in quotes.'
    STDOUT.puts 'To do this, this task will modify your source code files!'
    STDOUT.puts 'This feature is currently experimental and it is strongly recommended that you consider committing your code first in case you do not like the changes.'
    STDOUT.print 'Continue? [y/N] '
    answer = STDIN.gets.chomp.strip.downcase
    if answer[0] == 'y'
      Loba::TaskSupport.comment_all
    else
      STDOUT.puts 'Aborted.'
    end
  end

  desc '[EXPERIMENTAL] restores Loba lines commented-out by :comment rake task (recursive file search)'
  task :uncomment do
    STDOUT.puts 'This task will remove "#LOBA_COMMENTED" before Loba lines that do not appear in quotes, thereby making them active again.'
    STDOUT.puts 'To do this, this task will modify your source code files!'
    STDOUT.puts 'This feature is currently experimental and it is strongly recommended that you consider committing your code first in case you do not like the changes.'
    STDOUT.print 'Continue? [y/N] '
    answer = STDIN.gets.chomp.strip.downcase
    if answer[0] == 'y'
      Loba::TaskSupport.uncomment_all
    else
      STDOUT.puts 'Aborted.'
    end
  end

  desc '[EXPERIMENTAL] deletes any active Loba lines (recursive file search)'
  task :clean do
    STDOUT.puts 'This task will delete any Loba lines that are not commented out already and that do not appear in quotes.'
    STDOUT.puts 'To do this, this task will modify your source code files!'
    STDOUT.puts 'This feature is currently experimental and it is strongly recommended that you consider committing your code first in case you do not like the changes.'
    STDOUT.print 'Continue? [y/N] '
    answer = STDIN.gets.chomp.strip.downcase
    if answer[0] == 'y'
      Loba::TaskSupport.clean
    else
      STDOUT.puts 'Aborted.'
    end
  end

end
{'fraction_non_alphanumeric': 0.08029689608636977, 'fraction_numerical': 0.0010121457489878543, 'mean_word_length': 3.519817073170732, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6095042', 'n_tokens_mistral': 908, 'n_tokens_neox': 872, 'n_words': 356}