<commit_before>namespace :whitespace do
  desc 'Removes trailing whitespace'
  task :cleanup do
    sh %{find . -name '*.rb' -exec sed -i '' 's/ *$//g' {} \\;}
  end
  desc 'Converts hard-tabs into two-space soft-tabs'
  task :retab do
    sh %{find . -name '*.rb' -exec sed -i '' 's/\t/  /g' {} \\;}
  end
  desc 'Remove consecutive blank lines'
  task :scrub_gratuitous_newlines do
    sh %{find . -name '*.rb' -exec sed -i '' '/./,/^$/!d' {} \\;}
  end
end
<commit_msg>Improve whitespace:cleanup and whitespace:retab<commit_after>namespace :whitespace do
  desc 'Removes trailing whitespace'
  task :cleanup do
    sh %{for f in `find . -type f | grep -v -e '.git/' -e 'public/' -e '.png'`;
          do cat $f | sed 's/[ \t]*$//' > tmp; cp tmp $f; rm tmp; echo -n .;
        done}
  end
  desc 'Converts hard-tabs into two-space soft-tabs'
  task :retab do
    sh %{for f in `find . -type f | grep -v -e '.git/' -e 'public/' -e '.png'`;
          do cat $f | sed 's/\t/  /g' > tmp; cp tmp $f; rm tmp; echo -n .;
        done}
  end
  desc 'Remove consecutive blank lines'
  task :scrub_gratuitous_newlines do
    sh %{find . -name '*.rb' -exec sed -i '' '/./,/^$/!d' {} \\;}
  end
end
{'fraction_non_alphanumeric': 0.18771043771043772, 'fraction_numerical': 0.0, 'mean_word_length': 2.248633879781421, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1650050', 'n_tokens_mistral': 464, 'n_tokens_neox': 449, 'n_words': 159}