<commit_before>desc 'Retrieves all data from all sources'
task data: [:'data:git', :'data:controllr']

namespace :data do
  desc 'Retrieves all data from the git repositories (github and gitlab)'
  task :git do
    GitFetcher.new.run
  end

  desc 'Retrieves all data from the controllr'
  task :controllr do
    ControllrFetcher.new.run
  end

  desc 'Clears all the data'
  task :clear do
    DataStore.clear
  end
end

desc 'Clones and updates all git repositories (github and gitlab)'
task :clone_git_repositories do
  GitRepositoryCloner.new.run
end<commit_msg>rake tasks need to load the environment<commit_after>desc 'Retrieves all data from all sources'
task data: [:'data:git', :'data:controllr']

namespace :data do
  desc 'Retrieves all data from the git repositories (github and gitlab)'
  task git: :environment do
    GitFetcher.new.run
  end

  desc 'Retrieves all data from the controllr'
  task controllr: :environment do
    ControllrFetcher.new.run
  end

  desc 'Clears all the data'
  task clear: :environment do
    DataStore.clear
  end
end

desc 'Clones and updates all git repositories (github and gitlab)'
task clone_git_repositories: :environment do
  GitRepositoryCloner.new.run
end
{'fraction_non_alphanumeric': 0.07107438016528926, 'fraction_numerical': 0.0, 'mean_word_length': 4.242424242424242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4968475', 'n_tokens_mistral': 399, 'n_tokens_neox': 378, 'n_words': 145}