<commit_before>namespace :projects do
  task normalize_licenses: :environment do
    Project.where("licenses <> ''").find_each do |project|
      project.normalize_licenses
      project.save
    end
  end

  task recreate_index: :environment do
    # If the index doesn't exists can't be deleted, returns 404, carry on
    Project.__elasticsearch__.client.indices.delete index: 'projects' rescue nil
    Project.__elasticsearch__.create_index! force: true
  end

  task reindex: [:environment, :recreate_index] do
    Project.import query: -> { includes([:github_repository]) }
  end

  task update_source_ranks: :environment do
    ids = Project.order('updated_at ASC').limit(10_000).pluck(:id).to_a
    Project.includes([{:github_repository => :readme}, :versions, :github_contributions]).where(id: ids).find_each(&:update_source_rank)
  end

  task add_project_id_to_deps: :environment do
    Dependency.includes(version: :project).find_each do |dep|
      dep.update_attribute(:project_id, dep.find_project_id)
    end
  end

  task download_readmes: :environment do
    GithubRepository.find_each(&:download_readme)
  end

  task download_tags: :environment do
    GithubRepository.find_each(&:download_tags)<commit_msg>More efficient normalize_licenses task<commit_after>namespace :projects do
  task normalize_licenses: :environment do
    Project.where("licenses <> ''").find_each do |project|
      project.normalize_licenses
      project.save if project.changed?
    end
  end

  task recreate_index: :environment do
    # If the index doesn't exists can't be deleted, returns 404, carry on
    Project.__elasticsearch__.client.indices.delete index: 'projects' rescue nil
    Project.__elasticsearch__.create_index! force: true
  end

  task reindex: [:environment, :recreate_index] do
    Project.import query: -> { includes([:github_repository]) }
  end

  task update_source_ranks: :environment do
    ids = Project.order('updated_at ASC').limit(10_000).pluck(:id).to_a
    Project.includes([{:github_repository => :readme}, :versions, :github_contributions]).where(id: ids).find_each(&:update_source_rank)
  end

  task add_project_id_to_deps: :environment do
    Dependency.includes(version: :project).find_each do |dep|
      dep.update_attribute(:project_id, dep.find_project_id)
    end
  end

  task download_readmes: :environment do
    GithubRepository.find_each(&:download_readme)
  end

  task download_tags: :environment do
    GithubRepository.find_each(&:download_tags)
{'fraction_non_alphanumeric': 0.10092110532639167, 'fraction_numerical': 0.006407689227072487, 'mean_word_length': 4.742528735632184, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1271732', 'n_tokens_mistral': 772, 'n_tokens_neox': 731, 'n_words': 174}