<commit_before>namespace :metadata do

  metadata_directory = 'tmp/METADATA.jl'

  desc "clone metadata project"
  task clone: :environment do
    `git clone https://github.com/JuliaLang/METADATA.jl.git ./#{metadata_directory}`
  end

  desc "update local metadata project"
  task pull: :environment do
    `git -C #{metadata_directory} pull`
  end

  desc "digest metadata into database"
  task digest: :environment do
    Dir.foreach(metadata_directory) do |directory|
      next if directory.starts_with? '.'
      next if File.file? \
        "#{metadata_directory}/#{directory}"

      Package.create! name: directory
    end
  end
<commit_msg>Add repositories with urls to metadata digest<commit_after>namespace :metadata do

  metadata_directory = 'tmp/METADATA.jl'

  desc "clone metadata project"
  task clone: :environment do
    `git clone https://github.com/JuliaLang/METADATA.jl.git ./#{metadata_directory}`
  end

  desc "update local metadata project"
  task pull: :environment do
    `git -C #{metadata_directory} pull`
  end

  desc "digest metadata into database"
  task digest: :environment do
    Dir.foreach(metadata_directory) do |directory|
      next if directory.starts_with? '.'
      next if File.file? \
        "#{metadata_directory}/#{directory}"

      package = Package.create! name: directory

      url_file = "#{metadata_directory}/#{directory}/url"
      url = File.open(url_file).read.strip

      repository = Repository.create! url: url, package: package
    end
  end

{'fraction_non_alphanumeric': 0.10477453580901856, 'fraction_numerical': 0.0, 'mean_word_length': 3.464497041420118, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1271730', 'n_tokens_mistral': 458, 'n_tokens_neox': 440, 'n_words': 129}