<commit_before>namespace :scrape do 
  # Get all gems from github
  # Can call for a single letter and x amount of gems:
  #   ex. rake scrape:gems[F, 20]
  task :gems, [:letters_to_traverse, :upsert_limit] => :environment do |t, args|
    require_relative "../scraper/ruby_gems_scraper"

    binding.pry
    options = args.to_h
    if args.letters_to_traverse
      options[:letters_to_traverse] = args.letters_to_traverse.split(" ")
    end

    RubyGemsScraper.upsert_gems(options)
  end

  # Get github repo information for each gem
  task :github => :environment do <commit_msg>removed binding.pry<commit_after>namespace :scrape do 
  # Get all gems from github
  # Can call for a single letter and x amount of gems:
  #   ex. rake scrape:gems[F, 20]
  task :gems, [:letters_to_traverse, :upsert_limit] => :environment do |t, args|
    require_relative "../scraper/ruby_gems_scraper"

    options = args.to_h
    if args.letters_to_traverse
      options[:letters_to_traverse] = args.letters_to_traverse.split(" ")
    end

    RubyGemsScraper.upsert_gems(options)
  end

  # Get github repo information for each gem
  task :github => :environment do 
{'fraction_non_alphanumeric': 0.09342560553633218, 'fraction_numerical': 0.0034602076124567475, 'mean_word_length': 3.7224489795918365, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5802493', 'n_tokens_mistral': 424, 'n_tokens_neox': 382, 'n_words': 108}