<commit_before>require 'awesome_print'
require 'ruby-progressbar'
require 'thread/pool'

# Change so that every 100 is put into the memory instead of active record.
namespace :subreddits do
  desc 'Seeds SubReddits'
  task seed: :environment do
    pool = Thread.pool(20)

    SubReddit.delete_all
    RelatedSubReddit.delete_all
    require 'csv'
    path = "#{Rails.root}/lib/tasks/all_subreddits.csv"

    csv_text = File.read(path)
    csv = CSV.parse(csv_text, headers: false)
    limit_size = 1000
    # limit_size = csv.size
    
    to_ignore = ['photos', 'hackers']


    csv.map! do |_key, value|
      value.to_sym
    end
    p "#{csv.size} tables"
    subreddits = []

    #progress_bar = ProgressBar.create(title: 'subreddits:seed',
    #                                  starting_at: 0,
    #                                   total: limit_size,
    #                                  format: '%a %e %P% Processed: %c from %C')

    csv.first(limit_size).each do |sub_reddit_sym|
      #pool.process do
      sub_reddit_name = sub_reddit_sym.to_s
        url = "http://reddit.com/r/#{sub_reddit_sym}.json"
        if to_ignore.include?(sub_reddit_name)
          #p "ignoring #{sub_reddit_name}"
          p "ignoring #{url}"
          #progress_bar.increment
        else
          p "getting  #{url}"
          sub_reddit = SubReddit.new(name: sub_reddit_name.titleize, url: url)
<commit_msg>Minor (correct whitespace)<commit_after>require 'awesome_print'
require 'ruby-progressbar'
require 'thread/pool'

# Change so that every 100 is put into the memory instead of active record.
namespace :subreddits do
  desc 'Seeds SubReddits'
  task seed: :environment do
    pool = Thread.pool(20)

    SubReddit.delete_all
    RelatedSubReddit.delete_all
    require 'csv'
    path = "#{Rails.root}/lib/tasks/all_subreddits.csv"

    csv_text = File.read(path)
    csv = CSV.parse(csv_text, headers: false)
    limit_size = 1000
    # limit_size = csv.size
    
    to_ignore = ['photos', 'hackers']


    csv.map! do |_key, value|
      value.to_sym
    end
    p "#{csv.size} tables"
    subreddits = []

    #progress_bar = ProgressBar.create(title: 'subreddits:seed',
    #                                  starting_at: 0,
    #                                   total: limit_size,
    #                                  format: '%a %e %P% Processed: %c from %C')

    csv.first(limit_size).each do |sub_reddit_sym|
      #pool.process do
        sub_reddit_name = sub_reddit_sym.to_s
        url = "http://reddit.com/r/#{sub_reddit_sym}.json"
        if to_ignore.include?(sub_reddit_name)
          #p "ignoring #{sub_reddit_name}"
          p "ignoring #{url}"
          #progress_bar.increment
        else
          p "getting  #{url}"
          sub_reddit = SubReddit.new(name: sub_reddit_name.titleize, url: url)

{'fraction_non_alphanumeric': 0.10942463819272856, 'fraction_numerical': 0.0070596540769502295, 'mean_word_length': 2.216799091940976, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6694422', 'n_tokens_mistral': 973, 'n_tokens_neox': 939, 'n_words': 208}