<commit_before>module WaybackArchiver
  class Archive
    MAX_THREAD_COUNT = 8

    def self.post(all_urls)
      puts "Request will be sent with max #{MAX_THREAD_COUNT} parallel threads"
  
      puts "Total urls to be sent: #{all_urls.length}"
      threads    = []
      group_size = (all_urls.length / MAX_THREAD_COUNT) + 1
      all_urls.each_slice(group_size).to_a.each do |urls|
        threads << Thread.new do
          urls.each_with_index do |url, index|
            request_url = "#{BASE_URL}#{url}"
            begin
              res = Request.get_response(request_url)
              print "#{url}    #{res.code} => #{res.message} \n"
            rescue Exception => e
              puts "Error message: #{e.message}"
              puts "Failed to archive: #{url}"
            end
          end
        end
      end
      threads.each_with_index do |thread, index|
        progress = '['
        progress << '#' * index
        progress << ' ' * (threads.length - index)
        progress << ']'
        puts "[PROGRESS] #{progress} (#{index}/#{threads.length})"
        thread.join
      end
      all_urls
    end
  end
end<commit_msg>Progress bar fix<commit_after>module WaybackArchiver
  class Archive
    MAX_THREAD_COUNT = 8

    def self.post(all_urls)
      puts "Request will be sent with max #{MAX_THREAD_COUNT} parallel threads"
  
      puts "Total urls to be sent: #{all_urls.length}"
      threads    = []
      group_size = (all_urls.length / MAX_THREAD_COUNT) + 1
      all_urls.each_slice(group_size).to_a.each do |urls|
        threads << Thread.new do
          urls.each_with_index do |url, index|
            request_url = "#{BASE_URL}#{url}"
            begin
              res = Request.get_response(request_url)
              print "#{url}    #{res.code} => #{res.message} \n"
            rescue Exception => e
              puts "Error message: #{e.message}"
              puts "Failed to archive: #{url}"
            end
          end
        end
      end
      threads.each_with_index do |thread, index|
        print_index = index + 1
        progress = '['
        progress << '#' * print_index
        progress << ' ' * (threads.length - print_index)
        progress << ']'
        procent = ((print_index.to_f/threads.length.to_f) * 100).round(0)
        puts "[PROGRESS] #{progress} #{procent}% (#{print_index}/#{threads.length})"
        thread.join
      end
      all_urls
    end
  end
end
{'fraction_non_alphanumeric': 0.11593609176566981, 'fraction_numerical': 0.0036870135190495697, 'mean_word_length': 1.9457177322074788, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5426449', 'n_tokens_mistral': 740, 'n_tokens_neox': 754, 'n_words': 182}