<commit_before>require "./cerberus.rb"
require "./thread_pool.rb"
require "zlib"

puts "Loading login page..."
login_details = get_login_details
puts "Got CSRF token: #{login_details[:csrf]}"
puts "Got cookie: #{login_details[:cookie]}"


puts "Logging in..."
cookie = login(login_details[:csrf], login_details[:cookie], "doralon", "")

if (!cookie)
  puts "Failed to log in :("
else
  puts "Logged in!"

  puts "Got updated cookie: #{cookie}"
  puts "Requesting directory list..."
  files = get_file_listing(cookie)

  download_tasks = []

  Dir.mkdir './files' unless File.exist? './files'
  files.each_with_index do |filename, idx|
    download_tasks.push(Proc.new {
      attempt = 0
      success = false
      while (!success) do
        begin
          puts "Downloading file #{filename} (#{idx}/#{files.count}) Attempt #{attempt}"
          file_data = download_file(cookie, filename)

          if (filename.end_with?(".gz"))
            gz = Zlib::GzipReader.new(StringIO.new(file_data))
            buffer = gz.read
            filename = filename[0..-4]
          else
            buffer = file_data
          end

          File.open("./files/" + filename, "wb") do |f|
            f.write(buffer)
          end
          puts "Wrote file #{filename}"
          success = true
        rescue
          attempt += 1
        end
      end
<<<<<<< HEAD:fetcher/main.rb
=======


      File.open("./files/" + filename, "wb") do |f|
        f.write(buffer)
      end
      puts "Wrote file #{filename}"
>>>>>>> 9effded73d4e43a9642ce7411ff34897b71fd48f:main.rb
    })
  end

  parallelize(download_tasks, 100)
end<commit_msg>Remove merge junk<commit_after>require "./cerberus.rb"
require "./thread_pool.rb"
require "zlib"

puts "Loading login page..."
login_details = get_login_details
puts "Got CSRF token: #{login_details[:csrf]}"
puts "Got cookie: #{login_details[:cookie]}"


puts "Logging in..."
cookie = login(login_details[:csrf], login_details[:cookie], "doralon", "")

if (!cookie)
  puts "Failed to log in :("
else
  puts "Logged in!"

  puts "Got updated cookie: #{cookie}"
  puts "Requesting directory list..."
  files = get_file_listing(cookie)

  download_tasks = []

  Dir.mkdir './files' unless File.exist? './files'
  files.each_with_index do |filename, idx|
    download_tasks.push(Proc.new {
      attempt = 0
      success = false
      while (!success) do
        begin
          puts "Downloading file #{filename} (#{idx}/#{files.count}) Attempt #{attempt}"
          file_data = download_file(cookie, filename)

          if (filename.end_with?(".gz"))
            gz = Zlib::GzipReader.new(StringIO.new(file_data))
            buffer = gz.read
            filename = filename[0..-4]
          else
            buffer = file_data
          end

          File.open("./files/" + filename, "wb") do |f|
            f.write(buffer)
          end
          puts "Wrote file #{filename}"
          success = true
        rescue
          attempt += 1
        end
      end

      File.open("./files/" + filename, "wb") do |f|
        f.write(buffer)
      end
      puts "Wrote file #{filename}"
    })
  end

  parallelize(download_tasks, 100)
end
{'fraction_non_alphanumeric': 0.1418216199180586, 'fraction_numerical': 0.011660888748818153, 'mean_word_length': 2.4055793991416308, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5726062', 'n_tokens_mistral': 1038, 'n_tokens_neox': 998, 'n_words': 251}