<commit_before># encoding: utf-8

require "net/http"
require "openssl"

module Rbitter
  class DLThread
    MAX_THREADS = 20

    def initialize(dlfolder, large_flag)
      @dest = dlfolder
      if not File.directory?(dlfolder)
        warn "[dlthread] Given download location is not available for downloading."
        warn "[dlthread] Fallback to current directory."
        @dest = "./"
      end

      if large_flag.nil?
        @large_image = false
      else
        @large_image = large_flag
      end

      @pool = Array.new
    end

    def <<(url_array)
      if @pool.length >= MAX_THREADS
        job_cleanup
      else
        puts "[dlthread] Finishing thread [remains: #{@pool.length}]"
      end

      download_task = Thread.new do
        url_array.each { |url|
          uri = URI.parse(@large_image ? url + ":large" : url) 

          Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme.downcase == 'https') { |h|
            req = Net::HTTP::Get.new uri.request_uri
            h.request(req) { |res|
              case res
              when Net::HTTPOK
                fname = File.basename(url)

                puts "[fetch] remote: #{uri.path} => local: #{fname}"
                open(File.join(@dest, fname), "wb") { |file|
                  res.read_body { |chunk| file.write(chunk) }
                }
              end
            }
          }
        }
      end

      @pool.push download_task
    end<commit_msg>Fix incorrect message in dlthread<commit_after># encoding: utf-8

require "net/http"
require "openssl"

module Rbitter
  class DLThread
    MAX_THREADS = 20

    def initialize(dlfolder, large_flag)
      @dest = dlfolder
      if not File.directory?(dlfolder)
        warn "[dlthread] Given download location is not available for downloading."
        warn "[dlthread] Fallback to current directory."
        @dest = "./"
      end

      if large_flag.nil?
        @large_image = false
      else
        @large_image = large_flag
      end

      @pool = Array.new
    end

    def <<(url_array)
      if @pool.length >= MAX_THREADS
        job_cleanup
      else
        puts "[dlthread] Stacked threads: #{@pool.length}"
      end

      download_task = Thread.new do
        url_array.each { |url|
          uri = URI.parse(@large_image ? url + ":large" : url) 

          Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme.downcase == 'https') { |h|
            req = Net::HTTP::Get.new uri.request_uri
            h.request(req) { |res|
              case res
              when Net::HTTPOK
                fname = File.basename(url)

                puts "[fetch] remote: #{uri.path} => local: #{fname}"
                open(File.join(@dest, fname), "wb") { |file|
                  res.read_body { |chunk| file.write(chunk) }
                }
              end
            }
          }
        }
      end

      @pool.push download_task
    end
{'fraction_non_alphanumeric': 0.11229031153714482, 'fraction_numerical': 0.0020540910647038686, 'mean_word_length': 1.7105751391465678, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2063623', 'n_tokens_mistral': 896, 'n_tokens_neox': 865, 'n_words': 220}