<commit_before>require "archive/tar/reader"

class Archive::Tar::StreamReader < Archive::Tar::Reader
  def initialize(stream, options = {})
    options = {
      block_size: 2 ** 19,
      reload_time: 32
    }.merge(options)
  
    if options[:compression] == :auto
      raise "Automatic compression is not available for streams"
    end
    
    tmp_file = File.new("/tmp/" + rand(500000).to_s(16) + ".tar", "w+b")
    Thread.new do
      i = 0
    
      puts "Copy to tmp file..."
      until stream.eof?
        read = stream.read(options[:block_size])
        tmp_file.write(read)
        self.build_index if i % options[:reload_time] == 0
        puts "Rebuild" if i % options[:reload_time] == 0
        i += 1
      end
      puts "Done!"
      
      self.build_index<commit_msg>Allow fd's for StreamReader.initialize as stream<commit_after>require "archive/tar/reader"

class Archive::Tar::StreamReader < Archive::Tar::Reader
  def initialize(stream, options = {})
    options = {
      block_size: 2 ** 19,
      reload_time: 32
    }.merge(options)
    
    stream = IO.new(stream) if io.is_a? Integer
  
    if options[:compression] == :auto
      raise "Automatic compression is not available for streams"
    end
    
    tmp_file = File.new("/tmp/" + rand(500000).to_s(16) + ".tar", "w+b")
    Thread.new do
      i = 0
    
      puts "Copy to tmp file..."
      until stream.eof?
        read = stream.read(options[:block_size])
        tmp_file.write(read)
        self.build_index if i % options[:reload_time] == 0
        puts "Rebuild" if i % options[:reload_time] == 0
        i += 1
      end
      puts "Done!"
      
      self.build_index
{'fraction_non_alphanumeric': 0.12845138055222088, 'fraction_numerical': 0.02040816326530612, 'mean_word_length': 2.3473895582329316, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6252832', 'n_tokens_mistral': 565, 'n_tokens_neox': 535, 'n_words': 135}