<commit_before># this class acts as a streaming body for rails
# initialize it with an array of the files you want to zip
# right now only carrierwave is supported with file storage or S3
module Zipline
  class ZipGenerator
    # takes an array of pairs [[uploader, filename], ... ]
    def initialize(files)
      @files = files
    end

    #this is supposed to be streamed!
    def to_s
      throw "stop!"
    end

    def each(&block)
      output = new_output(&block)
      OutputStream.open(output) do |zip|
        @files.each do |file, name|
	  file = file.file if file.respond_to? :file

          #normalize file
          if file.class.to_s == 'CarrierWave::Storage::Fog::File'
            file = file.send(:file)
          end
          if file.class.to_s == 'CarrierWave::SanitizedFile'
            path = file.send(:file)
            file = File.open(path)
          end
	  if file.class.to_s == 'Paperclip::Attachment'
            path = file.send(:path)
            file = File.open(path)
	  end
          throw "bad_file" unless %w{Fog::Storage::AWS::File File}.include? file.class.to_s

          name = uniquify_name(name)
          write_file(zip, file, name)
        end
      end
    end

    def new_output(&block)
      FakeStream.new(&block)
    end

    def write_file(zip, file, name)
      size = get_size(file)

      zip.put_next_entry name, size

      if file.is_a? File
        while buffer = file.read(2048)
          zip << buffer
        end
      else
        the_remote_url = file.url(Time.now + 1.minutes)
        c = Curl::Easy.new(the_remote_url) do |curl|
          curl.on_body do |data|
            zip << data
            data.bytesize
          end
        end
        c.perform
      end
    end

    def get_size(file)
      case file.class.to_s
      when 'File'
        file.size
      when 'Fog::Storage::AWS::FILE'
        file.content_length
      end
    end

    def uniquify_name(name)
      @used_names ||= Set.new


      if @used_names.include?(name)

        #remove suffix e.g. ".foo"
        parts = name.split '.'
        name, extension =
          if parts.length == 1
            #no suffix, e.g. README
            parts << ''
          else
            extension = parts.pop<commit_msg>support subclasses of IO and StringIO<commit_after># this class acts as a streaming body for rails
# initialize it with an array of the files you want to zip
# right now only carrierwave is supported with file storage or S3
module Zipline
  class ZipGenerator
    # takes an array of pairs [[uploader, filename], ... ]
    def initialize(files)
      @files = files
    end

    #this is supposed to be streamed!
    def to_s
      throw "stop!"
    end

    def each(&block)
      output = new_output(&block)
      OutputStream.open(output) do |zip|
        @files.each do |file, name|
          file = file.file if file.respond_to? :file

          #normalize file
          unless is_io?(file)
            case file.class.to_s
            when 'CarrierWave::Storage::Fog::File'
              file = file.send(:file)
            when 'CarrierWave::SanitizedFile'
              path = file.send(:file)
              file = File.open(path)
            when 'Paperclip::Attachment'
              path = file.send(:path)
              file = File.open(path)
            else
              raise(ArgumentError, 'Bad File/Stream')
            end
          end

          name = uniquify_name(name)
          write_file(zip, file, name)
        end
      end
    end

    def new_output(&block)
      FakeStream.new(&block)
    end

    def write_file(zip, file, name)
      size = get_size(file)

      zip.put_next_entry name, size

      if is_io?(file)
        while buffer = file.read(2048)
          zip << buffer
        end
      else
        the_remote_url = file.url(Time.now + 1.minutes)
        c = Curl::Easy.new(the_remote_url) do |curl|
          curl.on_body do |data|
            zip << data
            data.bytesize
          end
        end
        c.perform
      end
    end

    def get_size(file)
      if is_io?(file)
        file.size
      elsif file.class.to_s == 'Fog::Storage::AWS::FILE'
        file.content_length
      else
        throw 'cannot determine file size'
      end
    end

    def is_io?(file)
      file.is_a?(IO) || (defined?(StringIO) && file.is_a?(StringIO))
    end

    def uniquify_name(name)
      @used_names ||= Set.new


      if @used_names.include?(name)

        #remove suffix e.g. ".foo"
        parts = name.split '.'
        name, extension =
          if parts.length == 1
            #no suffix, e.g. README
            parts << ''
          else
            extension = parts.pop
{'fraction_non_alphanumeric': 0.09623163721524378, 'fraction_numerical': 0.0029806259314456036, 'mean_word_length': 1.7329842931937174, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6060047', 'n_tokens_mistral': 1463, 'n_tokens_neox': 1410, 'n_words': 397}