<commit_before>require 'pipeline/pipe'

module Pipeline
  class ZipSevenPipe < Pipe
    def initialize
      super('7Z', 1, 0)
      @config[:part_size_mb] = '4'
      @config[:zip_binary] = '/opt/local/bin/7za'
    end
    
    def check_before_work
      @path = @source.items[0]
      filename = File.basename(@path)
      archive_to_name = "#{filename}.7z"
      @archive_to_dir = "#{@path}.7z.d"
      @archive_to_path = File.join(@archive_to_dir, archive_to_name)
      
      unless File.exists? @path
        log "error: source file not found #{@path}"
        return :fail
      end
      
      make_empty_dir @archive_to_dir
      
      @zip_binary = @config[:zip_binary]
      unless File.exists? @zip_binary
        log "error: zip binary not found #{@zip_binary}"
        return :fail
      end
      
      @part_size_mb = @config[:part_size_mb].to_i
      if @part_size_mb <= 0
        log "error: wrong part size #{@part_size_mb}"
        return :fail
      end
      
      return :ok
    end
    
    def work
      bin = @config[:zip_binary]
      options = "a -t7z -v#{@part_size_mb}m -y -m0=lzma -mx=9 -mfb=64 -md=32m -ms=on -mhe=on"
      run = "#{bin} #{options} '#{@archive_to_path}' '#{@path}' 2>&1"
      
      log "options: #{options}"
      log "run: #{run}"
      
      output = %x[#{run}]
      
      if not $?.success?
        log "error: #{output}"
        return :fail
      end
      
      @parts = ZipSevenSort.sort(Dir["#{@archive_to_dir}/*"])
      log "volumes created #{@parts}"
      
      return :ok
    end
    
    def check_after_work
      @parts.each do |part|
        if not File.exists?(part)
          log "error: archive part not found #{part}"
          return :fail
        end
        
        @target.add part
      end
      
      return :ok
    end
    
    def make_empty_dir(path)
      log "make empty dir: #{path}"
      destroy_fs_entry path
      Dir.mkdir path
    end

    def destroy_fs_entry(path)
      if File.exist?(path)
        if File.directory?(path)
          Dir["#{path}/*"].each do |f| <commit_msg>7Z: fixed result directory listing<commit_after>require 'pipeline/pipe'

module Pipeline
  class ZipSevenPipe < Pipe
    def initialize
      super('7Z', 1, 0)
      @config[:part_size_mb] = '4'
      @config[:zip_binary] = '/opt/local/bin/7za'
    end
    
    def check_before_work
      @path = @source.items[0]
      filename = File.basename(@path)
      archive_to_name = "#{filename}.7z"
      @archive_to_dir = "#{@path}.7z.d"
      @archive_to_path = File.join(@archive_to_dir, archive_to_name)
      
      unless File.exists? @path
        log "error: source file not found #{@path}"
        return :fail
      end
      
      make_empty_dir @archive_to_dir
      
      @zip_binary = @config[:zip_binary]
      unless File.exists? @zip_binary
        log "error: zip binary not found #{@zip_binary}"
        return :fail
      end
      
      @part_size_mb = @config[:part_size_mb].to_i
      if @part_size_mb <= 0
        log "error: wrong part size #{@part_size_mb}"
        return :fail
      end
      
      return :ok
    end
    
    def work
      bin = @config[:zip_binary]
      options = "a -t7z -v#{@part_size_mb}m -y -m0=lzma -mx=9 -mfb=64 -md=32m -ms=on -mhe=on"
      run = "#{bin} #{options} '#{@archive_to_path}' '#{@path}' 2>&1"
      
      log "options: #{options}"
      log "run: #{run}"
      
      output = %x[#{run}]
      
      if not $?.success?
        log "error: #{output}"
        return :fail
      end
      
      parts = []
      Dir.new(@archive_to_dir).each { |f| parts << f }
      @parts = ZipSevenSort.sort(parts)
      
      log "volumes created #{@parts}"
      
      return :ok
    end
    
    def check_after_work
      if @parts.length == 0
        log "error: nothing done"
        return :fail
      end
      
      @parts.each do |part|
        if not File.exists?(part)
          log "error: archive part not found #{part}"
          return :fail
        end
        @target.add part
      end
      
      return :ok
    end
    
    def make_empty_dir(path)
      log "make empty dir: #{path}"
      destroy_fs_entry path
      Dir.mkdir path
    end

    def destroy_fs_entry(path)
      if File.exist?(path)
        if File.directory?(path)
          Dir["#{path}/*"].each do |f| 
{'fraction_non_alphanumeric': 0.12525951557093426, 'fraction_numerical': 0.008765859284890426, 'mean_word_length': 1.884896872920825, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6319912', 'n_tokens_mistral': 1485, 'n_tokens_neox': 1474, 'n_words': 357}