<commit_before>require 'progress-bar'

module TSV
  
  attr_accessor :monitor

  class Traverser
    attr_accessor :new_key_field, :new_fields, :new_key_field_name, :new_field_names, :type, :uniq

    def process_null(key, values)
      [[key], values]
    end

    def process_subset_list(key, values)
      [key, @new_fields.collect{|field| field == :key ?  key : values[field] }]
    end

    def process_subset_all_but_list(key, values)
      new = values.dup
      new.delete_at(0 - @new_fields)
      [key, new]
    end

    def process_reorder_single(key, values)
      new_key = @new_key_field == :key ? key : values
      new_value = @new_fields.collect{|field| field == :key ?  key : values }.first
      return [new_key, new_value]
      [ [values[@new_key_field]], 
        @new_fields.collect{|field| field == :key ?  key : values[field] }]
    end

    def process_reorder_list(key, values)
      [ [values[@new_key_field]], 
        @new_fields.collect{|field| field == :key ?  key : values[field] }]
    end

    def process_reorder_double_uniq(key, values)
      [ values[@new_key_field].uniq, 
        @new_fields.collect{|field| field == :key ?  
          [key] : values[field] }
      ]
    end

    def process_subset_double(key, values)
      [[key], @new_fields.collect{|field| field == :key ?  [key] : values[field] }]
    end

    def process_subset_all_but_double(key, values)
      new = values.dup
      new.delete_at(0 - @new_fields)
      [[key], new]
    end

    def process_reorder_double(key, values)
      [ values[@new_key_field], 
        @new_fields.collect{|field| field == :key ?  
          [key] : values[field] }
      ]
    end

    def process_reorder_flat(key, values)
      [ values,
        @new_fields.collect{|field| field == :key ?  
          [key] : values[field] }.flatten
      ]
    end

    def initialize(key_field, fields, new_key_field, new_fields, type, uniq)
      @new_key_field = TSV.identify_field(key_field, fields, new_key_field)

      raise "Key field #{ new_key_field } not found" if @new_key_field.nil?
      @new_fields = case new_fields
                    when nil
                      case 
                      when @new_key_field == :key
                        :all
                      when fields.nil?
                        - @new_key_field
                      else 
                        new = (0..fields.length - 1).to_a
                        new.delete_at(@new_key_field)
                        new.unshift :key
                        new
                      end
                    when Array
                      new_fields.collect do |field|
                        TSV.identify_field(key_field, fields, field)
                      end
                    when String, Symbol
                      [TSV.identify_field(key_field, fields, new_fields)]
                    else
                      raise "Unknown format for new_fields (should be nil, Array or String): #{new_fields.inspect}"
                    end

      @new_key_field_name = case 
                            when @new_key_field == :key
                              key_field
                            else
                              fields[@new_key_field] if Array === fields
                            end

      if Array === fields
        @new_field_names = case
                           when fields.nil?
                             nil
                           when Array === @new_fields
                             @new_field_names = @new_fields.collect do |field|
                               case 
                               when field == :key
                                 key_field
                               else
                                 fields[field]
                               end
                             end
                           when @new_fields == :all
                             fields
                           when (Fixnum === @new_fields and @new_fields <= 0)
                             new = fields.dup
                             new.delete_at(- @new_fields)
                             new.unshift key_field
                             new
                           end
      end

      case
      when (@new_key_field == :key and (@new_fields == :all or fields.nil? or @new_fields == (0..fields.length - 1).to_a))
        self.instance_eval do alias process process_null end
      when @new_key_field == :key 
        if type == :double
          if Fixnum === @new_fields and @new_fields <= 0
            self.instance_eval do alias process process_subset_all_but_double end
          else
            self.instance_eval do alias process process_subset_double end
          end
        else
          if Fixnum === @new_fields and @new_fields <= 0
            self.instance_eval do alias process process_subset_all_but_list end
          else
            self.instance_eval do alias process process_subset_list end
          end
        end
      else
        case type 
        when :double
          if uniq
            self.instance_eval do alias process process_reorder_double_uniq end
          else
            self.instance_eval do alias process process_reorder_double end
          end
        when :flat
          self.instance_eval do alias process process_reorder_flat end
        when :single
          self.instance_eval do alias process process_reorder_single end
        else
          self.instance_eval do alias process process_reorder_list end
        end
      end
    end

  end

  #{{{ Methods

  def through(new_key_field = nil, new_fields = nil, uniq = false, zipped = false)

    traverser = Traverser.new key_field, fields, new_key_field, new_fields, type, uniq

    if @monitor
      desc = "Iterating TSV"
      step = 100
      if Hash === @monitor
        desc = @monitor[:desc] if @monitor.include? :desc 
        step = @monitor[:step] if @monitor.include? :step 
      end
      progress_monitor = Log::ProgressBar.new_bar(size, :desc => desc)
    else
      progress_monitor = nil
    end

    each do |key, value|
      progress_monitor.tick if progress_monitor

      keys, value = traverser.process(key, value)

      next if keys.nil?
      
      keys = [keys].compact unless Array === keys

      # Annotated with Entity and NamedArray
      if not @unnamed and not traverser.new_field_names.nil? 

        case type
        when :double, :list
          #Log.warn "Value frozen: #{ value }" if value.frozen?

          value.nil? ?
            nil :
            NamedArray.setup(value, traverser.new_field_names, key, entity_options, entity_templates)

        when :flat, :single
          prepare_entity(value, traverser.new_field_names.first, entity_options)
        end
      end

<commit_msg>Reuse progress bar when available during traverse<commit_after>require 'progress-bar'

module TSV
  
  attr_accessor :monitor

  class Traverser
    attr_accessor :new_key_field, :new_fields, :new_key_field_name, :new_field_names, :type, :uniq

    def process_null(key, values)
      [[key], values]
    end

    def process_subset_list(key, values)
      [key, @new_fields.collect{|field| field == :key ?  key : values[field] }]
    end

    def process_subset_all_but_list(key, values)
      new = values.dup
      new.delete_at(0 - @new_fields)
      [key, new]
    end

    def process_reorder_single(key, values)
      new_key = @new_key_field == :key ? key : values
      new_value = @new_fields.collect{|field| field == :key ?  key : values }.first
      return [new_key, new_value]
      [ [values[@new_key_field]], 
        @new_fields.collect{|field| field == :key ?  key : values[field] }]
    end

    def process_reorder_list(key, values)
      [ [values[@new_key_field]], 
        @new_fields.collect{|field| field == :key ?  key : values[field] }]
    end

    def process_reorder_double_uniq(key, values)
      [ values[@new_key_field].uniq, 
        @new_fields.collect{|field| field == :key ?  
          [key] : values[field] }
      ]
    end

    def process_subset_double(key, values)
      [[key], @new_fields.collect{|field| field == :key ?  [key] : values[field] }]
    end

    def process_subset_all_but_double(key, values)
      new = values.dup
      new.delete_at(0 - @new_fields)
      [[key], new]
    end

    def process_reorder_double(key, values)
      [ values[@new_key_field], 
        @new_fields.collect{|field| field == :key ?  
          [key] : values[field] }
      ]
    end

    def process_reorder_flat(key, values)
      [ values,
        @new_fields.collect{|field| field == :key ?  
          [key] : values[field] }.flatten
      ]
    end

    def initialize(key_field, fields, new_key_field, new_fields, type, uniq)
      @new_key_field = TSV.identify_field(key_field, fields, new_key_field)

      raise "Key field #{ new_key_field } not found" if @new_key_field.nil?
      @new_fields = case new_fields
                    when nil
                      case 
                      when @new_key_field == :key
                        :all
                      when fields.nil?
                        - @new_key_field
                      else 
                        new = (0..fields.length - 1).to_a
                        new.delete_at(@new_key_field)
                        new.unshift :key
                        new
                      end
                    when Array
                      new_fields.collect do |field|
                        TSV.identify_field(key_field, fields, field)
                      end
                    when String, Symbol
                      [TSV.identify_field(key_field, fields, new_fields)]
                    else
                      raise "Unknown format for new_fields (should be nil, Array or String): #{new_fields.inspect}"
                    end

      @new_key_field_name = case 
                            when @new_key_field == :key
                              key_field
                            else
                              fields[@new_key_field] if Array === fields
                            end

      if Array === fields
        @new_field_names = case
                           when fields.nil?
                             nil
                           when Array === @new_fields
                             @new_field_names = @new_fields.collect do |field|
                               case 
                               when field == :key
                                 key_field
                               else
                                 fields[field]
                               end
                             end
                           when @new_fields == :all
                             fields
                           when (Fixnum === @new_fields and @new_fields <= 0)
                             new = fields.dup
                             new.delete_at(- @new_fields)
                             new.unshift key_field
                             new
                           end
      end

      case
      when (@new_key_field == :key and (@new_fields == :all or fields.nil? or @new_fields == (0..fields.length - 1).to_a))
        self.instance_eval do alias process process_null end
      when @new_key_field == :key 
        if type == :double
          if Fixnum === @new_fields and @new_fields <= 0
            self.instance_eval do alias process process_subset_all_but_double end
          else
            self.instance_eval do alias process process_subset_double end
          end
        else
          if Fixnum === @new_fields and @new_fields <= 0
            self.instance_eval do alias process process_subset_all_but_list end
          else
            self.instance_eval do alias process process_subset_list end
          end
        end
      else
        case type 
        when :double
          if uniq
            self.instance_eval do alias process process_reorder_double_uniq end
          else
            self.instance_eval do alias process process_reorder_double end
          end
        when :flat
          self.instance_eval do alias process process_reorder_flat end
        when :single
          self.instance_eval do alias process process_reorder_single end
        else
          self.instance_eval do alias process process_reorder_list end
        end
      end
    end

  end

  #{{{ Methods

  def through(new_key_field = nil, new_fields = nil, uniq = false, zipped = false)

    traverser = Traverser.new key_field, fields, new_key_field, new_fields, type, uniq

    if @monitor
      if Log::ProgressBar === @monitor
        @monitor.max = size
        progress_monitor = @monitor
      else
        desc = "Iterating TSV"
        step = 100
        if Hash === @monitor
          desc = @monitor[:desc] if @monitor.include? :desc 
          step = @monitor[:step] if @monitor.include? :step 
        end
        progress_monitor = Log::ProgressBar.new_bar(size, :desc => desc)
      end
    else
      progress_monitor = nil
    end

    each do |key, value|
      progress_monitor.tick if progress_monitor

      keys, value = traverser.process(key, value)

      next if keys.nil?
      
      keys = [keys].compact unless Array === keys

      # Annotated with Entity and NamedArray
      if not @unnamed and not traverser.new_field_names.nil? 

        case type
        when :double, :list
          #Log.warn "Value frozen: #{ value }" if value.frozen?

          value.nil? ?
            nil :
            NamedArray.setup(value, traverser.new_field_names, key, entity_options, entity_templates)

        when :flat, :single
          prepare_entity(value, traverser.new_field_names.first, entity_options)
        end
      end


{'fraction_non_alphanumeric': 0.07929931676115715, 'fraction_numerical': 0.0017444395987788923, 'mean_word_length': 1.507106413994169, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4090713', 'n_tokens_mistral': 3883, 'n_tokens_neox': 3708, 'n_words': 955}