<commit_before>require 'observer'

# Namespace module for One on One Marketing.
module One

  # Class that can be used for mining data from lists of objects.
  class Pivoter
    include Observable

    # Pivots a list of Objects grouping them into an organized Hash.
    #
    # @example Pivot a list of numbers into 2 groups, those less than or equal to 5 and those greater than 5
    #   list = [1,2,3,4,5,6,7,8,9]
    #   result = pivot(list) {|num| num <=5 }
    #
    #   # the result will be a Hash with the following structure
    #   {
    #     true=>[1, 2, 3, 4, 5],
    #     false=>[6, 7, 8, 9] 
    #   }
    #
    # @param [Array<Object>] list The list to pivot
    # @param [optional, Hash] options Options to pivot with
    # @option options [Object] :identifier A name that uniquely identifies the pivot operation 
    # @yield [item] The block passed to pivot will be called for each item in the list 
    # @yieldparam [Object] item An item in the list
    # @yieldreturn [Object] The value returned from the pivot block will serve as the key in the pivot results
    # @return [Hash] The pivoted results
    def pivot(list, options={}, &block)
      pivoted = {}
      list.each do |item|
        value = yield(item)

        # notify observers that a pivot block was just called
        identifier = options[:identifier] || "#{item.hash}:#{block.hash}"
        changed
        notify_observers(identifier, item, value) 

        if value.is_a?(Array)
          if value.empty?
            pivoted[nil] ||= []
            pivoted[nil] << item 
          else
            value.each do |val|
              pivoted[val] ||= []
              pivoted[val] << item 
            end
          end
        else
          pivoted[value] ||= []
          pivoted[value] << item 
        end
      end

      pivoted
    end    

    # Runs multiple pivots against a list of Objects.
    #
    # @example Multi-pivot a list of numbers
    #   list = [1,2,3,4,5,6,7,8,9]
    #   pivots = []
    #
    #   pivots << lambda do |i|
    #     key = "less than or equal to 5" if i <= 5
    #     key ||= "greater than 5"
    #   end
    #
    #   pivots << lambda do |i|
    #     key = "greater than or equal to 3" if i >= 3
    #     key ||= "less than 3"
    #   end
    # 
    #   # note the last pivot is an options Hash
    #   pivots << {:delimiter => " & "}
    #    
    #   pivoter = One::Pivot.new
    #   result = pivoter.multi_pivot(list, *pivots) 
    #   
    #   # the result will be a Hash with the following structure
    #   {
    #     "less than or equal to 5 & greater than or equal to 3" => [3, 4, 5], 
    #     "less than or equal to 5 & less than 3" => [1, 2], 
    #     "greater than 5 & greater than or equal to 3" => [6, 7, 8, 9]
    #   }
    #
    # @param [Array<Object>] list The list to run the pivots against
    # @param [Array<Proc, One::Pivot>] pivots An argument list that accepts N number of pivots.<br />
    #   The last pivot in the list can be an options Hash for the multi_pivot operation.<br />
    #   Supported options are:<br />
    #   * :delimiter, The delimiter to use between pivot keys.  Defaults to '[PIVOT]']
    # @return [Hash] The pivoted results
    def multi_pivot(list, *pivots)
      options = pivots.pop if pivots.last.is_a?(Hash)
      options ||= {}
      delimiter = options[:delimiter] || "[PIVOT]"
      pivoted = nil
      pass = 0

      while pivots.length > 0
        p = pivots.shift
        
        # handle the case where the pivots are One::Pivot objects
        pivot_options = {}
        if p.is_a?(One::Pivot)
          pivot_options[:identifier] = p.identifier
          p = p.pivot_proc
        end

        if pass == 0
          pivoted = pivot(list, pivot_options, &p)
        else
          new_pivoted = {}
          pivoted.each do |old_key, old_list|
            tmp_pivoted = pivot(old_list, pivot_options, &p)
            tmp_pivoted.each do |key, list|
              new_key = "#{safe_key(old_key)}#{delimiter}#{safe_key(key)}"
              new_pivoted[new_key] = list
            end
          end
          pivoted = new_pivoted
        end

        pass += 1
      end

      pivoted
    end

    private

    def safe_key(key)
      key = "nil" if key.to_s.strip.empty?
      key.to_s
    end

  end
end
<commit_msg>Experimenting with forking the pivot process to speed certain pivots up. NOTE: this is still experimental and should probabl y be on its own branch but I'm tired and lazy right now.<commit_after>require 'observer'
require 'parallel'

#def do_in_child
#  read, write = IO.pipe
#
#  pid = fork do
#    read.close
#    result = yield
#    write.puts [Marshal.dump(result)].pack("m")
#    exit!
#  end
#
#  write.close
#  result = read.read
#  Process.wait2(pid)
#  Marshal.load(result.unpack("m")[0])
#  Marshal.load
#end
#Then you could run:
#
#do_in_child do
#  require "some_polluting_library"
#  SomePollutingLibrary.some_operation
#end



# Namespace module for One on One Marketing.
module One

  # Class that can be used for mining data from lists of objects.
  class Pivoter
    include Observable

    # Pivots a list of Objects grouping them into an organized Hash.
    #
    # @example Pivot a list of numbers into 2 groups, those less than or equal to 5 and those greater than 5
    #   list = [1,2,3,4,5,6,7,8,9]
    #   result = pivot(list) {|num| num <=5 }
    #
    #   # the result will be a Hash with the following structure
    #   {
    #     true=>[1, 2, 3, 4, 5],
    #     false=>[6, 7, 8, 9] 
    #   }
    #
    # @param [Array<Object>] list The list to pivot
    # @param [optional, Hash] options Options to pivot with
    # @option options [Object] :identifier A name that uniquely identifies the pivot operation 
    # @yield [item] The block passed to pivot will be called for each item in the list 
    # @yieldparam [Object] item An item in the list
    # @yieldreturn [Object] The value returned from the pivot block will serve as the key in the pivot results
    # @return [Hash] The pivoted results
    def pivot(list, options={}, &block)
      start = Time.now
      list = list.to_a
      processes = options[:processes] || 1
      if processes > 1
        chunks = list.each_slice(list.length/processes).map {|chunk| chunk}
        results = pivot_in_child(chunks, options, &block)
      else
        results = run_pivot(list, options={}, &block)
      end
      puts "#{Time.now - start} secs..."
      results
    end

    # Runs multiple pivots against a list of Objects.
    #
    # @example Multi-pivot a list of numbers
    #   list = [1,2,3,4,5,6,7,8,9]
    #   pivots = []
    #
    #   pivots << lambda do |i|
    #     key = "less than or equal to 5" if i <= 5
    #     key ||= "greater than 5"
    #   end
    #
    #   pivots << lambda do |i|
    #     key = "greater than or equal to 3" if i >= 3
    #     key ||= "less than 3"
    #   end
    # 
    #   # note the last pivot is an options Hash
    #   pivots << {:delimiter => " & "}
    #    
    #   pivoter = One::Pivot.new
    #   result = pivoter.multi_pivot(list, *pivots) 
    #   
    #   # the result will be a Hash with the following structure
    #   {
    #     "less than or equal to 5 & greater than or equal to 3" => [3, 4, 5], 
    #     "less than or equal to 5 & less than 3" => [1, 2], 
    #     "greater than 5 & greater than or equal to 3" => [6, 7, 8, 9]
    #   }
    #
    # @param [Array<Object>] list The list to run the pivots against
    # @param [Array<Proc, One::Pivot>] pivots An argument list that accepts N number of pivots.<br />
    #   The last pivot in the list can be an options Hash for the multi_pivot operation.<br />
    #   Supported options are:<br />
    #   * :delimiter, The delimiter to use between pivot keys.  Defaults to '[PIVOT]']
    # @return [Hash] The pivoted results
    def multi_pivot(list, *pivots)
      options = pivots.pop if pivots.last.is_a?(Hash)
      options ||= {}
      delimiter = options[:delimiter] || "[PIVOT]"
      pivoted = nil
      pass = 0

      while pivots.length > 0
        p = pivots.shift
        
        # handle the case where the pivots are One::Pivot objects
        pivot_options = {}
        if p.is_a?(One::Pivot)
          pivot_options[:identifier] = p.identifier
          p = p.pivot_proc
        end

        if pass == 0
          pivoted = pivot(list, pivot_options, &p)
        else
          new_pivoted = {}
          pivoted.each do |old_key, old_list|
            tmp_pivoted = pivot(old_list, pivot_options, &p)
            tmp_pivoted.each do |key, list|
              new_key = "#{safe_key(old_key)}#{delimiter}#{safe_key(key)}"
              new_pivoted[new_key] = list
            end
          end
          pivoted = new_pivoted
        end

        pass += 1
      end

      pivoted
    end

    private

    def safe_key(key)
      key = "nil" if key.to_s.strip.empty?
      key.to_s
    end

    def run_pivot(list, options={}, &block)
      pivoted = {}

      list.each do |item|
        value = yield(item)

        # notify observers that a pivot block was just called
        identifier = options[:identifier] || "#{item.hash}:#{block.hash}"
        changed
        notify_observers(identifier, item, value)

        if value.is_a?(Array)
          if value.empty?
            pivoted[nil] ||= []
            pivoted[nil] << item
          else
            value.each do |val|
              pivoted[val] ||= []
              pivoted[val] << item
            end
          end
        else
          pivoted[value] ||= []
          pivoted[value] << item
        end
      end

      pivoted
    end

    def pivot_in_child(lists, options, &block)
      reads = []
      writes = []
      pids = []

      lists.each_with_index do |list, index|
        reads[index], writes[index] = IO.pipe
        puts "forking now..."
        pids << fork do
          reads[index].close
          result = run_pivot(list, options, &block)
          writes[index].puts [Marshal.dump(result)].pack("m")
          exit!
        end
      end

      results = []
      (0..lists.length-1).each do |index|
        writes[index].close
        result = reads[index].read
        Process.wait2(pids[index])
        results << Marshal.load(result.unpack("m")[0])
      end

      pivoted = {}
      results.each do |chunked_pivoted|
        chunked_pivoted.each do |key, list|
          pivoted[key] ||= []
          pivoted[key].concat(list)
        end
      end

      pivoted
    end

  end
end

{'fraction_non_alphanumeric': 0.10865019011406844, 'fraction_numerical': 0.011406844106463879, 'mean_word_length': 1.9578296317121169, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6770225', 'n_tokens_mistral': 3564, 'n_tokens_neox': 3359, 'n_words': 1090}