<commit_before># Copyright 2014 Cognitect. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Transducers are composable algorithmic transformations. See
# http://clojure.org/transducers before reading on.
#
# ## Terminology
#
# We need to expand the terminology a bit in order to map the concepts
# described on http://clojure.org/transducers to an OO language like
# Ruby.
#
# A _reducer_ is an object with a `step` method that takes a result
# (so far) and an input and returns a new result. This is similar to
# the blocks we pass to Ruby's `reduce` (a.k.a `inject`), and serves a
# similar role in _transducing process_.
#
# A _handler_ is an object with a `process` method that a reducer uses
# to process input. In a `map` operation, this would transform the
# input, and in a `filter` operation it would act as a predicate.
#
# A _transducer_ is an object that transforms a reducer by adding
# additional processing for each element in a collection of inputs.
#
# A _transducing process_ is invoked by calling
# `Transducers.transduce` with a transducer, a reducer, an optional
# initial value, and an input collection.
#
# Because Ruby doesn't come with free-floating handlers (e.g. Clojure's
# `inc` function) or reducing functions (e.g. Clojure's `conj`), we have
# to build these things ourselves.
#
# ## Examples
#
# ```ruby
# # handler
# inc = Class.new do
#         def process(input) input += 1 end
#       end.new
#
# # reducer
# appender = Class.new do
#              def step(result, input) result << input end
#            end.new
#
# # transducing process
# Transducers.transduce(Transducers.map(inc), appender, [], 0..9)
# #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# ```
#
# You can pass a `Symbol` or a `Block` to transducer constructors
# (`Transducers.map` in this example), so the above can be achieved
# more easily e.g.
#
# ```
# Transducers.transduce(Transducers.map(:succ),   appender, [], 0..9)
# #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# Transducers.transduce(Transducers.map {|n|n+1}, appender, [], 0..9)
# #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# ```
#
# You can omit the initial value if the reducer (`appender` in this
# example) provides one:
#
# ```
# def appender.init() [] end
# Transducers.transduce(Transducers.map {|n|n+1}, appender, 0..9)
# #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# ```
#
# You can also just pass a `Symbol` and an initial value instead of a
# reducer object, and the `transduce` method will build one for you.
#
# ```
# Transducers.transduce(Transducers.map {|n|n+1}, :<<, [], 0..9)
# #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# ```
#
# ## Composition
#
# Imagine that you want to take a range of numbers, select all the even
# ones, double them, and then take the first 5. Here's one way to
# do that in Ruby:
#
# ```ruby
# (1..100).
#   select {|n| n.even?}.
#   map    {|n| n * 2}.
#   take(5)
# #=> [4, 8, 12, 16, 20]
# ```
#
# Here's the same process with transducers:
#
# ```ruby
# t = Transducers.compose(
#       Transducers.filter(:even?),
#       Transducers.map {|n| n * 2},
#       Transducers.take(5))
# Transducers.transduce(t, :<<, [], 1..100)
# #=> [4, 8, 12, 16, 20]
# ```
#
# Now that we've defined the transducer as a series of
# transformations, we can apply it to different contexts, e.g.
#
# ```ruby
# Transducers.transduce(t, :+, 0, 1..100)
# #=> 60
# Transducers.transduce(t, :*, 1, 1..100)
# #=> 122880
# ```
module Transducers
  class Reducer
    def initialize(init, sym=nil, &block)
      raise ArgumentError.new("No init provided") if init == :no_init_provided
      @init = init
      if sym
        @sym = sym
        (class << self; self; end).class_eval do
          def step(result, input)
            result.send(@sym, input)
          end
        end
      else
        @block = block
        (class << self; self; end).class_eval do
          def step(result, input)
            @block.call(result, input)
          end
        end
      end
    end

    def init()           @init  end
    def complete(result) result end
    def step(result, input)
      # placeholder for docs - overwritten in initalize
    end
  end

  class Reduced
    attr_reader :val

    def initialize(val)
      @val = val
    end
  end

  class PreservingReduced
    def apply(reducer)
      @reducer = reducer
    end

    def step(result, input)
      ret = @reducer.step(result, input)
      Reduced === ret ? Reduced.new(ret) : ret
    end
  end

  class WrappingReducer
    class MethodHandler
      def initialize(method)
        @method = method
      end

      def process(input)
        input.send @method
      end
    end

    def initialize(reducer, handler=nil, &block)
      @reducer = reducer
      @handler = if block
                   block.singleton_class.class_eval do
                     alias process call
                   end
                   block
                 elsif Symbol === handler
                   MethodHandler.new(handler)
                 else
                   handler
                 end
    end

    def init()
      @reducer.init
    end

    def complete(result)
      @reducer.complete(result)
    end
  end

  # @api private
  class BaseTransducer
    class << self
      attr_reader :reducer_class

      def define_reducer_class(&block)
        @reducer_class = Class.new(WrappingReducer)
        @reducer_class.class_eval(&block)
      end
    end

    def initialize(handler, &block)
      @handler = handler
      @block = block
    end

    def reducer_class
      self.class.reducer_class
    end
  end

  class << self
    # @overload transduce(transducer, reducer, coll)
    # @overload transduce(transducer, reducer, init, coll)
    # @param [Transducer] transducer
    # @param [Reducer, Symbol, Bock] reducer
    def transduce(transducer, reducer, init=:no_init_provided, coll)
      reducer = Reducer.new(init, reducer) unless reducer.respond_to?(:step)
      reducer = transducer.apply(reducer)
      result = init == :no_init_provided ? reducer.init : init
      m = case coll
          when Enumerable then :each
          when String     then :each_char
          end
      coll.send(m) do |input|
        return result.val if Transducers::Reduced === result
        result = reducer.step(result, input)
      end
      reducer.complete(result)
    end

    def self.define_transducer_class(name, &block)
      t = Class.new(BaseTransducer)
      t.class_eval(&block)
      unless t.instance_methods.include? :apply
        t.class_eval do
          define_method :apply do |reducer|
            reducer_class.new(reducer, @handler, &@block)
          end
        end
      end

      Transducers.send(:define_method, name) do |handler=nil, &b|
        t.new(handler, &b)
      end

      Transducers.send(:module_function, name)
    end

    # @macro [new] common_transducer
    #   @return [Transducer]
    #   @method $1(handler=nil, &block)
    #   @param [Object, Symbol] handler
    #     Given an object that responds to +process+, uses it as the
    #     handler.  Given a +Symbol+, builds a handler whose +process+
    #     method will send +Symbol+ to its argument.
    #   @param [Block] block <i>(optional)</i>
    #     Given a +Block+, builds a handler whose +process+ method will
    #     call the block with its argument(s).
    # Returns a transducer that adds a map transformation to the
    # reducer stack.
    define_transducer_class :map do
      define_reducer_class do
        # Can I doc this?
        def step(result, input)
          @reducer.step(result, @handler.process(input))
        end
      end
    end

    # @macro common_transducer
    define_transducer_class :filter do
      define_reducer_class do
        def step(result, input)
          @handler.process(input) ? @reducer.step(result, input) : result
        end
      end
    end

    # @macro common_transducer
    define_transducer_class :remove do
      define_reducer_class do
        def step(result, input)
          @handler.process(input) ? result : @reducer.step(result, input)
        end
      end
    end

    # @method take(n)
    # @return [Transducer]
    define_transducer_class :take do
      define_reducer_class do
        def initialize(reducer, n)
          super(reducer)
          @n = n
        end

        def step(result, input)
          @n -= 1
          ret = @reducer.step(result, input)
          @n > 0 ? ret : Reduced.new(ret)
        end
      end

      def initialize(n)
        @n = n
      end

      def apply(reducer)
        reducer_class.new(reducer, @n)
      end
    end

    # @macro common_transducer
    define_transducer_class :take_while do
      define_reducer_class do
        def step(result, input)
          @handler.process(input) ? @reducer.step(result, input) : Reduced.new(result)
        end
      end
    end

    # @method take_nth(n)
    # @return [Transducer]
    define_transducer_class :take_nth do
      define_reducer_class do
        def initialize(reducer, n)
          super(reducer)
          @n = n
          @count = 0
        end

        def step(result, input)
          @count += 1
          if @count % @n == 0
            @reducer.step(result, input)
          else
            result
          end
        end
      end

      def initialize(n)
        @n = n
      end

      def apply(reducer)
        reducer_class.new(reducer, @n)
      end
    end

    # @method replace(source_map)
    # @return [Transducer]
    define_transducer_class :replace do
      define_reducer_class do
        def initialize(reducer, smap)
          super(reducer)
          @smap = smap
        end

        def step(result, input)
          if @smap.has_key?(input)
            @reducer.step(result, @smap[input])
          else
            @reducer.step(result, input)
          end
        end
      end

      def initialize(smap)
        @smap = case smap
                when Hash
                  smap
                else
                  (0...smap.size).zip(smap).to_h
                end
      end

      def apply(reducer)
        reducer_class.new(reducer, @smap)
      end
    end

    # @macro common_transducer
    define_transducer_class :keep do
      define_reducer_class do
        def step(result, input)
          x = @handler.process(input)
          x.nil? ? result : @reducer.step(result, x)
        end
      end
    end

    # @macro common_transducer
    # @note the handler for this method requires two arguments: the
    #   index and the input.
    define_transducer_class :keep_indexed do
      define_reducer_class do
        def initialize(*)
          super
          @index = -1
        end

        def step(result, input)
          @index += 1
          x = @handler.process(@index, input)
          x.nil? ? result : @reducer.step(result, x)
        end
      end
    end

    # @method drop(n)
    # @return [Transducer]
    define_transducer_class :drop do
      define_reducer_class do
        def initialize(reducer, n)
          super(reducer)
          @n = n
        end

        def step(result, input)
          @n -= 1
          @n <= -1 ? @reducer.step(result, input) : result
        end
      end

      def initialize(n)
        @n = n
      end

      def apply(reducer)
        reducer_class.new(reducer, @n)
      end
    end

    # @macro common_transducer
    define_transducer_class :drop_while do
      define_reducer_class do
        def initalize(*)
          super
          @done_dropping = false
        end

        def step(result, input)
          @done_dropping ||= !@handler.process(input)
          @done_dropping ? @reducer.step(result, input) : result
        end
      end
    end

    # @method dedupe
    # @return [Transducer]
    define_transducer_class :dedupe do
      define_reducer_class do
        def initialize(*)
          super
          @prior = :no_value_provided_for_transducer
        end

        def step(result, input)
          ret = input == @prior ? result : @reducer.step(result, input)
          @prior = input
          ret
        end
      end
    end

    # @method partition_by
    # @return [Transducer]
    define_transducer_class :partition_by do
      define_reducer_class do
        def initialize(*)
          super
          @a = []
          @prev_val = :no_value_provided_for_transducer
        end

        def complete(result)
          result = if @a.empty?
                     result
                   else
                     a = @a.dup
                     @a.clear
                     @reducer.step(result, a)
                   end
          @reducer.complete(result)
        end

        def step(result, input)
          prev_val = @prev_val
          val = @handler.process(input)
          @prev_val = val
          if val == prev_val || prev_val == :no_value_provided_for_transducer
            @a << input
            result
          else
            a = @a.dup
            @a.clear
            ret = @reducer.step(result, a)
            @a << input unless (Reduced === ret)
            ret
          end
        end
      end
    end

    # @method partition_all
    # @return [Transducer]
    define_transducer_class :partition_all do
      define_reducer_class do
        def initialize(reducer, n)
          super(reducer)
          @n = n
          @a = []
        end

        def step(result, input)
          @a << input
          if @a.size == @n
            a = @a.dup
            @a.clear
            @reducer.step(result, a)
          else
            result
          end
        end

        def complete(result)
          if @a.empty?
            result
          else
            a = @a.dup
            @a.clear
            @reducer.step(result, a)
          end
        end
      end

      def initialize(n)
        @n = n
      end

      def apply(reducer)
        reducer_class.new(reducer, @n)
      end
    end

    class RandomSampleHandler
      def initialize(prob)
        @prob = prob
      end

      def process(_)
        @prob > Random.rand
      end
    end

    # @return [Transducer]
    def random_sample(prob)
      filter RandomSampleHandler.new(prob)
    end

    # @method cat
    # @return [Transducer]
    define_transducer_class :cat do
      define_reducer_class do
        def step(result, input)
          Transducers.transduce(PreservingReduced.new, @reducer, result, input)
        end
      end
    end

    # @api private
    class ComposedTransducer
      def initialize(*transducers)
        @transducers = transducers
      end

      def apply(reducer)
        @transducers.reverse.reduce(reducer) {|r,t| t.apply(r)}
      end
    end

    # @return [Transducer]
    def compose(*transducers)
      ComposedTransducer.new(*transducers)
    end

    # @return [Transducer]
    def mapcat(handler=nil, &block)
      compose(map(handler, &block), cat)
    end
  end
end
<commit_msg>minor optimization: send vs class_eval on singleton class<commit_after># Copyright 2014 Cognitect. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Transducers are composable algorithmic transformations. See
# http://clojure.org/transducers before reading on.
#
# ## Terminology
#
# We need to expand the terminology a bit in order to map the concepts
# described on http://clojure.org/transducers to an OO language like
# Ruby.
#
# A _reducer_ is an object with a `step` method that takes a result
# (so far) and an input and returns a new result. This is similar to
# the blocks we pass to Ruby's `reduce` (a.k.a `inject`), and serves a
# similar role in _transducing process_.
#
# A _handler_ is an object with a `process` method that a reducer uses
# to process input. In a `map` operation, this would transform the
# input, and in a `filter` operation it would act as a predicate.
#
# A _transducer_ is an object that transforms a reducer by adding
# additional processing for each element in a collection of inputs.
#
# A _transducing process_ is invoked by calling
# `Transducers.transduce` with a transducer, a reducer, an optional
# initial value, and an input collection.
#
# Because Ruby doesn't come with free-floating handlers (e.g. Clojure's
# `inc` function) or reducing functions (e.g. Clojure's `conj`), we have
# to build these things ourselves.
#
# ## Examples
#
# ```ruby
# # handler
# inc = Class.new do
#         def process(input) input += 1 end
#       end.new
#
# # reducer
# appender = Class.new do
#              def step(result, input) result << input end
#            end.new
#
# # transducing process
# Transducers.transduce(Transducers.map(inc), appender, [], 0..9)
# #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# ```
#
# You can pass a `Symbol` or a `Block` to transducer constructors
# (`Transducers.map` in this example), so the above can be achieved
# more easily e.g.
#
# ```
# Transducers.transduce(Transducers.map(:succ),   appender, [], 0..9)
# #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# Transducers.transduce(Transducers.map {|n|n+1}, appender, [], 0..9)
# #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# ```
#
# You can omit the initial value if the reducer (`appender` in this
# example) provides one:
#
# ```
# def appender.init() [] end
# Transducers.transduce(Transducers.map {|n|n+1}, appender, 0..9)
# #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# ```
#
# You can also just pass a `Symbol` and an initial value instead of a
# reducer object, and the `transduce` method will build one for you.
#
# ```
# Transducers.transduce(Transducers.map {|n|n+1}, :<<, [], 0..9)
# #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# ```
#
# ## Composition
#
# Imagine that you want to take a range of numbers, select all the even
# ones, double them, and then take the first 5. Here's one way to
# do that in Ruby:
#
# ```ruby
# (1..100).
#   select {|n| n.even?}.
#   map    {|n| n * 2}.
#   take(5)
# #=> [4, 8, 12, 16, 20]
# ```
#
# Here's the same process with transducers:
#
# ```ruby
# t = Transducers.compose(
#       Transducers.filter(:even?),
#       Transducers.map {|n| n * 2},
#       Transducers.take(5))
# Transducers.transduce(t, :<<, [], 1..100)
# #=> [4, 8, 12, 16, 20]
# ```
#
# Now that we've defined the transducer as a series of
# transformations, we can apply it to different contexts, e.g.
#
# ```ruby
# Transducers.transduce(t, :+, 0, 1..100)
# #=> 60
# Transducers.transduce(t, :*, 1, 1..100)
# #=> 122880
# ```
module Transducers
  class Reducer
    def initialize(init, sym=nil, &block)
      raise ArgumentError.new("No init provided") if init == :no_init_provided
      @init = init
      if sym
        @sym = sym
        (class << self; self; end).class_eval do
          def step(result, input)
            result.send(@sym, input)
          end
        end
      else
        @block = block
        (class << self; self; end).class_eval do
          def step(result, input)
            @block.call(result, input)
          end
        end
      end
    end

    def init()           @init  end
    def complete(result) result end
    def step(result, input)
      # placeholder for docs - overwritten in initalize
    end
  end

  class Reduced
    attr_reader :val

    def initialize(val)
      @val = val
    end
  end

  class PreservingReduced
    def apply(reducer)
      @reducer = reducer
    end

    def step(result, input)
      ret = @reducer.step(result, input)
      Reduced === ret ? Reduced.new(ret) : ret
    end
  end

  class WrappingReducer
    class MethodHandler
      def initialize(method)
        @method = method
      end

      def process(input)
        input.send @method
      end
    end

    def initialize(reducer, handler=nil, &block)
      @reducer = reducer
      @handler = if block
                   block.singleton_class.send(:alias_method, :process, :call)
                   block
                 elsif Symbol === handler
                   MethodHandler.new(handler)
                 else
                   handler
                 end
    end

    def init()
      @reducer.init
    end

    def complete(result)
      @reducer.complete(result)
    end
  end

  # @api private
  class BaseTransducer
    class << self
      attr_reader :reducer_class

      def define_reducer_class(&block)
        @reducer_class = Class.new(WrappingReducer)
        @reducer_class.class_eval(&block)
      end
    end

    def initialize(handler, &block)
      @handler = handler
      @block = block
    end

    def reducer_class
      self.class.reducer_class
    end
  end

  class << self
    # @overload transduce(transducer, reducer, coll)
    # @overload transduce(transducer, reducer, init, coll)
    # @param [Transducer] transducer
    # @param [Reducer, Symbol, Bock] reducer
    def transduce(transducer, reducer, init=:no_init_provided, coll)
      reducer = Reducer.new(init, reducer) unless reducer.respond_to?(:step)
      reducer = transducer.apply(reducer)
      result = init == :no_init_provided ? reducer.init : init
      m = case coll
          when Enumerable then :each
          when String     then :each_char
          end
      coll.send(m) do |input|
        return result.val if Transducers::Reduced === result
        result = reducer.step(result, input)
      end
      reducer.complete(result)
    end

    def self.define_transducer_class(name, &block)
      t = Class.new(BaseTransducer)
      t.class_eval(&block)
      unless t.instance_methods.include? :apply
        t.class_eval do
          define_method :apply do |reducer|
            reducer_class.new(reducer, @handler, &@block)
          end
        end
      end

      Transducers.send(:define_method, name) do |handler=nil, &b|
        t.new(handler, &b)
      end

      Transducers.send(:module_function, name)
    end

    # @macro [new] common_transducer
    #   @return [Transducer]
    #   @method $1(handler=nil, &block)
    #   @param [Object, Symbol] handler
    #     Given an object that responds to +process+, uses it as the
    #     handler.  Given a +Symbol+, builds a handler whose +process+
    #     method will send +Symbol+ to its argument.
    #   @param [Block] block <i>(optional)</i>
    #     Given a +Block+, builds a handler whose +process+ method will
    #     call the block with its argument(s).
    # Returns a transducer that adds a map transformation to the
    # reducer stack.
    define_transducer_class :map do
      define_reducer_class do
        # Can I doc this?
        def step(result, input)
          @reducer.step(result, @handler.process(input))
        end
      end
    end

    # @macro common_transducer
    define_transducer_class :filter do
      define_reducer_class do
        def step(result, input)
          @handler.process(input) ? @reducer.step(result, input) : result
        end
      end
    end

    # @macro common_transducer
    define_transducer_class :remove do
      define_reducer_class do
        def step(result, input)
          @handler.process(input) ? result : @reducer.step(result, input)
        end
      end
    end

    # @method take(n)
    # @return [Transducer]
    define_transducer_class :take do
      define_reducer_class do
        def initialize(reducer, n)
          super(reducer)
          @n = n
        end

        def step(result, input)
          @n -= 1
          ret = @reducer.step(result, input)
          @n > 0 ? ret : Reduced.new(ret)
        end
      end

      def initialize(n)
        @n = n
      end

      def apply(reducer)
        reducer_class.new(reducer, @n)
      end
    end

    # @macro common_transducer
    define_transducer_class :take_while do
      define_reducer_class do
        def step(result, input)
          @handler.process(input) ? @reducer.step(result, input) : Reduced.new(result)
        end
      end
    end

    # @method take_nth(n)
    # @return [Transducer]
    define_transducer_class :take_nth do
      define_reducer_class do
        def initialize(reducer, n)
          super(reducer)
          @n = n
          @count = 0
        end

        def step(result, input)
          @count += 1
          if @count % @n == 0
            @reducer.step(result, input)
          else
            result
          end
        end
      end

      def initialize(n)
        @n = n
      end

      def apply(reducer)
        reducer_class.new(reducer, @n)
      end
    end

    # @method replace(source_map)
    # @return [Transducer]
    define_transducer_class :replace do
      define_reducer_class do
        def initialize(reducer, smap)
          super(reducer)
          @smap = smap
        end

        def step(result, input)
          if @smap.has_key?(input)
            @reducer.step(result, @smap[input])
          else
            @reducer.step(result, input)
          end
        end
      end

      def initialize(smap)
        @smap = case smap
                when Hash
                  smap
                else
                  (0...smap.size).zip(smap).to_h
                end
      end

      def apply(reducer)
        reducer_class.new(reducer, @smap)
      end
    end

    # @macro common_transducer
    define_transducer_class :keep do
      define_reducer_class do
        def step(result, input)
          x = @handler.process(input)
          x.nil? ? result : @reducer.step(result, x)
        end
      end
    end

    # @macro common_transducer
    # @note the handler for this method requires two arguments: the
    #   index and the input.
    define_transducer_class :keep_indexed do
      define_reducer_class do
        def initialize(*)
          super
          @index = -1
        end

        def step(result, input)
          @index += 1
          x = @handler.process(@index, input)
          x.nil? ? result : @reducer.step(result, x)
        end
      end
    end

    # @method drop(n)
    # @return [Transducer]
    define_transducer_class :drop do
      define_reducer_class do
        def initialize(reducer, n)
          super(reducer)
          @n = n
        end

        def step(result, input)
          @n -= 1
          @n <= -1 ? @reducer.step(result, input) : result
        end
      end

      def initialize(n)
        @n = n
      end

      def apply(reducer)
        reducer_class.new(reducer, @n)
      end
    end

    # @macro common_transducer
    define_transducer_class :drop_while do
      define_reducer_class do
        def initalize(*)
          super
          @done_dropping = false
        end

        def step(result, input)
          @done_dropping ||= !@handler.process(input)
          @done_dropping ? @reducer.step(result, input) : result
        end
      end
    end

    # @method dedupe
    # @return [Transducer]
    define_transducer_class :dedupe do
      define_reducer_class do
        def initialize(*)
          super
          @prior = :no_value_provided_for_transducer
        end

        def step(result, input)
          ret = input == @prior ? result : @reducer.step(result, input)
          @prior = input
          ret
        end
      end
    end

    # @method partition_by
    # @return [Transducer]
    define_transducer_class :partition_by do
      define_reducer_class do
        def initialize(*)
          super
          @a = []
          @prev_val = :no_value_provided_for_transducer
        end

        def complete(result)
          result = if @a.empty?
                     result
                   else
                     a = @a.dup
                     @a.clear
                     @reducer.step(result, a)
                   end
          @reducer.complete(result)
        end

        def step(result, input)
          prev_val = @prev_val
          val = @handler.process(input)
          @prev_val = val
          if val == prev_val || prev_val == :no_value_provided_for_transducer
            @a << input
            result
          else
            a = @a.dup
            @a.clear
            ret = @reducer.step(result, a)
            @a << input unless (Reduced === ret)
            ret
          end
        end
      end
    end

    # @method partition_all
    # @return [Transducer]
    define_transducer_class :partition_all do
      define_reducer_class do
        def initialize(reducer, n)
          super(reducer)
          @n = n
          @a = []
        end

        def step(result, input)
          @a << input
          if @a.size == @n
            a = @a.dup
            @a.clear
            @reducer.step(result, a)
          else
            result
          end
        end

        def complete(result)
          if @a.empty?
            result
          else
            a = @a.dup
            @a.clear
            @reducer.step(result, a)
          end
        end
      end

      def initialize(n)
        @n = n
      end

      def apply(reducer)
        reducer_class.new(reducer, @n)
      end
    end

    class RandomSampleHandler
      def initialize(prob)
        @prob = prob
      end

      def process(_)
        @prob > Random.rand
      end
    end

    # @return [Transducer]
    def random_sample(prob)
      filter RandomSampleHandler.new(prob)
    end

    # @method cat
    # @return [Transducer]
    define_transducer_class :cat do
      define_reducer_class do
        def step(result, input)
          Transducers.transduce(PreservingReduced.new, @reducer, result, input)
        end
      end
    end

    # @api private
    class ComposedTransducer
      def initialize(*transducers)
        @transducers = transducers
      end

      def apply(reducer)
        @transducers.reverse.reduce(reducer) {|r,t| t.apply(r)}
      end
    end

    # @return [Transducer]
    def compose(*transducers)
      ComposedTransducer.new(*transducers)
    end

    # @return [Transducer]
    def mapcat(handler=nil, &block)
      compose(map(handler, &block), cat)
    end
  end
end

{'fraction_non_alphanumeric': 0.09800988553590011, 'fraction_numerical': 0.008779916753381894, 'mean_word_length': 1.9687228496959166, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '187897', 'n_tokens_mistral': 9997, 'n_tokens_neox': 9362, 'n_words': 2826}