<commit_before>module Pacer
  module Routes
    module RouteOperations
      def sort_section(section_name = nil, &block)
        chain_route transform: :sort_section, block: block, section_name: section_name
      end
    end
  end

  module Transform
    module SortSection
      class SortSectionPipe < Pacer::Pipes::RubyPipe
        def initialize(section, block)
          super()
          @to_emit = []
          @section = section
          @block = block
          if section
            section.on_start &method(:on_start)
            section.on_end &method(:on_end)
          else
            on_start nil, 0
          end
        end

        def processNextStart
          while @to_emit.empty?
            element = @starts.next
            @to_sort << element
          end
          element, sort_value = @to_emit.shift
          element
        rescue NativeException => e
          if e.cause.getClass == Pacer::NoSuchElementException.getClass
            if @to_emit.empty?
              raise e.cause
            else
              on_end(nil, 0)
              retry
            end
          else
            raise e
          end
        end

        def on_start(element, count)
          @section_element = element
          @section_number = count
          @to_sort = []
        end

        def on_end(section_element, count)
          if @to_sort.any?
            if @block
              sorted = @to_sort.sort_by do |element|
                @block.call element, section_element, count
              end
              @to_emit.concat sorted 
            else
              @to_emit.concat @to_sort.sort
            end
          end
        end
      end

      attr_accessor :block
      attr_reader :section_name

      def section_name=(name)
        @section_name = name
        @section_route = @back.get_section_route(name)
      end

      protected

      def attach_pipe(end_pipe)
        pipe = SortSectionPipe.new(@section_route.section_events, block)
        pipe.setStarts end_pipe if end_pipe
        pipe
      end
    end
  end
end<commit_msg>Fix double last element.<commit_after>module Pacer
  module Routes
    module RouteOperations
      def sort_section(section_name = nil, &block)
        chain_route transform: :sort_section, block: block, section_name: section_name
      end
    end
  end

  module Transform
    module SortSection
      class SortSectionPipe < Pacer::Pipes::RubyPipe
        def initialize(section, block)
          super()
          @to_emit = []
          @section = section
          @to_sort = []
          @block = block
          if section
            section.on_start &method(:on_start)
            section.on_end &method(:on_end)
          else
            on_start nil, 0
          end
        end

        def processNextStart
          while @to_emit.empty?
            element = @starts.next
            @to_sort << element
          end
          element, sort_value = @to_emit.shift
          element
        rescue NativeException => e
          if e.cause.getClass == Pacer::NoSuchElementException.getClass
            if @to_emit.empty?
              raise e.cause
            else
              on_end(nil, 0)
              retry
            end
          else
            raise e
          end
        end

        def on_start(element, count)
          @section_element = element
          @section_number = count
        end

        def on_end(section_element, count)
          if @to_sort.any?
            if @block
              sorted = @to_sort.sort_by do |element|
                @block.call element, section_element, count
              end
              @to_emit.concat sorted 
              @to_sort = []
            else
              @to_emit.concat @to_sort.sort
            end
          end
        end
      end

      attr_accessor :block
      attr_reader :section_name

      def section_name=(name)
        @section_name = name
        @section_route = @back.get_section_route(name)
      end

      protected

      def attach_pipe(end_pipe)
        pipe = SortSectionPipe.new(@section_route.send(:section_events), block)
        pipe.setStarts end_pipe if end_pipe
        pipe
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.06252960682141165, 'fraction_numerical': 0.0009474182851729038, 'mean_word_length': 1.4638273045507584, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3374188', 'n_tokens_mistral': 1162, 'n_tokens_neox': 1139, 'n_words': 285}