<commit_before>module Hamster

  class << self

    def list(*items)
      items.reverse.inject(EmptyList) { |list, item| list.cons(item) }
    end

    def stream(&block)
      block_given? or return EmptyList
      Stream.new(yield) { stream(&block) }
    end

    def interval(from, to)
      if from > to
        EmptyList
      else
        Stream.new(from) { interval(from.succ, to) }
      end
    end
    alias_method :range, :interval

  end

  module List

    def empty?
      false
    end

    def size
      reduce(0) { |memo, item| memo + 1 }
    end
    alias_method :length, :size

    def cons(item)
      Sequence.new(item, self)
    end
    alias_method :>>, :cons

    def each
      block_given? or return self

      list = self
      while !list.empty?
        yield(list.head)
        list = list.tail
      end

      nil
    end

    def map(&block)
      block_given? or return self
      Stream.new(yield(head)) { tail.map(&block) }
    end
    alias_method :collect, :map

    def reduce(memo)
      block_given? or return memo

      list = self
      while !list.empty?
        memo = yield(memo, list.head)
        list = list.tail
      end
      memo
    end
    alias_method :inject, :reduce

    def filter(&block)
      block_given? or return self

      list = self
      while !yield(list.head)
        list = list.tail
        return list if list.empty?
      end
      Stream.new(list.head) { list.tail.filter(&block) }
    end
    alias_method :select, :filter

    def reject(&block)
      block_given? or return self

      list = self
      while yield(list.head)
        list = list.tail
        return list if list.empty?
      end
      Stream.new(list.head) { list.tail.reject(&block) }
    end

    def take_while(&block)
      block_given? or return self
      if yield(head)
        Stream.new(head) { tail.take_while(&block) }
      else
        EmptyList
      end
    end

    def drop_while
      block_given? or return self
      list = self
      while !list.empty? && yield(list.head)
        list = list.tail
      end
      list
    end

    def take(number)
      if number > 0
        Stream.new(head) { tail.take(number - 1) }
      else
        EmptyList
      end
    end

    def drop(number)
      list = self
      while !list.empty? && number > 0
        number -= 1
        list = list.tail
      end
      list
    end

    def include?(item)
      list = self
      while !list.empty?
        return true if item == list.head
        list = list.tail
      end
      false
    end
    alias_method :member?, :include?

    def eql?(other)
      return true if other.equal?(self)
      return false unless other.is_a?(List)
      other.head == head && other.tail.eql?(tail)
    end
    alias_method :==, :eql?

    def dup
      self
    end
    alias_method :clone, :dup

    def to_a
      reduce([]) { |a, item| a << item }
    end
<commit_msg>Try simplify given we're now using iteration rather than recursion for List methods.<commit_after>module Hamster

  class << self

    def list(*items)
      items.reverse.inject(EmptyList) { |list, item| list.cons(item) }
    end

    def stream(&block)
      block_given? or return EmptyList
      Stream.new(yield) { stream(&block) }
    end

    def interval(from, to)
      if from > to
        EmptyList
      else
        Stream.new(from) { interval(from.succ, to) }
      end
    end
    alias_method :range, :interval

  end

  module List

    def empty?
      false
    end

    def size
      reduce(0) { |memo, item| memo + 1 }
    end
    alias_method :length, :size

    def cons(item)
      Sequence.new(item, self)
    end
    alias_method :>>, :cons

    def each
      block_given? or return self
      list = self
      while !list.empty?
        yield(list.head)
        list = list.tail
      end
      nil
    end

    def map(&block)
      block_given? or return self
      Stream.new(yield(head)) { tail.map(&block) }
    end
    alias_method :collect, :map

    def reduce(memo)
      each { |item| memo = yield(memo, item)  } if block_given?
      memo
    end
    alias_method :inject, :reduce

    def filter(&block)
      block_given? or return self
      reject { |item| !yield(item) }
    end
    alias_method :select, :filter

    def reject(&block)
      block_given? or return self
      list = self
      while yield(list.head)
        list = list.tail
        return list if list.empty?
      end
      Stream.new(list.head) { list.tail.reject(&block) }
    end

    def take_while(&block)
      block_given? or return self
      if yield(head)
        Stream.new(head) { tail.take_while(&block) }
      else
        EmptyList
      end
    end

    def drop_while
      block_given? or return self
      list = self
      while !list.empty? && yield(list.head)
        list = list.tail
      end
      list
    end

    def take(number)
      if number > 0
        Stream.new(head) { tail.take(number - 1) }
      else
        EmptyList
      end
    end

    def drop(number)
      list = self
      while !list.empty? && number > 0
        number -= 1
        list = list.tail
      end
      list
    end

    def include?(object)
      each { |item| return true if object == item }
      false
    end
    alias_method :member?, :include?

    def eql?(other)
      return true if other.equal?(self)
      return false unless other.is_a?(List)
      other.head == head && other.tail.eql?(tail)
    end
    alias_method :==, :eql?

    def dup
      self
    end
    alias_method :clone, :dup

    def to_a
      reduce([]) { |a, item| a << item }
    end

{'fraction_non_alphanumeric': 0.09524657290368524, 'fraction_numerical': 0.0021363717286807904, 'mean_word_length': 1.7936350074589755, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1505336', 'n_tokens_mistral': 1838, 'n_tokens_neox': 1754, 'n_words': 510}