<commit_before>require 'forwardable'
require 'monitor'

require 'hamster/set'

module Hamster

  class << self

    extend Forwardable

    def list(*items)
      items.reverse.reduce(EmptyList) { |list, item| list.cons(item) }
    end

    def stream(&block)
      return EmptyList unless block_given?
      Stream.new { Sequence.new(yield, stream(&block)) }
    end

    def interval(from, to)
      return EmptyList if from > to
      Stream.new { Sequence.new(from, interval(from.succ, to)) }
    end
    def_delegator :self, :interval, :range

    def repeat(item)
      Stream.new { Sequence.new(item, repeat(item)) }
    end

    def replicate(number, item)
      repeat(item).take(number)
    end

    def iterate(item, &block)
      Stream.new { Sequence.new(item, iterate(yield(item), &block)) }
    end

  end

  module List

    extend Forwardable

    Undefined = Object.new

    CADR = /^c([ad]+)r$/

    def first
      head
    end

    def empty?
      false
    end
    def_delegator :self, :empty?, :null?

    def size
      reduce(0) { |memo, item| memo.succ }
    end
    def_delegator :self, :size, :length

    def cons(item)
      Sequence.new(item, self)
    end
    def_delegator :self, :cons, :>>

    def each
      return self unless block_given?
      list = self
      while !list.empty?
        yield(list.head)
        list = list.tail
      end
    end
    def_delegator :self, :each, :foreach

    def map(&block)
      return self if empty?
      return self unless block_given?
      Stream.new { Sequence.new(yield(head), tail.map(&block)) }
    end
    def_delegator :self, :map, :collect

    def reduce(memo = Undefined, &block)
      return tail.reduce(head, &block) if memo.equal?(Undefined)
      return memo unless block_given?
      each { |item| memo = yield(memo, item)  }
      memo
    end
    def_delegator :self, :reduce, :inject
    def_delegator :self, :reduce, :fold

    def filter(&block)
      return self if empty?
      return self unless block_given?
      Stream.new do
        if yield(head)
          Sequence.new(head, tail.filter(&block))
        else
          tail.filter(&block)
        end
      end
    end
    def_delegator :self, :filter, :select
    def_delegator :self, :filter, :find_all

    def remove(&block)
      return self if empty?
      return self unless block_given?
      filter { |item| !yield(item) }
    end
    def_delegator :self, :remove, :reject
    def_delegator :self, :remove, :delete_if

    def take_while(&block)
      return self if empty?
      return self unless block_given?
      Stream.new do
        if yield(head)
          Sequence.new(head, tail.take_while(&block))
        else
          EmptyList
        end
      end
    end

    def drop_while(&block)
      return self if empty?
      return self unless block_given?
      Stream.new do
        if yield(head)
          tail.drop_while(&block)
        else
          self
        end
      end
    end

    def take(number)
      Stream.new do
        if empty?
          self
        elsif number > 0
          Sequence.new(head, tail.take(number - 1))
        else
          EmptyList
        end
      end<commit_msg>List#take_while and #drop_while are now lazy<commit_after>require 'forwardable'
require 'monitor'

require 'hamster/set'

module Hamster

  class << self

    extend Forwardable

    def list(*items)
      items.reverse.reduce(EmptyList) { |list, item| list.cons(item) }
    end

    def stream(&block)
      return EmptyList unless block_given?
      Stream.new { Sequence.new(yield, stream(&block)) }
    end

    def interval(from, to)
      return EmptyList if from > to
      Stream.new { Sequence.new(from, interval(from.succ, to)) }
    end
    def_delegator :self, :interval, :range

    def repeat(item)
      Stream.new { Sequence.new(item, repeat(item)) }
    end

    def replicate(number, item)
      repeat(item).take(number)
    end

    def iterate(item, &block)
      Stream.new { Sequence.new(item, iterate(yield(item), &block)) }
    end

  end

  module List

    extend Forwardable

    Undefined = Object.new

    CADR = /^c([ad]+)r$/

    def first
      head
    end

    def empty?
      false
    end
    def_delegator :self, :empty?, :null?

    def size
      reduce(0) { |memo, item| memo.succ }
    end
    def_delegator :self, :size, :length

    def cons(item)
      Sequence.new(item, self)
    end
    def_delegator :self, :cons, :>>

    def each
      return self unless block_given?
      list = self
      while !list.empty?
        yield(list.head)
        list = list.tail
      end
    end
    def_delegator :self, :each, :foreach

    def map(&block)
      return self if empty?
      return self unless block_given?
      Stream.new { Sequence.new(yield(head), tail.map(&block)) }
    end
    def_delegator :self, :map, :collect

    def reduce(memo = Undefined, &block)
      return tail.reduce(head, &block) if memo.equal?(Undefined)
      return memo unless block_given?
      each { |item| memo = yield(memo, item)  }
      memo
    end
    def_delegator :self, :reduce, :inject
    def_delegator :self, :reduce, :fold

    def filter(&block)
      return self if empty?
      return self unless block_given?
      Stream.new do
        if yield(head)
          Sequence.new(head, tail.filter(&block))
        else
          tail.filter(&block)
        end
      end
    end
    def_delegator :self, :filter, :select
    def_delegator :self, :filter, :find_all

    def remove(&block)
      return self if empty?
      return self unless block_given?
      filter { |item| !yield(item) }
    end
    def_delegator :self, :remove, :reject
    def_delegator :self, :remove, :delete_if

    def take_while(&block)
      return self unless block_given?
      Stream.new do
        if empty?
          self
        elsif yield(head)
          Sequence.new(head, tail.take_while(&block))
        else
          EmptyList
        end
      end
    end

    def drop_while(&block)
      return self unless block_given?
      Stream.new do
        if empty?
          self
        elsif yield(head)
          tail.drop_while(&block)
        else
          self
        end
      end
    end

    def take(number)
      Stream.new do
        if empty?
          self
        elsif number > 0
          Sequence.new(head, tail.take(number - 1))
        else
          EmptyList
        end
      end
{'fraction_non_alphanumeric': 0.09590534018371477, 'fraction_numerical': 0.0009341429238673517, 'mean_word_length': 2.061963775023832, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3175250', 'n_tokens_mistral': 2081, 'n_tokens_neox': 1986, 'n_words': 560}