<commit_before>;; Copyright (c) 2013-2014 by <NAME>, All Rights Reserved.

;; A stream is a pair created by `(cons a (delay b))`.

(define-macro (stream-cons a b) `(cons ,a (delay ,b)))

;; Functions for working with sequences.
;; A sequence can be either a normal list or a stream.

(define (first stream) (head stream))
(define (rest stream) (force (tail stream)))

(define (is_stream obj)
  (and (pair? obj)
       (not (list? obj))
       (is_promise (cdr obj))))

(define (stream-at i stream)
  (if (= i 0)
      (first stream)
      (stream-at (- i 1) (rest stream))))

(define (nth n seq) 
  (if (<= n 0)
      (first seq)
      (nth (- n 1) (rest seq))))

(define (nth_tail n seq)
  (if (<= n 0) (force seq)
      (nth_tail (- n 1) (rest seq))))
      
(define (second seq) (nth 1 seq))
(define (third seq) (nth 2 seq))
(define (fourth seq) (nth 3 seq))
(define (fifth seq) (nth 4 seq))
(define (sixth seq) (nth 5 seq))
(define (seventh seq) (nth 6 seq))
(define (eighth seq) (nth 7 seq))
(define (ninth seq) (nth 8 seq))
(define (tenth seq) (nth 9 seq))

;; Based on the `map` implementation from 
;; Kent Dybvig's The Scheme Programming Language.
(define (generic-map f ls . more)
  (let ((stream? (is_stream ls)))
    (let ((car (if stream? first car))
	  (cdr (if stream? rest cdr)))
      (if (null? more)
	  (let map1 ((ls ls))
	    (if (null? ls)
		'()
		(let ((a (f (car ls)))
		      (b (map1 (cdr ls))))
		  (if stream?
		      (stream-cons a b)
		      (cons a b)))))
	  (let map-more ((ls ls) (more more))
	    (if (null? ls)
		'()
		(let ((a (apply f (car ls) (generic-map car more)))
		      (b (map-more (cdr ls) (generic-map cdr more))))
		  (if stream?
		      (stream-cons a b)
		      (cons a b)))))))))

(define map generic-map)

(define (generic-for-each f ls . more)
  (let ((stream? (is_stream ls)))
    (let ((car (if stream? first car))
	  (cdr (if stream? rest cdr)))
      (do ([ls ls (cdr ls)] [more more (map cdr more)])
	  ((null? ls))
	(apply f (car ls) (map car more))))))

(define for_each generic-for-each)

(define (stream-filter fn stream #!key drill)
  (cond ((null? stream) 
	 '())
	((and (is_stream (first stream)) drill)
	 (stream-cons
	  (stream-filter fn (first stream) drill: drill)
	  (stream-filter fn (rest stream) drill: drill)))
	((fn (first stream))
	 (stream-cons 
	  (first stream)
	  (stream-filter fn (rest stream) drill: drill)))
	(else
	 (stream-filter fn (rest stream) drill: drill))))      

(define (filter fn ls #!key drill)
  (if (is_stream ls)
      (stream-filter fn ls drill: drill)
      (let loop ((ls ls)
		 (result '()))
	(cond ((null? ls)
	       (reverse result))
	      ((and (list? (car ls)) drill)
	       (loop (cdr ls) (cons (filter fn (car ls) drill: drill) result)))
	      ((fn (car ls))
	       (loop (cdr ls) (cons (car ls) result)))
	      (else
	       (loop (cdr ls) result))))))

(define (stream-accumulate fn initial stream)
  (if (null? stream)
      initial
      (fn 
       (first stream)
       (stream-accumulate fn initial (rest stream)))))

(define (accumulate fn initial seq)
  (if (is_stream seq)
      (stream-accumulate fn initial seq)
      (fold_right fn initial seq)))<commit_msg>enumerate to generate streams.<commit_after>;; Copyright (c) 2013-2014 by <NAME>, All Rights Reserved.

;; A stream is a pair created by `(cons a (delay b))`.

(define-macro (stream-cons a b) `(cons ,a (delay ,b)))

;; Functions for working with sequences.
;; A sequence can be either a normal list or a stream.

(define (first stream) (head stream))
(define (rest stream) (force (tail stream)))

(define (is_stream obj)
  (and (pair? obj)
       (not (list? obj))
       (is_promise (cdr obj))))

(define (stream-at i stream)
  (if (= i 0)
      (first stream)
      (stream-at (- i 1) (rest stream))))

(define (nth n seq) 
  (if (<= n 0)
      (first seq)
      (nth (- n 1) (rest seq))))

(define (nth_tail n seq)
  (if (<= n 0) (force seq)
      (nth_tail (- n 1) (rest seq))))
      
(define (second seq) (nth 1 seq))
(define (third seq) (nth 2 seq))
(define (fourth seq) (nth 3 seq))
(define (fifth seq) (nth 4 seq))
(define (sixth seq) (nth 5 seq))
(define (seventh seq) (nth 6 seq))
(define (eighth seq) (nth 7 seq))
(define (ninth seq) (nth 8 seq))
(define (tenth seq) (nth 9 seq))

;; Based on the `map` implementation from 
;; <NAME>'s The Scheme Programming Language.
(define (generic-map f ls . more)
  (let ((stream? (is_stream ls)))
    (let ((car (if stream? first car))
	  (cdr (if stream? rest cdr)))
      (if (null? more)
	  (let map1 ((ls ls))
	    (if (null? ls)
		'()
		(let ((a (f (car ls)))
		      (b (map1 (cdr ls))))
		  (if stream?
		      (stream-cons a b)
		      (cons a b)))))
	  (let map-more ((ls ls) (more more))
	    (if (null? ls)
		'()
		(let ((a (apply f (car ls) (generic-map car more)))
		      (b (map-more (cdr ls) (generic-map cdr more))))
		  (if stream?
		      (stream-cons a b)
		      (cons a b)))))))))

(define map generic-map)

(define (generic-for-each f ls . more)
  (let ((stream? (is_stream ls)))
    (let ((car (if stream? first car))
	  (cdr (if stream? rest cdr)))
      (do ([ls ls (cdr ls)] [more more (map cdr more)])
	  ((null? ls))
	(apply f (car ls) (map car more))))))

(define for_each generic-for-each)

(define (stream-filter fn stream #!key drill)
  (cond ((null? stream) 
	 '())
	((and (is_stream (first stream)) drill)
	 (stream-cons
	  (stream-filter fn (first stream) drill: drill)
	  (stream-filter fn (rest stream) drill: drill)))
	((fn (first stream))
	 (stream-cons 
	  (first stream)
	  (stream-filter fn (rest stream) drill: drill)))
	(else
	 (stream-filter fn (rest stream) drill: drill))))      

(define (filter fn ls #!key drill)
  (if (is_stream ls)
      (stream-filter fn ls drill: drill)
      (let loop ((ls ls)
		 (result '()))
	(cond ((null? ls)
	       (reverse result))
	      ((and (list? (car ls)) drill)
	       (loop (cdr ls) (cons (filter fn (car ls) drill: drill) result)))
	      ((fn (car ls))
	       (loop (cdr ls) (cons (car ls) result)))
	      (else
	       (loop (cdr ls) result))))))

(define (stream-accumulate fn initial stream)
  (if (null? stream)
      initial
      (fn 
       (first stream)
       (stream-accumulate fn initial (rest stream)))))

(define (accumulate fn initial seq)
  (if (is_stream seq)
      (stream-accumulate fn initial seq)
      (fold_right fn initial seq)))

(define (enumerate start end #!optional (cmpr <=) (next inc))
    (if (cmpr start end)
        (let ((elem (next start)))
          (stream-cons start (enumerate elem end cmpr next)))
        '()))

{'fraction_non_alphanumeric': 0.1702095353780747, 'fraction_numerical': 0.007591861524445794, 'mean_word_length': 2.843057176196033, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3715876', 'n_tokens_mistral': 2562, 'n_tokens_neox': 2315, 'n_words': 892}