<commit_before>;
; file-utils.scm
;
; Assorted file and directory utils.
; Copyright (c) 2008 <NAME>
;

(use-modules (ice-9 rdelim))
(use-modules (ice-9 popen))
(use-modules (ice-9 rw))

; ---------------------------------------------------------------------
; Given a directory, return a list of all of the files in the directory
; Do not return anything that is a subdirectory, pipe, special file etc.
;
(define (list-files dir)

	(define (isfile? file)
		(eq? 'regular (stat:type (stat 
				(string-join (list dir file) "/")))
		)
	)

	; suck all the filenames off a port
	(define (suck-in-filenames port lst)
		(let ((one-file (readdir port)))
			(if (eof-object? one-file)
				lst
				(suck-in-filenames port 
					(if (isfile? one-file)
						(cons one-file lst)
						lst
					)
				)
			)
		)
	)
	(let* ((dirport (opendir dir))
			(filelist (suck-in-filenames dirport '()))
		)
		(closedir dirport)
		filelist
	)
)

; ---------------------------------------------------------------------
; exec-scm-from-port port
;
; Load data read from the indicated port
; The port should contain valid scheme; this routine will read and 
; execute that scheme data.
;
(define (exec-scm-from-port port)

	; Suck in a bunch of UTF-8 text off of a port, until the port is
	; empty (#eof) and return a string holding the port (file) contents.
	; Use read-string!/partial for speed.
	(define (speedy-suck-in-text port str)
		(let* ((str-buff (make-string 1123123))
		       (line-len (read-string!/partial str-buff port)))
			(if (eq? #f line-len)
				str 
				(speedy-suck-in-text port 
					(string-append str (substring/shared str-buff 0 line-len))
				)
			)
		)
	)

	; Suck in a bunch of UTF-8 text off of a port, until the port is
	; empty (#eof) and return a string holding the port (file) contents.
	; The code blow is "obvious" but *painfully slowwww* !!!
;	(define (suck-in-text port str)
;		(let ((one-line (read-line port)))
;			(if (eof-object? one-line)
;				str
;				(suck-in-text port 
;					(string-join (list str one-line "\n"))
;				)
;			)
;		)
;	)
;
	(let ((data (speedy-suck-in-text port "")))
		; read data from port
		(eval-string data)
	)
)

; ---------------------------------------------------------------------
; exec-scm-from-cmd cmd-string
;
; Load data generated by the system command cmd-string. The command 
; should generate valid scheme, and send its data to stdout; this
; routine will read and execute that scheme data.
;
; Example:
; (exec-scm-from-cmd "cat /tmp/some-file.scm")
; (exec-scm-from-cmd "cat /tmp/some-file.txt | perl ./bin/some-script.pl")
;
(define (exec-scm-from-cmd cmd-string)

	(let ((port (open-input-pipe cmd-string)))
		(exec-scm-from-port port)
		(close-pipe port)
	)<commit_msg>Fix a UTF-8 bug.<commit_after>;
; file-utils.scm
;
; Assorted file and directory utils.
; Copyright (c) 2008, 2013 <NAME>
;

(use-modules (ice-9 rdelim))
(use-modules (ice-9 popen))
(use-modules (ice-9 rw))
(use-modules (rnrs io ports))

; ---------------------------------------------------------------------
; Given a directory, return a list of all of the files in the directory
; Do not return anything that is a subdirectory, pipe, special file etc.
;
(define (list-files dir)

	(define (isfile? file)
		(eq? 'regular (stat:type (stat 
				(string-join (list dir file) "/")))
		)
	)

	; suck all the filenames off a port
	(define (suck-in-filenames port lst)
		(let ((one-file (readdir port)))
			(if (eof-object? one-file)
				lst
				(suck-in-filenames port 
					(if (isfile? one-file)
						(cons one-file lst)
						lst
					)
				)
			)
		)
	)
	(let* ((dirport (opendir dir))
			(filelist (suck-in-filenames dirport '()))
		)
		(closedir dirport)
		filelist
	)
)

; ---------------------------------------------------------------------
; exec-scm-from-port port
;
; Load (UTF-8 encoded) data read from the indicated port
; The port should contain valid scheme; this routine will read and 
; execute that scheme data.
;
(define (exec-scm-from-port port)

	; get-string-all is a new r6rs proceedure, usck in all bytes until
	; EOF on the port. Seems like TCP/IP ports end up being textual in
	; guile, and the default r6rs transcoder is UTF8 and soe everyone
	; is happy, these days.  Note, in the good-old bad days, we used
	; ice-9 rw read-string!/partial for this, which went buggy, and
	; started mangling at some point.
	(eval-string (get-string-all port))
)

; ---------------------------------------------------------------------
; exec-scm-from-cmd cmd-string
;
; Load data generated by the system command cmd-string. The command 
; should generate valid scheme, and send its data to stdout; this
; routine will read and execute that scheme data.
;
; Example:
; (exec-scm-from-cmd "cat /tmp/some-file.scm")
; (exec-scm-from-cmd "cat /tmp/some-file.txt | perl ./bin/some-script.pl")
;
(define (exec-scm-from-cmd cmd-string)

	(let ((port (open-input-pipe cmd-string)))
		(exec-scm-from-port port)
		(close-pipe port)
	)
{'fraction_non_alphanumeric': 0.20906344410876132, 'fraction_numerical': 0.006847935548841893, 'mean_word_length': 2.247874427730543, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5599554', 'n_tokens_mistral': 1857, 'n_tokens_neox': 1690, 'n_words': 544}