<commit_before>;; -*- Mode: Scheme -*-

(cinclude "fcntl.h")

(define O_RDONLY (%%cexp int "O_RDONLY"))
(define O_WRONLY (%%cexp int "O_WRONLY"))
(define O_RDWR   (%%cexp int "O_RDWR"))
(define O_CREAT  (%%cexp int "O_CREAT"))

(define (open path oflag mode)
  (let ((fd (%%cexp (string int int -> int) "open (%s, %s, %s)" (zero-terminate path) oflag mode)))
    (if (>= fd 0)
	fd
	(error "open() failed"))))

(define (read fd size)
  (let* ((buffer (make-string size))
	 (r (%%cexp (int string int -> int) "read (%s, %s, %s)" fd buffer size)))
    (if (= r size)
	buffer
	(if (< r size)
	    (copy-string buffer r)
	    (error "read() failed")))))

(define (read-into-buffer fd buffer)
  (let* ((size (string-length buffer))
	 ;; XXX range check
	 (r (%%cexp (int string int -> int) "read (%s, %s, %s)" fd buffer size)))
    r))

(define (write fd s)
  (%%cexp (int string int -> int) "write (%s, %s, %s)" fd s (string-length s)))

(define (write-substring fd s start len)
  ;; XXX range check
  (%%cexp (int string int int -> int) "write (%s, %s+%s, %s)" fd s start len))

(define (read-stdin)
  (read 0 1024))

(define (close fd)
  (%%cexp (int -> int) "close (%s)" fd))

;; file I/O 'object'

(define (file/open-read path)
  { fd = (open path O_RDONLY 0)
    buf = (make-string 16384)
    pos = 0
    end = 0 })

(define (file/open-write path create? mode)
  { fd = (open path (if create? (+ O_WRONLY O_CREAT) O_WRONLY) mode)
    buf = (make-string 16384)
    pos = 0 })

(define (file/close self)
  (close self.fd))

(define (file/fill-buffer self)
  (let ((n (read-into-buffer self.fd self.buf)))
    (set! self.end n)
    (set! self.pos 0)
    n))

(define (file/read-buffer self)
  (cond ((< self.pos self.end)
	 (let ((opos self.pos))
	   (set! self.pos self.end)
	   (substring self.buf opos self.end)))
	((= (file/fill-buffer self) 0) "")
	(else
	 (let ((r (substring self.buf self.pos self.end)))
	   (set! self.end 0)
	   (set! self.pos 0)
	   r))))

(define (file/read-char self)
  (cond ((< self.pos self.end)
	 (set! self.pos (+ self.pos 1))
	 (string-ref self.buf (- self.pos 1)))
	((= (file/fill-buffer self) 0) #\eof)
	(else
	 (file/read-char self))))
<commit_msg>STDIN_FILENO STDOUT_FILENO STDERR_FILENO   defined using %%cexp file/open-stdin file/open-stdout   get file-object handles to stdin and stdout<commit_after>;; -*- Mode: Scheme -*-

(cinclude "fcntl.h")
(cinclude "unistd.h")

(define O_RDONLY (%%cexp int "O_RDONLY"))
(define O_WRONLY (%%cexp int "O_WRONLY"))
(define O_RDWR   (%%cexp int "O_RDWR"))
(define O_CREAT  (%%cexp int "O_CREAT"))

(define STDIN_FILENO  (%%cexp int "STDIN_FILENO"))
(define STDOUT_FILENO (%%cexp int "STDOUT_FILENO"))
(define STDERR_FILENO (%%cexp int "STDERR_FILENO"))

(define (open path oflag mode)
  (let ((fd (%%cexp (string int int -> int) "open (%s, %s, %s)" (zero-terminate path) oflag mode)))
    (if (>= fd 0)
	fd
	(error "open() failed"))))

(define (read fd size)
  (let* ((buffer (make-string size))
	 (r (%%cexp (int string int -> int) "read (%s, %s, %s)" fd buffer size)))
    (if (= r size)
	buffer
	(if (< r size)
	    (copy-string buffer r)
	    (error "read() failed")))))

(define (read-into-buffer fd buffer)
  (let* ((size (string-length buffer))
	 ;; XXX range check
	 (r (%%cexp (int string int -> int) "read (%s, %s, %s)" fd buffer size)))
    r))

(define (write fd s)
  (%%cexp (int string int -> int) "write (%s, %s, %s)" fd s (string-length s)))

(define (write-substring fd s start len)
  ;; XXX range check
  (%%cexp (int string int int -> int) "write (%s, %s+%s, %s)" fd s start len))

(define (read-stdin)
  (read 0 1024))

(define (close fd)
  (%%cexp (int -> int) "close (%s)" fd))

;; file I/O 'object'

(define (file/open-read path)
  { fd = (open path O_RDONLY 0)
    buf = (make-string 16384)
    pos = 0
    end = 0 })

(define (file/open-write path create? mode)
  { fd = (open path (if create? (+ O_WRONLY O_CREAT) O_WRONLY) mode)
    buf = (make-string 16384)
    pos = 0 })

(define (file/open-stdin)
  { fd = STDIN_FILENO
    buf = (make-string 16384)
    pos = 0
    end = 0 })

(define (file/open-stdout)
  { fd = STDOUT_FILENO
    buf = (make-string 16384)
    pos = 0 })

(define (file/close self)
  (close self.fd))

(define (file/fill-buffer self)
  (let ((n (read-into-buffer self.fd self.buf)))
    (set! self.end n)
    (set! self.pos 0)
    n))

(define (file/read-buffer self)
  (cond ((< self.pos self.end)
	 (let ((opos self.pos))
	   (set! self.pos self.end)
	   (substring self.buf opos self.end)))
	((= (file/fill-buffer self) 0) "")
	(else
	 (let ((r (substring self.buf self.pos self.end)))
	   (set! self.end 0)
	   (set! self.pos 0)
	   r))))

(define (file/read-char self)
  (cond ((< self.pos self.end)
	 (set! self.pos (+ self.pos 1))
	 (string-ref self.buf (- self.pos 1)))
	((= (file/fill-buffer self) 0) #\eof)
	(else
	 (file/read-char self))))

{'fraction_non_alphanumeric': 0.19548717948717947, 'fraction_numerical': 0.013743589743589744, 'mean_word_length': 3.2363162467419637, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '282325', 'n_tokens_mistral': 2030, 'n_tokens_neox': 1923, 'n_words': 603}