<commit_before>;; Copyright (c) 2012-2018 <NAME>.  All rights reserved.
;; BSD 3-Clause License: http://opensource.org/licenses/BSD-3-Clause

(define-syntax raise-error
  (syntax-rules ()
    ((_ location message obj ...)
     (error location message obj ...))))

(define *shen-globals* (make-eq-hashtable))

(define (shen-global-set! var value)
  (hashtable-set! *shen-globals* var value))

(define *hash-table-default* (string-append "_" "-"))

(define (shen-global-get var default)
  (let ((res (hashtable-ref *shen-globals* var *hash-table-default*)))
    (if (eq? res *hash-table-default*)
        (default)
        res)))

(define (kl-var-clean sym)
  (if (symbol? sym)
      (let* ((str (symbol->string sym))
             (len (string-length str)))
        (if (and (> len 3)
                 (string=? "kl:" (substring str 0 3)))
         (string->symbol (substring str 3 len))
         sym))
      sym))

(define (error-message e)
  (let* ((msg (condition-message e))
         (irritants (condition-irritants e)))
    (cond ((format-condition? e)
           (apply format msg (map kl-var-clean irritants)))
          ((null? irritants) msg)
          (else (format "~a: ~{~s~}" msg irritants)))))

(define (error-location e)
  (if (who-condition? e)
      (condition-who e)
      '*unknown-location*))

(define (full-path-for-file filename)
  (if (path-absolute? filename)
      filename
      (string-append (current-directory)
                     (string (directory-separator))
                     filename)))

(define (open-binary-input-file filename)
  (open-file-input-port filename))

(define (open-binary-output-file filename)
  (open-file-output-port filename (file-options no-fail)))

(define (time->float t)
  (+ (time-second t)
     (/ (time-nanosecond t) 1e+9)))

(define (write-byte byte o)
  (put-u8 o byte)
  (flush-output-port o)
  byte)

(define (read-byte i)
  (let ((byte (get-u8 i)))
    (if (eof-object? byte)
        -1
        byte)))

(define (read-file-as-string filename)
  (call-with-input-file (full-path-for-file filename)
    (lambda (in)
      (let ((s (get-string-all in)))
        (if (eof-object? s)
            ""
            s)))))

(define (read-file-as-bytelist filename)
  (let* ((in (open-file-input-port (full-path-for-file filename)))
         (bytes (get-bytevector-all in)))
    (close-input-port in)
    (if (eof-object? bytes)
        '()
        (bytevector->u8-list bytes))))

(define (hashtable-fold ht f init)
  (let-values (((keys values) (hashtable-entries ht)))
    (let ((limit (vector-length keys)))
      (let loop ((i 0)
                 (acc init))
        (if (fx=? i limit)
            acc
            (let ((k (vector-ref keys i))
                  (v (vector-ref values i)))
              (loop (fx+ i 1) (f k v acc))))))))
<commit_msg>Only flish on write-byte if output port is stdout or stderr<commit_after>;; Copyright (c) 2012-2018 <NAME>.  All rights reserved.
;; BSD 3-Clause License: http://opensource.org/licenses/BSD-3-Clause

(define-syntax raise-error
  (syntax-rules ()
    ((_ location message obj ...)
     (error location message obj ...))))

(define *shen-globals* (make-eq-hashtable))

(define (shen-global-set! var value)
  (hashtable-set! *shen-globals* var value))

(define *hash-table-default* (string-append "_" "-"))

(define (shen-global-get var default)
  (let ((res (hashtable-ref *shen-globals* var *hash-table-default*)))
    (if (eq? res *hash-table-default*)
        (default)
        res)))

(define (kl-var-clean sym)
  (if (symbol? sym)
      (let* ((str (symbol->string sym))
             (len (string-length str)))
        (if (and (> len 3)
                 (string=? "kl:" (substring str 0 3)))
         (string->symbol (substring str 3 len))
         sym))
      sym))

(define (error-message e)
  (let* ((msg (condition-message e))
         (irritants (condition-irritants e)))
    (cond ((format-condition? e)
           (apply format msg (map kl-var-clean irritants)))
          ((null? irritants) msg)
          (else (format "~a: ~{~s~}" msg irritants)))))

(define (error-location e)
  (if (who-condition? e)
      (condition-who e)
      '*unknown-location*))

(define (full-path-for-file filename)
  (if (path-absolute? filename)
      filename
      (string-append (current-directory)
                     (string (directory-separator))
                     filename)))

(define (open-binary-input-file filename)
  (open-file-input-port filename))

(define (open-binary-output-file filename)
  (open-file-output-port filename (file-options no-fail)))

(define (time->float t)
  (+ (time-second t)
     (/ (time-nanosecond t) 1e+9)))

(define (write-byte byte o)
  (put-u8 o byte)
  (let ((name (port-name o)))
    (if (or (equal? name "stdout") (equal? name "stderr"))
        (flush-output-port o)
        #f))
  byte)

(define (read-byte i)
  (let ((byte (get-u8 i)))
    (if (eof-object? byte)
        -1
        byte)))

(define (read-file-as-string filename)
  (call-with-input-file (full-path-for-file filename)
    (lambda (in)
      (let ((s (get-string-all in)))
        (if (eof-object? s)
            ""
            s)))))

(define (read-file-as-bytelist filename)
  (let* ((in (open-file-input-port (full-path-for-file filename)))
         (bytes (get-bytevector-all in)))
    (close-input-port in)
    (if (eof-object? bytes)
        '()
        (bytevector->u8-list bytes))))

(define (hashtable-fold ht f init)
  (let-values (((keys values) (hashtable-entries ht)))
    (let ((limit (vector-length keys)))
      (let loop ((i 0)
                 (acc init))
        (if (fx=? i limit)
            acc
            (let ((k (vector-ref keys i))
                  (v (vector-ref values i)))
              (loop (fx+ i 1) (f k v acc))))))))

{'fraction_non_alphanumeric': 0.17273042273042272, 'fraction_numerical': 0.007623007623007623, 'mean_word_length': 2.3760233918128657, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3353351', 'n_tokens_mistral': 2001, 'n_tokens_neox': 1948, 'n_words': 521}