<commit_before>;; -*- Mode: Irken -*-

(include "lib/basis.scm")
(include "lib/map.scm")

;; to prepare:
;;    $ ffi/gen/genffi -gen ffi/sodium.ffi
;;    $ sudo cp ffi/sodium_ffi.scm /usr/local/lib/irken/ffi/
;;
;; to build:
;;    $ self/compile tests/t_sodium.scm -l sodium

(require-ffi 'sodium)

(define (string->hex s)
  (format
   (join
    (map
     (lambda (x) (format (zpad 2 (hex x))))
     (map char->int (string->list s))))))

(define (hash-string m)
  (let ((h (make-string 32))
        (h* (%c-cast uchar (%string->cref #f h)))
        (m* (%c-cast uchar (%string->cref #f m)))<commit_msg>use the BYTES constant.<commit_after>;; -*- Mode: Irken -*-

(include "lib/basis.scm")
(include "lib/map.scm")

;; to prepare:
;;    $ ffi/gen/genffi -gen ffi/sodium.ffi
;;    $ sudo cp ffi/sodium_ffi.scm /usr/local/lib/irken/ffi/
;;
;; to build:
;;    $ self/compile tests/t_sodium.scm -l sodium

(require-ffi 'sodium)

(define (string->hex s)
  (format
   (join
    (map
     (lambda (x) (format (zpad 2 (hex x))))
     (map char->int (string->list s))))))

(define (hash-string m)
  (let ((h (make-string crypto_hash_sha256_BYTES))
        (h* (%c-cast uchar (%string->cref #f h)))
        (m* (%c-cast uchar (%string->cref #f m)))
{'fraction_non_alphanumeric': 0.20954692556634305, 'fraction_numerical': 0.0056634304207119745, 'mean_word_length': 2.9022082018927446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4075677', 'n_tokens_mistral': 525, 'n_tokens_neox': 508, 'n_words': 122}