<commit_before>;; memoize.scm -- caching and memoization utilities
;; Copyright (c) 2003-2013 <NAME>.  All rights reserved.
;; BSD-style license: http://synthcode.com/license.txt

;; Memory and persistent caching with various levels of control, based
;; on a combination of lru-cache from Hato and an older memoization
;; library for Gauche.

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; types

(define-record-type Lru-Cache
  (%make-lru-cache table front back size size-limit compute-size)
  lru-cache?
  (table lru-table)
  (front lru-front lru-front-set!)
  (back lru-back lru-back-set!)
  (size lru-size lru-size-set!)
  (size-limit lru-size-limit)
  (compute-size lru-compute-size))

(define-record-type Lru-Entry
  (make-lru-entry key value size prev)
  lru-entry?
  (key lru-entry-key)
  (value lru-entry-value lru-entry-value-set!)
  (size lru-entry-size lru-entry-size-set!)
  (prev lru-entry-prev lru-entry-prev-set!))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; high-level interface

;;> Analagous to the procedure form of \scheme{define} but
;;> automatically memoizes the function.  Uses \scheme{equal?} for
;;> equality comparisons and reasonable defaults - for finer grained
;;> control use \scheme{memoize}.

(define-syntax define-memoized
  (syntax-rules ()
    ((define-memoized (proc x ...) . body)
     (define proc
       (make-memoizer (lambda (x ...) . body) (length '(x ...)) (make-lru-cache))))
    ((define-memoized (proc . x) . body)
     (define proc
       (make-memoizer (lambda x . body) #f (make-lru-cache ))))))

(define (make-memoizer proc arity cache)
  (let ((ref! (if (lru-cache? cache) lru-ref! hash-table-ref!)))
    (case arity
      ((0)
       proc)
      ((1)
       (lambda (x) (ref! cache x proc)))
      ((2)
       (lambda (x y)
         (ref! cache (cons x y) (lambda (xy) (proc (car xy) (cdr xy))))))
      (else
       (lambda args
         (ref! cache args (lambda (args) (apply proc args))))))))

;;> Returns a memoized version of the procedure \var{proc}.  By
;;> default uses a least-recently-used (LRU) cache, which can be tuned
;;> with the following keyword arguments:
;;>
;;> \items[
;;> \item{cache: an explicit pre-existing cache (LRU or hash-table)}
;;> \item{equal: an equality predicate defaulting to \scheme{equal?}}
;;> \item{hash: a hash function to match the equality predicate, defaulting to \scheme{hash} from \scheme{(srfi 69)}}
;;> \item{init-size: a hint for the initial size of the backing hash table}
;;> \item{size-limit: the maximum size of the cache}
;;> \item{compute-size: compute the size of a cache entry}
;;> ]
;;>
;;> \var{compute-size} is a procedure of two arguments, the key and
;;> value to be stored, and defaults to a constant 1 per entry.  After
;;> every insertion the oldest elements will be removed until the size
;;> is under \var{size-limit}.  You may find
;;>
;;>   \scheme{(lambda (k v) (+ (object-size k) (object-size v)))}
;;>
;;> using \scheme{object-size} from \scheme{(chibi ast)} to be a
;;> useful \var{compute-size}.
;;>
;;> If \var{size-limit} is \scheme{#f} then the cache is unlimited,
;;> and a simple hash-table will be used in place of an LRU cache.

(define (memoize proc . o)
  (let-keywords* o
      ((equal equal: equal?)
       (hash hash: hash)
       (arity arity: (and (not (procedure-variadic? proc))
                          (procedure-arity proc)))
       (init-size init-size: 31)
       (limit size-limit: 1000)
       (compute-size compute-size: (lambda (k v) 1))
       (cache cache: (if limit
                         (make-lru-cache 'equal: equal
                                         'hash: hash
                                         'init-size: init-size
                                         'size-limit: limit
                                         'compute-size: compute-size)
                         (make-hash-table equal hash))))
    (make-memoizer proc arity cache)))

;;> Equivalent to memoize except that the procedure's first argument
;;> must be a pathname.  If the corresponding file has been modified
;;> since the memoized value, the value is recomputed.  Useful to
;;> automatically reflect external changes to a file-backed resource.
;;> The additional keyword argument \scheme{reloader?:}, if true,
;;> indicates that the result of loading is itself a procedure which
;;> should check for updates on each call.<commit_msg>Allowing alist initializers for memoize cache.<commit_after>;; memoize.scm -- caching and memoization utilities
;; Copyright (c) 2003-2013 <NAME>.  All rights reserved.
;; BSD-style license: http://synthcode.com/license.txt

;; Memory and persistent caching with various levels of control, based
;; on a combination of lru-cache from Hato and an older memoization
;; library for Gauche.

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; types

(define-record-type Lru-Cache
  (%make-lru-cache table front back size size-limit compute-size)
  lru-cache?
  (table lru-table)
  (front lru-front lru-front-set!)
  (back lru-back lru-back-set!)
  (size lru-size lru-size-set!)
  (size-limit lru-size-limit)
  (compute-size lru-compute-size))

(define-record-type Lru-Entry
  (make-lru-entry key value size prev)
  lru-entry?
  (key lru-entry-key)
  (value lru-entry-value lru-entry-value-set!)
  (size lru-entry-size lru-entry-size-set!)
  (prev lru-entry-prev lru-entry-prev-set!))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; high-level interface

;;> Analagous to the procedure form of \scheme{define} but
;;> automatically memoizes the function.  Uses \scheme{equal?} for
;;> equality comparisons and reasonable defaults - for finer grained
;;> control use \scheme{memoize}.

(define-syntax define-memoized
  (syntax-rules ()
    ((define-memoized (proc x ...) . body)
     (define proc
       (make-memoizer (lambda (x ...) . body) (length '(x ...)) (make-lru-cache))))
    ((define-memoized (proc . x) . body)
     (define proc
       (make-memoizer (lambda x . body) #f (make-lru-cache ))))))

(define (make-memoizer proc arity cache)
  (let ((ref! (if (lru-cache? cache) lru-ref! hash-table-ref!)))
    (case arity
      ((0)
       proc)
      ((1)
       (lambda (x) (ref! cache x proc)))
      ((2)
       (lambda (x y)
         (ref! cache (cons x y) (lambda (xy) (proc (car xy) (cdr xy))))))
      (else
       (lambda args
         (ref! cache args (lambda (args) (apply proc args))))))))

;;> Returns a memoized version of the procedure \var{proc}.  By
;;> default uses a least-recently-used (LRU) cache, which can be tuned
;;> with the following keyword arguments:
;;>
;;> \items[
;;> \item{cache: an explicit pre-existing cache (LRU or hash-table)}
;;> \item{equal: an equality predicate defaulting to \scheme{equal?}}
;;> \item{hash: a hash function to match the equality predicate, defaulting to \scheme{hash} from \scheme{(srfi 69)}}
;;> \item{init-size: a hint for the initial size of the backing hash table}
;;> \item{size-limit: the maximum size of the cache}
;;> \item{compute-size: compute the size of a cache entry}
;;> ]
;;>
;;> \var{compute-size} is a procedure of two arguments, the key and
;;> value to be stored, and defaults to a constant 1 per entry.  After
;;> every insertion the oldest elements will be removed until the size
;;> is under \var{size-limit}.  You may find
;;>
;;>   \scheme{(lambda (k v) (+ (object-size k) (object-size v)))}
;;>
;;> using \scheme{object-size} from \scheme{(chibi ast)} to be a
;;> useful \var{compute-size}.
;;>
;;> If \var{size-limit} is \scheme{#f} then the cache is unlimited,
;;> and a simple hash-table will be used in place of an LRU cache.

(define (memoize proc . o)
  (let-keywords* o
      ((equal equal: equal?)
       (hash hash: hash)
       (arity arity: (and (not (procedure-variadic? proc))
                          (procedure-arity proc)))
       (init-size init-size: 31)
       (limit size-limit: 1000)
       (compute-size compute-size: (lambda (k v) 1))
       (cache-init cache: '()))
    (let ((cache (cond ((lru-cache? cache-init)
                        cache-init)
                       (limit
                        (make-lru-cache 'equal: equal
                                        'hash: hash
                                        'init-size: init-size
                                        'size-limit: limit
                                        'compute-size: compute-size))
                       (else
                        (make-hash-table equal hash)))))
      ;; allow an alist initializer for the cache
      (if (pair? cache-init)
          (for-each (lambda (x) (lru-add! cache (car x) (cdr x)))
                    cache-init))
      (make-memoizer proc arity cache))))

;;> Equivalent to memoize except that the procedure's first argument
;;> must be a pathname.  If the corresponding file has been modified
;;> since the memoized value, the value is recomputed.  Useful to
;;> automatically reflect external changes to a file-backed resource.
;;> The additional keyword argument \scheme{reloader?:}, if true,
;;> indicates that the result of loading is itself a procedure which
;;> should check for updates on each call.
{'fraction_non_alphanumeric': 0.16471987886653688, 'fraction_numerical': 0.004542504866969501, 'mean_word_length': 2.7913079130791307, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6775687', 'n_tokens_mistral': 2984, 'n_tokens_neox': 2816, 'n_words': 968}