<commit_before>;;;; © 2016-2019 <NAME>         - license: GNU AGPLv3 -*- coding: utf-8 -*-

(in-package #:petalisp.native-backend)

(defun make-dummy-kernel ()
  (make-kernel :iteration-space (~) :reduction-range (range 0)))

(defmethod compute-immediates ((lazy-arrays list) (native-backend native-backend))
  (let ((root-buffers (ir-from-lazy-arrays lazy-arrays)))
    (normalize-ir root-buffers)
    (loop for root-buffer in root-buffers
          for lazy-array in lazy-arrays
          ;; We add a fictitious kernel to the outputs of each root buffer,
          ;; to avoid that their memory is reclaimed.
          do (push (make-dummy-kernel) (buffer-outputs root-buffer))
          collect
          (if (immediatep lazy-array)
              lazy-array
              (coerce-to-lazy-array
               (buffer-storage
                (compute-buffer root-buffer native-backend)))))))

(defmethod compute-buffer ((buffer buffer) (native-backend native-backend))
  (when (null (buffer-storage buffer))
    (setf (buffer-storage buffer)
          (memory-pool-allocate
           (memory-pool native-backend)
           (petalisp.type-codes:type-specifier-from-type-code
            (buffer-type-code buffer))
           (mapcar #'range-size (ranges (buffer-shape buffer)))))
    (map-buffer-inputs
     (lambda (kernel)
       (execute-kernel kernel native-backend))
     buffer))
  buffer)

(defmethod execute-kernel :before ((kernel kernel) (native-backend native-backend))
  (map-kernel-inputs
   (lambda (buffer)
     (compute-buffer buffer native-backend))
   kernel))<commit_msg>Clear the native backend's memory pool after each run.<commit_after>;;;; © 2016-2019 <NAME>         - license: GNU AGPLv3 -*- coding: utf-8 -*-

(in-package #:petalisp.native-backend)

(defun make-dummy-kernel ()
  (make-kernel :iteration-space (~) :reduction-range (range 0)))

(defmethod compute-immediates ((lazy-arrays list) (native-backend native-backend))
  (let ((root-buffers (ir-from-lazy-arrays lazy-arrays)))
    (normalize-ir root-buffers)
    (loop for root-buffer in root-buffers
          for lazy-array in lazy-arrays
          ;; We add a fictitious kernel to the outputs of each root buffer,
          ;; to avoid that their memory is reclaimed.
          do (push (make-dummy-kernel) (buffer-outputs root-buffer))
          collect
          (if (immediatep lazy-array)
              lazy-array
              (coerce-to-lazy-array
               (buffer-storage
                (compute-buffer root-buffer native-backend)))))))

(defmethod compute-immediates :after ((lazy-arrays list) (native-backend native-backend))
  (memory-pool-reset
   (memory-pool native-backend)))

(defmethod compute-buffer ((buffer buffer) (native-backend native-backend))
  (when (null (buffer-storage buffer))
    (setf (buffer-storage buffer)
          (memory-pool-allocate
           (memory-pool native-backend)
           (petalisp.type-codes:type-specifier-from-type-code
            (buffer-type-code buffer))
           (mapcar #'range-size (ranges (buffer-shape buffer)))))
    (map-buffer-inputs
     (lambda (kernel)
       (execute-kernel kernel native-backend))
     buffer))
  buffer)

(defmethod execute-kernel :before ((kernel kernel) (native-backend native-backend))
  (map-kernel-inputs
   (lambda (buffer)
     (compute-buffer buffer native-backend))
   kernel))
{'fraction_non_alphanumeric': 0.1289653127779425, 'fraction_numerical': 0.006522383634746516, 'mean_word_length': 2.691466083150985, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3932726', 'n_tokens_mistral': 1021, 'n_tokens_neox': 978, 'n_words': 255}