<commit_before>(in-package :cepl.pipelines)
(in-readtable fn:fn-reader)

;; extract details from args and delegate to %def-gpu-function
;; for the main logic
(defmacro defun-g (name args &body body)
  (defun-g-common name args body nil))

(defmacro defun-g-equiv (name args &body body)
  (defun-g-common name args body t))

(defun defun-g-common (name args body equiv)
  "Define a function that runs on the gpu."
  ;; The code here splits and validates the arguments but the meat
  ;; of gpu function definition happens in the %def-gpu-function call
  ;; at the tail
  ;; -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
  ;; seperate any doc-string or declarations from the body
  (let ((doc-string (when (stringp (first body)) (pop body))))
    ;; split the argument list into the categoried we care aboutn
    (assoc-bind ((in-args nil) (uniforms :&uniform) (context :&context)
                 (instancing :&instancing))
        (varjo:lambda-list-split '(:&uniform :&context :&instancing) args)
      ;; check the arguments are sanely formatted
      (mapcar #'(lambda (x) (assert-arg-format name x)) in-args)
      (mapcar #'(lambda (x) (assert-arg-format name x)) uniforms)
      ;; now the meat
      (%def-gpu-function name in-args uniforms body instancing
                         doc-string equiv context))))

(defun assert-arg-format (gfunc-name x)
  (unless (listp x)
    (error 'gfun-invalid-arg-format :gfun-name gfunc-name :invalid-pair x))
  x)

;;--------------------------------------------------

(defun %def-gpu-function (name in-args uniforms body instancing
                          doc-string equiv context)
  "This is the meat of defun-g. it is broken down as follows:

   [0] makes a gpu-func-spec that will be populated a stored later.

   [1] Adds a external function definition to varjo also make sure it will be
       called on load

   [2] %test-&-update-spec compiles the code to check for errors and log
       dependencies. (this is called at runtime)

   [3] %make-gpu-func-spec is called at expand time to write a lisp function
       with the same signature as the gpu-function. This gives code hinting and
       also a decent error message if you try calling it from the cpu.
       We don't do this when using defun-g-equiv as we want to shadow the lisp
       function.

   [4] the purpose of %recompile-gpu-function-and-pipelines is to recompile and
       functions or pipelines that depend on this gpu function. It does this
       by calling %recompile-gpu-function-and-pipelines on all the gpu function
       that depend on this func and then the recompile-function for all
       pipelines that depend on this gpu function. To this end it walks depth
       first too all affected pipelines.

   [5] At runtime this looks for any gpu function that listed this function as
       one of its missing dependencies and calls %test-&-update-spec on them.
       Note that this will (possibly) update the spec but will not trigger a
       recompile in the pipelines."
  (let ((spec (%make-gpu-func-spec name in-args uniforms context body instancing
                                   nil nil nil doc-string nil
                                   nil));;[0]
        (valid-glsl-versions (get-versions-from-context context)))
    ;; this gets the functions used in the body of this function
    ;; it is *not* recursive
    (%update-gpu-function-data spec nil nil)
    (varjo:add-external-function name in-args uniforms body
                                 valid-glsl-versions);;[1]
    `(progn
       (varjo:add-external-function ',name ',in-args ',uniforms ',body
                                    ',valid-glsl-versions);;[1]
       (%test-&-update-spec ,spec);;[2]
       ,(unless equiv (make-stand-in-lisp-func spec));;[3]
       (%recompile-gpu-function-and-pipelines ,(spec->func-key spec));;[4]
       (update-specs-with-missing-dependencies);;[5]
       ',name)))

(defun get-versions-from-context (context)
  (%sort-versions
   (remove-if-not λ(member _ varjo:*supported-versions*)
                  context)))

(defun %sort-versions (versions)
  (mapcar #'first
          (sort (mapcar λ(list _ (parse-integer (symbol-name _)))
                        versions)
                #'< :key #'second)))

(defun swap-version (glsl-version context)
  (cons glsl-version (remove-if λ(find _ varjo:*supported-versions*) context)))

(defun lowest-suitable-glsl-version (context)
  (let* ((versions (or (get-versions-from-context context)
                       (list (cepl.context::get-best-glsl-version)))))
    (case= (length versions)
      (0 (cepl.context::get-best-glsl-version))
      (otherwise (first versions)))))

(defvar *warn-when-cant-test-compile* t)

(defun %test-&-update-spec (spec)
  "Use varjo to compile the code.
   [0] If the compilation throws a could-not-find-function error, then record
   that missing function's name as a missing dependency.

   [1] If it succeeds then look at the list of used external-functions, and
   check which of the names of the macros match the names of a gpu function.
   The ones that match are the dependencies.

   [2] We also record the uniforms in the compiled result. The uniforms in the
   definition are the public interface, but the compiler may have removed or
   modified the uniforms. To this end we store the final uniforms and the forms
   that transform between the public uniform arguments and the internal ones."
  (with-gpu-func-spec spec
    (handler-case
        (varjo:with-constant-inject-hook #'try-injecting-a-constant
          (varjo:with-stemcell-infer-hook #'try-guessing-a-varjo-type-for-symbol
            (handler-bind
                ((varjo-conditions:cannot-establish-exact-function
                  (lambda (c)
                    (declare (ignore c))
                    (invoke-restart
                     'varjo-conditions:allow-call-function-signature))))
              (let* ((context (swap-version (lowest-suitable-glsl-version context)
                                            context))
                     (compiled
                      (first
                       (varjo:test-translate-raising
                        (varjo:make-stage nil in-args uniforms context body t nil)
                        :stages varjo:*stage-names*))))
                (setf actual-uniforms ;;[2]
                      (mapcar #'varjo:to-arg-form
                              (remove-if #'varjo:ephemeral-p
                                         (varjo:uniform-variables compiled))))

                (%update-gpu-function-data
                 spec
                 (remove-if-not #'gpu-func-spec
                                (varjo:used-external-functions compiled)) ;;[1]
                 compiled)))))
      ;; vv- called if failed
      (varjo-conditions:could-not-find-function (e) ;;[0]
        (setf missing-dependencies (list (slot-value e 'varjo:name)))
        (when *warn-when-cant-test-compile*
          (format t "~% cepl: the function ~s was not found when compiling ~s"
                  (first missing-dependencies) name))
        (%update-gpu-function-data spec nil nil)))
    spec))




(defmethod %recompile-gpu-function-and-pipelines (key)
  (%recompile-gpu-function-and-pipelines (func-key key)))

(defmethod %recompile-gpu-function-and-pipelines ((key func-key))
  "Recompile all pipelines that depend on the named gpu function or any other
   gpu function that depends on the named gpu function. It does this by doing
   the following:

   [0] Recursively call this function on all gpu functions that use the<commit_msg>Use new varjo function for test compiling gpu functions<commit_after>(in-package :cepl.pipelines)
(in-readtable fn:fn-reader)

;; extract details from args and delegate to %def-gpu-function
;; for the main logic
(defmacro defun-g (name args &body body)
  (defun-g-common name args body nil))

(defmacro defun-g-equiv (name args &body body)
  (defun-g-common name args body t))

(defun defun-g-common (name args body equiv)
  "Define a function that runs on the gpu."
  ;; The code here splits and validates the arguments but the meat
  ;; of gpu function definition happens in the %def-gpu-function call
  ;; at the tail
  ;; -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
  ;; seperate any doc-string or declarations from the body
  (let ((doc-string (when (stringp (first body)) (pop body))))
    ;; split the argument list into the categoried we care aboutn
    (assoc-bind ((in-args nil) (uniforms :&uniform) (context :&context)
                 (instancing :&instancing))
        (varjo:lambda-list-split '(:&uniform :&context :&instancing) args)
      ;; check the arguments are sanely formatted
      (mapcar #'(lambda (x) (assert-arg-format name x)) in-args)
      (mapcar #'(lambda (x) (assert-arg-format name x)) uniforms)
      ;; now the meat
      (%def-gpu-function name in-args uniforms body instancing
                         doc-string equiv context))))

(defun assert-arg-format (gfunc-name x)
  (unless (listp x)
    (error 'gfun-invalid-arg-format :gfun-name gfunc-name :invalid-pair x))
  x)

;;--------------------------------------------------

(defun %def-gpu-function (name in-args uniforms body instancing
                          doc-string equiv context)
  "This is the meat of defun-g. it is broken down as follows:

   [0] makes a gpu-func-spec that will be populated a stored later.

   [1] Adds a external function definition to varjo also make sure it will be
       called on load

   [2] %test-&-update-spec compiles the code to check for errors and log
       dependencies. (this is called at runtime)

   [3] %make-gpu-func-spec is called at expand time to write a lisp function
       with the same signature as the gpu-function. This gives code hinting and
       also a decent error message if you try calling it from the cpu.
       We don't do this when using defun-g-equiv as we want to shadow the lisp
       function.

   [4] the purpose of %recompile-gpu-function-and-pipelines is to recompile and
       functions or pipelines that depend on this gpu function. It does this
       by calling %recompile-gpu-function-and-pipelines on all the gpu function
       that depend on this func and then the recompile-function for all
       pipelines that depend on this gpu function. To this end it walks depth
       first too all affected pipelines.

   [5] At runtime this looks for any gpu function that listed this function as
       one of its missing dependencies and calls %test-&-update-spec on them.
       Note that this will (possibly) update the spec but will not trigger a
       recompile in the pipelines."
  (let ((spec (%make-gpu-func-spec name in-args uniforms context body instancing
                                   nil nil nil doc-string nil
                                   nil));;[0]
        (valid-glsl-versions (get-versions-from-context context)))
    ;; this gets the functions used in the body of this function
    ;; it is *not* recursive
    (%update-gpu-function-data spec nil nil)
    (varjo:add-external-function name in-args uniforms body
                                 valid-glsl-versions);;[1]
    `(progn
       (varjo:add-external-function ',name ',in-args ',uniforms ',body
                                    ',valid-glsl-versions);;[1]
       (%test-&-update-spec ,spec);;[2]
       ,(unless equiv (make-stand-in-lisp-func spec));;[3]
       (%recompile-gpu-function-and-pipelines ,(spec->func-key spec));;[4]
       (update-specs-with-missing-dependencies);;[5]
       ',name)))

(defun get-versions-from-context (context)
  (%sort-versions
   (remove-if-not λ(member _ varjo:*supported-versions*)
                  context)))

(defun %sort-versions (versions)
  (mapcar #'first
          (sort (mapcar λ(list _ (parse-integer (symbol-name _)))
                        versions)
                #'< :key #'second)))

(defun swap-version (glsl-version context)
  (cons glsl-version (remove-if λ(find _ varjo:*supported-versions*) context)))

(defun lowest-suitable-glsl-version (context)
  (let* ((versions (or (get-versions-from-context context)
                       (list (cepl.context::get-best-glsl-version)))))
    (case= (length versions)
      (0 (cepl.context::get-best-glsl-version))
      (otherwise (first versions)))))

(defvar *warn-when-cant-test-compile* t)

(defun %test-&-update-spec (spec)
  "Use varjo to compile the code.
   [0] If the compilation throws a could-not-find-function error, then record
   that missing function's name as a missing dependency.

   [1] If it succeeds then look at the list of used external-functions, and
   check which of the names of the macros match the names of a gpu function.
   The ones that match are the dependencies.

   [2] We also record the uniforms in the compiled result. The uniforms in the
   definition are the public interface, but the compiler may have removed or
   modified the uniforms. To this end we store the final uniforms and the forms
   that transform between the public uniform arguments and the internal ones."
  (with-gpu-func-spec spec
    (handler-case
        (varjo:with-constant-inject-hook #'try-injecting-a-constant
          (varjo:with-stemcell-infer-hook #'try-guessing-a-varjo-type-for-symbol
            (handler-bind
                ((varjo-conditions:cannot-establish-exact-function
                  (lambda (c)
                    (declare (ignore c))
                    (invoke-restart
                     'varjo-conditions:allow-call-function-signature))))
              (let* ((context (swap-version (lowest-suitable-glsl-version context)
                                            context))
                     (compiled
                      (first
                       (varjo:test-translate-function-split-details
                        name in-args uniforms context body varjo:*stage-names* t))))
                (setf actual-uniforms ;;[2]
                      (mapcar #'varjo:to-arg-form
                              (remove-if #'varjo:ephemeral-p
                                         (varjo:uniform-variables compiled))))

                (%update-gpu-function-data
                 spec
                 (remove-if-not #'gpu-func-spec
                                (varjo:used-external-functions compiled)) ;;[1]
                 compiled)))))
      ;; vv- called if failed
      (varjo-conditions:could-not-find-function (e) ;;[0]
        (setf missing-dependencies (list (slot-value e 'varjo:name)))
        (when *warn-when-cant-test-compile*
          (format t "~% cepl: the function ~s was not found when compiling ~s"
                  (first missing-dependencies) name))
        (%update-gpu-function-data spec nil nil)))
    spec))




(defmethod %recompile-gpu-function-and-pipelines (key)
  (%recompile-gpu-function-and-pipelines (func-key key)))

(defmethod %recompile-gpu-function-and-pipelines ((key func-key))
  "Recompile all pipelines that depend on the named gpu function or any other
   gpu function that depends on the named gpu function. It does this by doing
   the following:

   [0] Recursively call this function on all gpu functions that use the
{'fraction_non_alphanumeric': 0.10975208785427763, 'fraction_numerical': 0.0027618859735648055, 'mean_word_length': 2.2295604162242513, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7568025', 'n_tokens_mistral': 4449, 'n_tokens_neox': 4216, 'n_words': 1490}