<commit_before>(in-package :varjo)
(in-readtable fn:fn-reader)

(defun rolling-translate (stages &optional (compile-func #'translate))
  (labels ((valid-for-rt (x)
             (and (typep x 'stage) (not (typep x 'multi-stage)))))
    (assert (every #'valid-for-rt stages)
            () 'rolling-translate-invalid-stage
            :invalid (remove-if #'valid-for-rt stages)))
  ;;
  (let ((result (reduce λ(compile-stage _ _1 compile-func) stages
                        :initial-value (make-instance 'rolling-result))))
    (reverse (slot-value result 'compiled-stages))))

(defun compile-stage (accum stage compile-func)
  (with-slots (remaining-stages compiled-stages) accum
    (let* ((last-stage (first compiled-stages))
           (remaining-stages (check-order (extract-stage-type stage)
                                          remaining-stages)))
      (if (typep stage 'varjo-compile-result)
          (splice-in-precompiled-stage
           last-stage stage remaining-stages accum )
          (let* ((merged-stage (merge-in-previous-stage-args last-stage stage))
                 (new-compile-result (funcall compile-func merged-stage)))
            (make-instance 'rolling-result
                           :compiled-stages (cons new-compile-result
                                                  compiled-stages)
                           :remaining-stages remaining-stages))))))

(defun merge-in-previous-stage-args (previous-stage stage)
  (declare (optimize debug))
  (labels ((merge-in-arg (previous current)
             (make-instance
              'input-variable<commit_msg>remove old reference to multi-stage<commit_after>(in-package :varjo)
(in-readtable fn:fn-reader)

(defun rolling-translate (stages &optional (compile-func #'translate))
  (labels ((valid-for-rt (x) (typep x 'stage)))
    (assert (every #'valid-for-rt stages)
            () 'rolling-translate-invalid-stage
            :invalid (remove-if #'valid-for-rt stages)))
  ;;
  (let ((result (reduce λ(compile-stage _ _1 compile-func) stages
                        :initial-value (make-instance 'rolling-result))))
    (reverse (slot-value result 'compiled-stages))))

(defun compile-stage (accum stage compile-func)
  (with-slots (remaining-stages compiled-stages) accum
    (let* ((last-stage (first compiled-stages))
           (remaining-stages (check-order (extract-stage-type stage)
                                          remaining-stages)))
      (if (typep stage 'varjo-compile-result)
          (splice-in-precompiled-stage
           last-stage stage remaining-stages accum )
          (let* ((merged-stage (merge-in-previous-stage-args last-stage stage))
                 (new-compile-result (funcall compile-func merged-stage)))
            (make-instance 'rolling-result
                           :compiled-stages (cons new-compile-result
                                                  compiled-stages)
                           :remaining-stages remaining-stages))))))

(defun merge-in-previous-stage-args (previous-stage stage)
  (declare (optimize debug))
  (labels ((merge-in-arg (previous current)
             (make-instance
              'input-variable
{'fraction_non_alphanumeric': 0.12625944584382873, 'fraction_numerical': 0.0006297229219143577, 'mean_word_length': 2.1486620416253714, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6622419', 'n_tokens_mistral': 889, 'n_tokens_neox': 870, 'n_words': 200}