<commit_before>;;; Currently testing!

(defvar ems-node-buffer nil)
(defvar ems-node-process nil)
(defvar ems-output-buffer nil)
(defvar ems-request-counter 0)

(add-hook
 'inferior-js-mode-hook
      #'(lambda ()
        (add-to-list
         'comint-preoutput-filter-functions
         #'(lambda (output)
           (string-match (format "BEGINOUTPUT" ems-request-counter) output))
           (replace-regexp-in-string "[.]*BEGINOUTPUT[0-9]+" "" output)
           (replace-regexp-in-string "ENDOUTPUT[.]*" "" output))))

(defun ems-load-node ()
  (run-js inferior-js-program-command t)
  (setq ems-node-buffer (get-buffer "*js*"))
  (setq ems-node-process (get-buffer-process "*js*"))
  (let ((output-buffer (get-buffer "*ems-output*")))
    (if output-buffer
        (setq ems-output-buffer output-buffer)
      (setq ems-output-buffer (generate-new-buffer "*ems-output*"))
      )))

(defun ems-load-mathjax ()
  (comint-redirect-send-command-to-process
   "var mjx = require('mathjax-node');" ems-output-buffer ems-node-buffer t))

(defun ems-load-sre ()
  (comint-redirect-send-command-to-process
   "var sre = require('speech-rule-engine');" ems-output-buffer ems-node-buffer t))


;; Input: LaTeX expression
(defun ems-start-walker (expr)
  (comint-redirect-send-command-to-process
   (format "mjx.typeset({math: '%s', format: 'TeX', mml:true}, function(data) {sre.walk(data.mml)});" expr)
   ems-output-buffer ems-node-buffer t))

;; Input: A key value
(defun ems-move-walker (key)
  (let* ((counter (incf ems-request-counter))
         (command (format "console.log('BEGINOUTPUT%d:' + sre.move(%d) + 'ENDOUTPUT%d');"
             counter key counter)))
    (comint-redirect-send-command-to-process
     command
     ems-output-buffer ems-node-buffer t)
    (ems-parse-output-buffer counter)))


;; TODO Error handling after each step.
(defun ems-setup-bridge ()
  (ems-load-node)
  (ems-load-mathjax)
  (ems-load-sre))

(defun ems-teardown-bridge ()
  (kill-buffer ems-output-buffer)
  (setf ems-output-buffer nil)
  (setf ems-node-buffer nil))


;; This does not work!
;; Number.
;; Return: The relevant output in the buffer.
(defun ems-parse-output-buffer (counter)
  (accept-process-output ems-node-process 1)
  (with-current-buffer ems-output-buffer
    (let* ((str (buffer-string))
           (start (string-match (format "BEGINOUTPUT%d" counter) str))
           (end (string-match (format "ENDOUTPUT%d" counter) str)))
      (if (and start end) (subseq str (+ start 12) end))
      )))


;;; API
(defun ems-start ()
  (ems-setup-bridge))

(defun ems-enter (expr)
  (ems-start-walker expr)
  (accept-process-output (get-buffer-process "*js*") 1)
  ;; Need to wait for callback!
  (ems-repeat 2)
  )

(defun ems-up ()
  (ems-move-walker 38))

(defun ems-down ()
  (ems-move-walker 40))

(defun ems-left ()
  (ems-move-walker 37))

(defun ems-right ()
  (ems-move-walker 39))

(defun ems-repeat ()
  (ems-move-walker 9))

(defun ems-depth ()
  (ems-move-walker 32))
<commit_msg>Buffer communication works.<commit_after>;;; Currently testing!

(defvar ems-node-buffer nil)
(defvar ems-node-process nil)
(defvar ems-output-buffer nil)
(defvar ems-request-counter 0)
(defvar ems-results ())


(add-hook
 'inferior-js-mode-hook
      #'(lambda ()
        (add-to-list
         'comint-preoutput-filter-functions
         #'(lambda (output)
             (with-current-buffer (get-buffer "*ems-output*")
               (let ((result (ems-parse-output output)))
                 (when result 
                   (insert result)
                   (insert "\n"))
                 output))))))


(defun ems-load-node ()
  (run-js inferior-js-program-command t)
  (setq ems-node-buffer (get-buffer "*js*"))
  (setq ems-node-process (get-buffer-process "*js*"))
  (let ((output-buffer (get-buffer "*ems-output*")))
    (if output-buffer
        (setq ems-output-buffer output-buffer)
      (setq ems-output-buffer (generate-new-buffer "*ems-output*"))
      )))

(defun ems-load-setup ()
  (process-send-string 
   ems-node-process
   (concat 
    "var mjx = require('mathjax-node');" 
    "var sre = require('speech-rule-engine');"
    "var runWithCounter = function(counter, callback, args) {"
    "  var result = callback.apply(sre, args);"
    "  console.log('BEGINOUTPUT' + counter + ': ' + result + ' :ENDOUTPUT');"
    "};\n"
    )))


(defun ems-next-counter ()
  (incf ems-request-counter))

;; Input: LaTeX expression
(defun ems-start-walker (expr)
  (process-send-string
   ems-node-process
   ;TODO: Here we can get LaTeX errors.
   (concat 
    (format "mjx.typeset({math: '%s', format: 'TeX', mml:true}, " expr)
    (format "function(data) {sre.walk(data.mml)});\n"))))


;; Input: A key value
(defun ems-move-walker (key)
  (process-send-string
   ems-node-process
   (format "runWithCounter(%d, sre.move, [%d]);\n" (incf ems-request-counter) key)))


;; TODO Error handling after each step.
(defun ems-setup-bridge ()
  (ems-load-node)
  (ems-load-setup))

(defun ems-teardown-bridge ()
  (kill-buffer ems-output-buffer)
  (setf ems-output-buffer nil)
  (setf ems-node-buffer nil))


;; Number: The full buffer output.
;; Return: Cons with call number and result.
(defun ems-parse-output (output)
  (let* ((start (string-match "BEGINOUTPUT[0-9]+: .* :ENDOUTPUT" output)))
    (when start
      (let* ((rest (subseq output (+ start 11)))
             (colon (string-match ": " rest))
             (number (subseq rest 0 colon))
             (rest (subseq rest (+ colon 2)))
             (end (string-match " :ENDOUTPUT" rest))
             (result (subseq rest 0 end)))
        (print result)
        (print number)
        (push (cons (car (read-from-string number)) result) ems-results)
        result))))


;;; API
(defun ems-start ()
  (ems-setup-bridge))

(defun ems-enter (expr)
  (ems-start-walker expr)
  (accept-process-output (get-buffer-process "*js*") 1)
  ;; Need to wait for callback!
  (ems-repeat 2)
  )

(defun ems-up ()
  (ems-move-walker 38))

(defun ems-down ()
  (ems-move-walker 40))

(defun ems-left ()
  (ems-move-walker 37))

(defun ems-right ()
  (ems-move-walker 39))

(defun ems-repeat ()
  (ems-move-walker 9))

(defun ems-depth ()
  (ems-move-walker 32))

{'fraction_non_alphanumeric': 0.17295293359123146, 'fraction_numerical': 0.006447453255963894, 'mean_word_length': 2.992921492921493, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1000830', 'n_tokens_mistral': 2321, 'n_tokens_neox': 2220, 'n_words': 495}