<commit_before>(in-package :med)

(defvar *repl-key-map*)
(defvar *repl-history* '())
(defvar *repl-history-number* 0)
(defvar *repl-buffer-stream* nil)

(defun start-repl ()
  (initialize-repl-key-map)
  (let ((buffer (get-buffer "*repl*")))
    (unless buffer
      (setf buffer (make-instance 'buffer
                                  :key-map *repl-key-map*)
            (buffer-property buffer 'name) "*repl*"
            (buffer-property buffer 'default-pathname-defaults)
              (or (buffer-property (current-buffer *editor*) 'default-pathname-defaults)
                  *default-pathname-defaults*)
            (last-buffer *editor*) (current-buffer *editor*))
      (setf *repl-buffer-stream* (make-instance 'buffer-stream
                                                :buffer buffer
                                                :filter #'repl-buffer-filter)))
    (setf (buffer-property buffer 'repl-prompt-end) (copy-mark (buffer-point buffer)))
    (format *repl-buffer-stream* "~%~A> " (sys.int::package-shortest-name *package*))
    (move-mark-to-mark (buffer-property buffer 'repl-prompt-end) (buffer-point buffer))
    (push buffer (buffer-list))
    (setf (last-buffer *editor*) (current-buffer *editor*))
    (switch-to-buffer buffer)))

(defun repl-buffer-filter (buffer c)
  (when (char= c #\>)
    (move-mark-to-mark (buffer-property buffer 'repl-prompt-end) (buffer-point buffer))))

(defun repl-eval (code)
  (if (string= code "")
       ""
       (let* ((buffer (get-buffer "*repl*"))
              (s *repl-buffer-stream*)
              (*standard-input* s)
              (*standard-output* s)
              (*error-output* s)
              (*trace-output* s)
              (*query-io* s)
              (*debug-io* s)
              (*default-pathname-defaults* (buffer-property buffer
                                                            'default-pathname-defaults)))
         (unwind-protect
           (progn
             (setf (buffer-key-map buffer) (make-hash-table))
             (handler-case
                 (format t "~%~S" (eval (read-from-string code)))
                 (error (e) (format t "~%~S~%" e) "")))
           (format t "~%~A> " (sys.int::package-shortest-name *package*))
           (finish-output)
           (force-redisplay))
           (setf (buffer-key-map buffer) *repl-key-map*))))

(defun repl-finish-input-command ()
  (let ((buffer (current-buffer *editor*)))
    (move-end-of-line buffer)
    ;; FIXME: clearing the buffer by cutting the text causes the 
    ;; editor to crash when you hit enter
    (let ((code (buffer-string buffer
                               (buffer-property buffer 'repl-prompt-end)
                               (buffer-point buffer))))
      (when (and (> (length code) 0)
                 (not (string= code (car *repl-history*))))
        (push code *repl-history*))
      (mezzano.supervisor::make-thread (lambda () 
                                         (repl-eval code))
                                       :name "repl"
                                       :initial-bindings `((*editor* ,*editor*)))
      (setf *repl-history-number* 0))))

(defun repl-clear-output ()
  (let ((buffer (current-buffer *editor*)))
    (move-end-of-buffer buffer)
    (with-mark (point (buffer-point buffer))
      (move-beginning-of-buffer buffer)
      (delete-region buffer point (buffer-point buffer)))
    (repl-prompt buffer)))

(defun repl-delete-input ()
  (let ((buffer (current-buffer *editor*)))
    (move-end-of-buffer buffer)
    (delete-region buffer
                   (buffer-property buffer 'input-start) 
                   (buffer-point buffer))))

(defun repl-previous-history ()
  (when (< *repl-history-number* (length *repl-history*))
    (repl-delete-input)
    (insert (current-buffer *editor*) (nth *repl-history-number* *repl-history*))<commit_msg>Fix newlines again in repl.<commit_after>(in-package :med)

(defvar *repl-key-map*)
(defvar *repl-history* '())
(defvar *repl-history-number* 0)
(defvar *repl-buffer-stream* nil)

(defun start-repl ()
  (initialize-repl-key-map)
  (let ((buffer (get-buffer "*repl*")))
    (unless buffer
      (setf buffer (make-instance 'buffer
                                  :key-map *repl-key-map*)
            (buffer-property buffer 'name) "*repl*"
            (buffer-property buffer 'default-pathname-defaults)
              (or (buffer-property (current-buffer *editor*) 'default-pathname-defaults)
                  *default-pathname-defaults*)
            (last-buffer *editor*) (current-buffer *editor*))
      (setf *repl-buffer-stream* (make-instance 'buffer-stream
                                                :buffer buffer
                                                :filter #'repl-buffer-filter)))
    (setf (buffer-property buffer 'repl-prompt-end) (copy-mark (buffer-point buffer)))
    (format *repl-buffer-stream* "~%~A> " (sys.int::package-shortest-name *package*))
    (move-mark-to-mark (buffer-property buffer 'repl-prompt-end) (buffer-point buffer))
    (push buffer (buffer-list))
    (setf (last-buffer *editor*) (current-buffer *editor*))
    (switch-to-buffer buffer)))

(defun repl-buffer-filter (buffer c)
  (when (char= c #\>)
    (move-mark-to-mark (buffer-property buffer 'repl-prompt-end) (buffer-point buffer))))

(defun repl-eval (code)
  (if (string= code "")
       ""
       (let* ((buffer (get-buffer "*repl*"))
              (s *repl-buffer-stream*)
              (*standard-input* s)
              (*standard-output* s)
              (*error-output* s)
              (*trace-output* s)
              (*query-io* s)
              (*debug-io* s)
              (*default-pathname-defaults* (buffer-property buffer
                                                            'default-pathname-defaults)))
         (unwind-protect
           (progn
             (setf (buffer-key-map buffer) (make-hash-table))
             (handler-case
                 (format t "~S" (eval (read-from-string code)))
                 (error (e) (format t "~S~%" e) "")))
           (format t "~%~A> " (sys.int::package-shortest-name *package*))
           (finish-output)
           (force-redisplay))
           (setf (buffer-key-map buffer) *repl-key-map*))))

(defun repl-finish-input-command ()
  (let ((buffer (current-buffer *editor*)))
    (move-end-of-line buffer)
    ;; FIXME: clearing the buffer by cutting the text causes the 
    ;; editor to crash when you hit enter
    (let ((code (buffer-string buffer
                               (buffer-property buffer 'repl-prompt-end)
                               (buffer-point buffer))))
      (when (and (> (length code) 0)
                 (not (string= code (car *repl-history*))))
        (push code *repl-history*))
      (format *repl-buffer-stream* "~%")
      (mezzano.supervisor::make-thread (lambda () 
                                         (repl-eval code))
                                       :name "repl"
                                       :initial-bindings `((*editor* ,*editor*)))
      (setf *repl-history-number* 0))))

(defun repl-clear-output ()
  (let ((buffer (current-buffer *editor*)))
    (move-end-of-buffer buffer)
    (with-mark (point (buffer-point buffer))
      (move-beginning-of-buffer buffer)
      (delete-region buffer point (buffer-point buffer)))
    (repl-prompt buffer)))

(defun repl-delete-input ()
  (let ((buffer (current-buffer *editor*)))
    (move-end-of-buffer buffer)
    (delete-region buffer
                   (buffer-property buffer 'input-start) 
                   (buffer-point buffer))))

(defun repl-previous-history ()
  (when (< *repl-history-number* (length *repl-history*))
    (repl-delete-input)
    (insert (current-buffer *editor*) (nth *repl-history-number* *repl-history*))
{'fraction_non_alphanumeric': 0.1546444586803331, 'fraction_numerical': 0.0007687379884689302, 'mean_word_length': 1.9246908954664668, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5242154', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2292, 'n_words': 523}