<commit_before>; feed - copies data from an input file (or stdin) to some subsequent
; REPL and then turns interaction with said REPL over to the user. as
; a special case "lisp" requires no external program
;
; TODO needs to implement more of expect(1) (look for different shell
; prompts or error conditions, throwing away leading REPL noise, etc)
; and offer a means of customization without recompiling, e.g. to use
; READ when piping to an external CL process, etc

#-quicklisp
(let ((quicklisp-init (merge-pathnames "quicklisp/setup.lisp"
                                       (user-homedir-pathname))))
  (when (probe-file quicklisp-init)
    (load quicklisp-init)))

(ql:quickload :cl-ppcre)

(defpackage :feedsalot (:use :cl :sb-ext))
(in-package :feedsalot)

(defun file-as-stream (input)
  "Open a stream or - as standard input."
  (handler-case
      (if (string= input "-") *standard-input* (open input))
    (file-does-not-exist (c)
      (format *error-output* "feed: no such file: ~s~&" input)
      (exit :code 1))))

; on the possibly wild assumption that this is available and works
(defun user-stream ()
  "Open /dev/tty to interact with the user."
  (open "/dev/tty" :direction :io
        :if-exists :append :if-does-not-exist :error))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;
; INTERNAL LISP INTERACTION

;   #-SBCL 'where-is-your-repl-now
;
; a prompt re-paint might be nice with the above, instead of the "huh?
; ... oh, I see" confusion (unix shells may show a continuation prompt
; when a command is left open over multiple lines)
(defun lisp-interact (stream)
  (loop
    (handler-case
        (progn
          (format t "~&SBCL> ")
          (finish-output)
          (format t "~&~s" (eval (read stream t))))
      (end-of-file () (exit :code 0))
      (sb-sys:interactive-interrupt () (exit :code 130))
      (error (c) (format t "~&ERR ~s" c))
      (condition (c) (format t "~&COND ~s" c)))))

(defun lisp-replay (stream)
  "Eval all the LISP objects in a stream."
  (handler-case
      (loop for object = (read stream t) do
            (format t "~&SBCL> ~s" object)
            (format t "~&~s" (eval object)))
    (end-of-file ())
    (sb-sys:interactive-interrupt () (exit :code 130))
    (error (c) (format t "~&ERR ~s" c))))

(defun internal-lisp (input &aux (stream (file-as-stream input)))
  "Process input with LISP."
  (cond ((interactive-stream-p stream)        ; "-" attached to a TTY
          (lisp-interact stream))
        (t (lisp-replay stream)               ; "-" pipe or a file
           (lisp-interact (user-stream)))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;
; EXTERNAL COMMAND

(defmacro match (string regex)
  `(cl-ppcre:do-matches (s e ,regex ,string) (return t)))

; NOTE non-UTF-8 inputs may cause sb-int:stream-decoding-error; if that
; happens more often will need to be able to change the encoding system
; or maybe to simply copy the bytes around like some C caveman
;   HERE 256 - 256 TYPE ( isn't the best of memory viewers in FORTH )
;   : PEEK SWAP 0 DO DUP I + C@ . LOOP DROP ;
;   256 HERE 256 - PEEK ( is less bad )
(defparameter *buf*
  (make-array 4096 :adjustable t :element-type 'extended-char
              :fill-pointer 0)
  "Buffer for external program PTY output reads.")

(defun empty-buf ()
  "Empty the *buf* buffer."
  (setf (fill-pointer *buf*) 0))

(defun drain-buf ()
  "Print and empty *buf*."
  (declare (inline empty-buf))
  (princ *buf*)
  (finish-output)
  (empty-buf))

(defun match-on (string)
  "Generate a function that looks for string in successive characters."<commit_msg>simplify the match macro<commit_after>; feed - copies data from an input file (or stdin) to some subsequent
; REPL and then turns interaction with said REPL over to the user. as
; a special case "lisp" requires no external program
;
; TODO needs to implement more of expect(1) (look for different shell
; prompts or error conditions, throwing away leading REPL noise, etc)
; and offer a means of customization without recompiling, e.g. to use
; READ when piping to an external CL process, etc

#-quicklisp
(let ((quicklisp-init (merge-pathnames "quicklisp/setup.lisp"
                                       (user-homedir-pathname))))
  (when (probe-file quicklisp-init)
    (load quicklisp-init)))

(ql:quickload :cl-ppcre)

(defpackage :feedsalot (:use :cl :sb-ext))
(in-package :feedsalot)

(defun file-as-stream (input)
  "Open a stream or - as standard input."
  (handler-case
      (if (string= input "-") *standard-input* (open input))
    (file-does-not-exist (c)
      (format *error-output* "feed: no such file: ~s~&" input)
      (exit :code 1))))

; on the possibly wild assumption that this is available and works
(defun user-stream ()
  "Open /dev/tty to interact with the user."
  (open "/dev/tty" :direction :io
        :if-exists :append :if-does-not-exist :error))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;
; INTERNAL LISP INTERACTION

;   #-SBCL 'where-is-your-repl-now
;
; a prompt re-paint might be nice with the above, instead of the "huh?
; ... oh, I see" confusion (unix shells may show a continuation prompt
; when a command is left open over multiple lines)
(defun lisp-interact (stream)
  (loop
    (handler-case
        (progn
          (format t "~&SBCL> ")
          (finish-output)
          (format t "~&~s" (eval (read stream t))))
      (end-of-file () (exit :code 0))
      (sb-sys:interactive-interrupt () (exit :code 130))
      (error (c) (format t "~&ERR ~s" c))
      (condition (c) (format t "~&COND ~s" c)))))

(defun lisp-replay (stream)
  "Eval all the LISP objects in a stream."
  (handler-case
      (loop for object = (read stream t) do
            (format t "~&SBCL> ~s" object)
            (format t "~&~s" (eval object)))
    (end-of-file ())
    (sb-sys:interactive-interrupt () (exit :code 130))
    (error (c) (format t "~&ERR ~s" c))))

(defun internal-lisp (input &aux (stream (file-as-stream input)))
  "Process input with LISP."
  (cond ((interactive-stream-p stream)        ; "-" attached to a TTY
          (lisp-interact stream))
        (t (lisp-replay stream)               ; "-" pipe or a file
           (lisp-interact (user-stream)))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;
; EXTERNAL COMMAND

; NOTE 0-is-true on a first position match, otherwise NIL
(defmacro match (string regex)
  `(cl-ppcre:scan ,regex ,string))

; NOTE non-UTF-8 inputs may cause sb-int:stream-decoding-error; if that
; happens more often will need to be able to change the encoding system
; or maybe to simply copy the bytes around like some C caveman
;   HERE 256 - 256 TYPE ( isn't the best of memory viewers in FORTH )
;   : PEEK SWAP 0 DO DUP I + C@ . LOOP DROP ;
;   256 HERE 256 - PEEK ( is less bad )
(defparameter *buf*
  (make-array 4096 :adjustable t :element-type 'extended-char
              :fill-pointer 0)
  "Buffer for external program PTY output reads.")

(defun empty-buf ()
  "Empty the *buf* buffer."
  (setf (fill-pointer *buf*) 0))

(defun drain-buf ()
  "Print and empty *buf*."
  (declare (inline empty-buf))
  (princ *buf*)
  (finish-output)
  (empty-buf))

(defun match-on (string)
  "Generate a function that looks for string in successive characters."
{'fraction_non_alphanumeric': 0.16516639388979815, 'fraction_numerical': 0.008046917621385706, 'mean_word_length': 3.112731351654515, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5471843', 'n_tokens_mistral': 2551, 'n_tokens_neox': 2370, 'n_words': 861}