<commit_before>;;
(defpackage cim
  (:use :CL)
  (:export :*argv*
           :*interrupt-condition*
           :getenv
           :parse-options
           :opt
           :exit))
(in-package :cim)
(defvar *raw-argv*
  #+allegro  (cdr (system:command-line-arguments))
  #+sbcl (do*  ((var sb-ext:*posix-argv* (cdr list))
                (list var var))
               ((string= (car list) "--") (return (cdr list))))
  #+clisp ext:*args*
  #+ecl (do*  ((var (si:command-args) (cdr list))
               (list var var))
              ((string= (car list) "--") (return (cdr list))))
  #+abcl extensions:*command-line-argument-list*
  #+gcl (do*  ((var si::*command-args* (cdr list))
               (list var var))
              ((string= (car list) "--") (return (cdr list))))
  #+cmu ext:*command-line-application-arguments*
  #+ccl ccl:*unprocessed-command-line-arguments*
  #+lispworks system:*line-arguments-list*)
(defvar *argv*)

(defun getenv (name &optional default)
  #+cmu
  (let ((x (assoc name ext:*environment-list*
                  :test #'string=)))
    (if x (cdr x) default))
  #-cmu
  (or
   #+abcl (java:jstatic "getenv" "java.lang.System" name)
   #+allegro (sys:getenv name)
   #+clisp (ext:getenv name)
   #+ccl (ccl:getenv name)
   #+ECL (si:getenv name)
   #+gcl (si::getenv name)
   #+sbcl (sb-unix::posix-getenv name)
   #+lispworks (lispworks:environment-variable name)
   default))

(defun exit (&optional (status 0))
 #+ccl (ccl:quit status)
 ;; http://www.sbcl.org/manual/#Exit
 #+sbcl (sb-ext:exit :code status)
 ;; http://franz.com/support/documentation/9.0/doc/operators/excl/exit.htm
 #+allegro (excl:exit status :quiet t)
 #+clisp (ext:quit status)
 #+cmu (unix:unix-exit status)
 #+ecl (ext:quit status)
 #-(or ccl sbcl allegro clisp cmu ecl) (cl-user::quit))

(defun cim_home (path)
  (concatenate 'string (getenv "CIM_HOME") path))
(defun ql_home (path)
  (concatenate 'string (cim_home "/quicklisp") path))

(defvar *help*)
(defvar *options* (make-hash-table))

#+(or allegro ccl clisp ecl sbcl)
(defvar *interrupt-condition*
  ;; It seems abcl does not raise any conditions
  #+allegro 'excl:interrupt-signal
  #+ccl 'ccl:interrupt-signal-condition
  #+clisp 'system::simple-interrupt-condition
  #+ecl 'ext:interactive-interrupt
  #+sbcl 'sb-sys:interactive-interrupt)

#+cmu
(system:enable-interrupt Unix:sigint (lambda () (exit 1)))


(setf *load-verbose* nil)

(let ((v))
  (defun version ()
    (or v
        (setf v
              (with-open-file (in (cim_home "/VERSION"))
                (read-line in))))))

(defun opt (key)
  (gethash key *options*))
(defsetf opt (key) (obj)
  `(setf (gethash ,key *options*) ,obj))

(defun remove-shebang (in)
  (let ((line (read-line in nil "#!")))
    (cond
      ((and (> (length line) 1) (string= line "#!" :end1 2))
       in)
      (t
       (make-concatenated-stream (make-string-input-stream (format nil "~A~%" line)) in)))))

(defun read-stream-into-string (stream)
  (let* ((buffer-size 4096)
         (buffer (make-array buffer-size :element-type 'character)))
    (with-output-to-string (str)
     (loop
        :for bytes-read = (read-sequence buffer stream)
        :do (write-sequence buffer str :start 0 :end bytes-read)
        :while (= bytes-read buffer-size)))))

(defun canonicalize-path (path)
  (if (char= #\/ (aref path (1- (length path))))
      path
      (concatenate 'string path "/")))

(defun ensure-quicklisp ()
  (unless (member :quicklisp *features*)
    (let ((quicklisp-init (ql_home "/setup.lisp")))
      (when (probe-file quicklisp-init)
        (load quicklisp-init :verbose nil)))))

(load (cim_home "/lib/option-parser.lisp")  :verbose nil :print nil)

(let ((sexps))
  (setf *argv*
        (parse-options *raw-argv*
                       (("-c" "--compile") (file)
                        "compile FILE."
                        (push `(compile-file ,file) sexps))
                       (("-C") (dir)
                        "set *default-pathname-defaults* DIR."
                        (let ((dir (if (char= #\/ (elt dir 1 ))
                                       (pathname dir)
                                       (merge-pathnames (pathname dir)))))
                          (push `(setf *default-pathname-defaults* ,dir) sexps)))
                       (("-d" "--debug") ()
                        "set debugging flags (push :debug into *features*)"
                        (push '(push :debug *features*) sexps))
                       (("-e" "--eval") (sexp)
                        "one line of script. Several -e's are allowed. Omit [programfile]"
                        (setf (opt :sexp) (concatenate 'string (opt :sexp) sexp)))
                       (("-f" "--load") (file)
                        "load the FILE"
                        (push `(load ,file) sexps))
                       (("-i") (ext)
                        "edit *argv* files in place and make backup with the extension .EXT"
                        (push `(setf (opt :extension) ,ext) sexps))
                       (("-l") (library)
                        "quickload the LIBRARY"
                        (push `(progn
                                 (ensure-quicklisp)
                                 (funcall (intern "QUICKLOAD" :ql) ,(intern (format nil "~:@(~A~)" library) :keyword))) sexps))
                       (("-L") (library)
                        "quickload and use-package the LIBRARY"
                        (let ((sys (intern (format nil "~:@(~A~)" library) :keyword)))
                          (push `(progn
                                   (ensure-quicklisp)
                                   (funcall (intern "QUICKLOAD" :ql) ,sys)
                                   (use-package  ,sys))<commit_msg>canonicalize dirname<commit_after>;;
(defpackage cim
  (:use :CL)
  (:export :*argv*
           :*interrupt-condition*
           :getenv
           :parse-options
           :opt
           :exit))
(in-package :cim)
(defvar *raw-argv*
  #+allegro  (cdr (system:command-line-arguments))
  #+sbcl (do*  ((var sb-ext:*posix-argv* (cdr list))
                (list var var))
               ((string= (car list) "--") (return (cdr list))))
  #+clisp ext:*args*
  #+ecl (do*  ((var (si:command-args) (cdr list))
               (list var var))
              ((string= (car list) "--") (return (cdr list))))
  #+abcl extensions:*command-line-argument-list*
  #+gcl (do*  ((var si::*command-args* (cdr list))
               (list var var))
              ((string= (car list) "--") (return (cdr list))))
  #+cmu ext:*command-line-application-arguments*
  #+ccl ccl:*unprocessed-command-line-arguments*
  #+lispworks system:*line-arguments-list*)
(defvar *argv*)

(defun getenv (name &optional default)
  #+cmu
  (let ((x (assoc name ext:*environment-list*
                  :test #'string=)))
    (if x (cdr x) default))
  #-cmu
  (or
   #+abcl (java:jstatic "getenv" "java.lang.System" name)
   #+allegro (sys:getenv name)
   #+clisp (ext:getenv name)
   #+ccl (ccl:getenv name)
   #+ECL (si:getenv name)
   #+gcl (si::getenv name)
   #+sbcl (sb-unix::posix-getenv name)
   #+lispworks (lispworks:environment-variable name)
   default))

(defun exit (&optional (status 0))
 #+ccl (ccl:quit status)
 ;; http://www.sbcl.org/manual/#Exit
 #+sbcl (sb-ext:exit :code status)
 ;; http://franz.com/support/documentation/9.0/doc/operators/excl/exit.htm
 #+allegro (excl:exit status :quiet t)
 #+clisp (ext:quit status)
 #+cmu (unix:unix-exit status)
 #+ecl (ext:quit status)
 #-(or ccl sbcl allegro clisp cmu ecl) (cl-user::quit))

(defun cim_home (path)
  (concatenate 'string (getenv "CIM_HOME") path))
(defun ql_home (path)
  (concatenate 'string (cim_home "/quicklisp") path))

(defvar *help*)
(defvar *options* (make-hash-table))

#+(or allegro ccl clisp ecl sbcl)
(defvar *interrupt-condition*
  ;; It seems abcl does not raise any conditions
  #+allegro 'excl:interrupt-signal
  #+ccl 'ccl:interrupt-signal-condition
  #+clisp 'system::simple-interrupt-condition
  #+ecl 'ext:interactive-interrupt
  #+sbcl 'sb-sys:interactive-interrupt)

#+cmu
(system:enable-interrupt Unix:sigint (lambda () (exit 1)))


(setf *load-verbose* nil)

(let ((v))
  (defun version ()
    (or v
        (setf v
              (with-open-file (in (cim_home "/VERSION"))
                (read-line in))))))

(defun opt (key)
  (gethash key *options*))
(defsetf opt (key) (obj)
  `(setf (gethash ,key *options*) ,obj))

(defun remove-shebang (in)
  (let ((line (read-line in nil "#!")))
    (cond
      ((and (> (length line) 1) (string= line "#!" :end1 2))
       in)
      (t
       (make-concatenated-stream (make-string-input-stream (format nil "~A~%" line)) in)))))

(defun read-stream-into-string (stream)
  (let* ((buffer-size 4096)
         (buffer (make-array buffer-size :element-type 'character)))
    (with-output-to-string (str)
     (loop
        :for bytes-read = (read-sequence buffer stream)
        :do (write-sequence buffer str :start 0 :end bytes-read)
        :while (= bytes-read buffer-size)))))

(defun canonicalize-path (path)
  (if (char= #\/ (aref path (1- (length path))))
      path
      (concatenate 'string path "/")))

(defun ensure-quicklisp ()
  (unless (member :quicklisp *features*)
    (let ((quicklisp-init (ql_home "/setup.lisp")))
      (when (probe-file quicklisp-init)
        (load quicklisp-init :verbose nil)))))

(load (cim_home "/lib/option-parser.lisp")  :verbose nil :print nil)

(let ((sexps))
  (setf *argv*
        (parse-options *raw-argv*
                       (("-c" "--compile") (file)
                        "compile FILE."
                        (push `(compile-file ,file) sexps))
                       (("-C") (dir)
                        "set *default-pathname-defaults* DIR."
                        (let* ((dir (canonicalize-path dir))
                               (dir (if (char= #\/ (elt dir 1 ))
                                       (pathname dir)
                                       (merge-pathnames (pathname dir)))))
                          (push `(setf *default-pathname-defaults* ,dir) sexps)))
                       (("-d" "--debug") ()
                        "set debugging flags (push :debug into *features*)"
                        (push '(push :debug *features*) sexps))
                       (("-e" "--eval") (sexp)
                        "one line of script. Several -e's are allowed. Omit [programfile]"
                        (setf (opt :sexp) (concatenate 'string (opt :sexp) sexp)))
                       (("-f" "--load") (file)
                        "load the FILE"
                        (push `(load ,file) sexps))
                       (("-i") (ext)
                        "edit *argv* files in place and make backup with the extension .EXT"
                        (push `(setf (opt :extension) ,ext) sexps))
                       (("-l") (library)
                        "quickload the LIBRARY"
                        (push `(progn
                                 (ensure-quicklisp)
                                 (funcall (intern "QUICKLOAD" :ql) ,(intern (format nil "~:@(~A~)" library) :keyword))) sexps))
                       (("-L") (library)
                        "quickload and use-package the LIBRARY"
                        (let ((sys (intern (format nil "~:@(~A~)" library) :keyword)))
                          (push `(progn
                                   (ensure-quicklisp)
                                   (funcall (intern "QUICKLOAD" :ql) ,sys)
                                   (use-package  ,sys))
{'fraction_non_alphanumeric': 0.16353725015142337, 'fraction_numerical': 0.0024227740763173833, 'mean_word_length': 1.9238552997723248, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4069210', 'n_tokens_mistral': 3869, 'n_tokens_neox': 3728, 'n_words': 971}