<commit_before>; Copyright (c) 2010-2015, <NAME>

; All rights reserved.

; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
; 1. Redistributions of source code must retain the above copyright
;    notice, this list of conditions and the following disclaimer.
; 2. Redistributions in binary form must reproduce the above copyright
;    notice, this list of conditions and the following disclaimer in the
;    documentation and/or other materials provided with the distribution.
; 3. The name of Mark Tarver may not be used to endorse or promote products
;    derived from this software without specific prior written permission.

; THIS SOFTWARE IS PROVIDED BY <NAME> ''AS IS'' AND ANY
; EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL <NAME> BE LIABLE FOR ANY
; DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

(PROCLAIM '(OPTIMIZE (DEBUG 0) (SPEED 3) (SAFETY 3)))
(IN-PACKAGE :CL-USER)
(SETF (READTABLE-CASE *READTABLE*) :PRESERVE)
(DEFVAR *language* "Common Lisp")
(DEFVAR *port* 2.1)
(DEFVAR *porters* "<NAME>")
(DEFCONSTANT KLAMBDA-PATH "./kernel/klambda/")

;
; Implementation-Specific Declarations
;

#+CLISP
(PROGN
  (DEFVAR *implementation* "GNU CLisp")
  (DEFVAR *release* (LET ((V (LISP-IMPLEMENTATION-VERSION))) (SUBSEQ V 0 (POSITION #\SPACE V :START 0))))
  (DEFVAR *os* (OR #+WIN32 "Windows" #+LINUX "Linux" #+MACOS "macOS" #+UNIX "Unix" "Unknown"))
  (DEFCONSTANT COMPILED-SUFFIX ".fas")
  (DEFCONSTANT NATIVE-PATH "./native/clisp/")
  (DEFCONSTANT BINARY-NAME #+WIN32 "shen.exe" #-WIN32 "shen")
  (SETQ CUSTOM:*COMPILE-WARNINGS* NIL)
  (SETQ *COMPILE-VERBOSE* NIL))

#+CCL
(PROGN
  (DEFVAR *implementation* "Clozure CL")
  (DEFVAR *release* (LISP-IMPLEMENTATION-VERSION))
  (DEFVAR *os* (OR #+WINDOWS "Windows" #+LINUX "Linux" #+DARWIN "macOS" #+UNIX "Unix" "Unknown"))
  (DEFCONSTANT COMPILED-SUFFIX (FORMAT NIL "~A" *.FASL-PATHNAME*))
  (DEFCONSTANT NATIVE-PATH "./native/ccl/")
  (DEFCONSTANT BINARY-NAME #+WINDOWS "shen.exe" #-WINDOWS "shen"))

#+ECL
(PROGN
  (DEFVAR *object-files* NIL)
  (DEFCONSTANT COMPILED-SUFFIX ".fas")
  (DEFCONSTANT OBJECT-SUFFIX ".o")
  (DEFCONSTANT NATIVE-PATH "./native/ecl/")
  (DEFCONSTANT BINARY-NAME #+(OR :WIN32 :MINGW32) "shen.exe" #-(OR :WIN32 :MINGW32) "shen")
  (EXT:INSTALL-C-COMPILER)
  (SETQ COMPILER::*COMPILE-VERBOSE* NIL)
  (SETQ COMPILER::*SUPPRESS-COMPILER-MESSAGES* NIL))

#+SBCL
(PROGN
  (DEFVAR *implementation* "SBCL")
  (DEFVAR *release* (LISP-IMPLEMENTATION-VERSION))
  (DEFVAR *os* (OR #+WIN32 "Windows" #+LINUX "Linux" #+DARWIN "macOS" #+UNIX "Unix" "Unknown"))
  (DEFCONSTANT COMPILED-SUFFIX ".fasl")
  (DEFCONSTANT NATIVE-PATH "./native/sbcl/")
  (DEFCONSTANT BINARY-NAME #+WIN32 "shen.exe" #-WIN32 "shen")
  (DECLAIM (SB-EXT:MUFFLE-CONDITIONS SB-EXT:COMPILER-NOTE))
  (SETF SB-EXT:*MUFFLED-WARNINGS* T))

;
; Shared KL-Loading Procedure
;

(DEFUN import-lsp (File)
  (LET ((LspFile (FORMAT NIL "~A.lsp" File))
        (FasFile (FORMAT NIL "~A~A~A" NATIVE-PATH File COMPILED-SUFFIX)))
    #-ECL
    (COMPILE-FILE LspFile :OUTPUT-FILE FasFile)
    #+ECL
    (LET ((ObjFile (FORMAT NIL "~A~A~A" NATIVE-PATH File OBJECT-SUFFIX)))
      (COMPILE-FILE LspFile :OUTPUT-FILE ObjFile :SYSTEM-P T)
      (PUSH ObjFile *object-files*)
      (C:BUILD-FASL FasFile :LISP-FILES (LIST ObjFile)))
    (LOAD FasFile)))

(DEFUN import-kl (File)
  (LET ((KlFile  (FORMAT NIL "~A~A.kl" KLAMBDA-PATH File))
        (LspFile (FORMAT NIL "~A~A.lsp" NATIVE-PATH File))
        (FasFile (FORMAT NIL "~A~A~A" NATIVE-PATH File COMPILED-SUFFIX)))
    (write-lsp-file LspFile (translate-kl (read-kl-file KlFile)))
    #-ECL
    (COMPILE-FILE LspFile)
    #+ECL
    (LET ((ObjFile (FORMAT NIL "~A~A~A" NATIVE-PATH File OBJECT-SUFFIX)))
      (COMPILE-FILE LspFile :SYSTEM-P T)
      (PUSH ObjFile *object-files*)
      (C:BUILD-FASL FasFile :LISP-FILES (LIST ObjFile)))
    (LOAD FasFile)))

(DEFUN read-kl-file (File)
  (WITH-OPEN-FILE
    (In File :DIRECTION :INPUT)
    (LET ((CleanedCode (clean-kl (READ-CHAR In NIL NIL) In NIL NIL)))
      (READ-FROM-STRING (FORMAT NIL "(~A)" (COERCE CleanedCode 'STRING))))))

(DEFUN clean-kl (Char In Chars InsideQuote)
  (IF (NULL Char)
    (REVERSE Chars)
    (clean-kl
      (READ-CHAR In NIL NIL)
      In
      (IF (AND (NOT InsideQuote) (MEMBER Char '(#\: #\; #\,) :TEST 'CHAR-EQUAL))
        (LIST* #\| Char #\| Chars)
        (CONS Char Chars))
      (IF (CHAR-EQUAL Char #\")
        (NOT InsideQuote)
        InsideQuote))))

(DEFUN translate-kl (KlCode)
  (MAPCAR #'(LAMBDA (X) (shen.kl-to-lisp NIL X)) KlCode))

(DEFUN write-lsp-file (File Code)
  (WITH-OPEN-FILE<commit_msg>Fixed handling of object file suffixes for ecl<commit_after>; Copyright (c) 2010-2015, <NAME>

; All rights reserved.

; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
; 1. Redistributions of source code must retain the above copyright
;    notice, this list of conditions and the following disclaimer.
; 2. Redistributions in binary form must reproduce the above copyright
;    notice, this list of conditions and the following disclaimer in the
;    documentation and/or other materials provided with the distribution.
; 3. The name of <NAME> may not be used to endorse or promote products
;    derived from this software without specific prior written permission.

; THIS SOFTWARE IS PROVIDED BY <NAME> ''AS IS'' AND ANY
; EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL <NAME> BE LIABLE FOR ANY
; DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

(PROCLAIM '(OPTIMIZE (DEBUG 0) (SPEED 3) (SAFETY 3)))
(IN-PACKAGE :CL-USER)
(SETF (READTABLE-CASE *READTABLE*) :PRESERVE)
(DEFVAR *language* "Common Lisp")
(DEFVAR *port* 2.1)
(DEFVAR *porters* "<NAME>")
(DEFCONSTANT KLAMBDA-PATH "./kernel/klambda/")

;
; Implementation-Specific Declarations
;

#+CLISP
(PROGN
  (DEFVAR *implementation* "GNU CLisp")
  (DEFVAR *release* (LET ((V (LISP-IMPLEMENTATION-VERSION))) (SUBSEQ V 0 (POSITION #\SPACE V :START 0))))
  (DEFVAR *os* (OR #+WIN32 "Windows" #+LINUX "Linux" #+MACOS "macOS" #+UNIX "Unix" "Unknown"))
  (DEFCONSTANT COMPILED-SUFFIX ".fas")
  (DEFCONSTANT NATIVE-PATH "./native/clisp/")
  (DEFCONSTANT BINARY-NAME #+WIN32 "shen.exe" #-WIN32 "shen")
  (SETQ CUSTOM:*COMPILE-WARNINGS* NIL)
  (SETQ *COMPILE-VERBOSE* NIL))

#+CCL
(PROGN
  (DEFVAR *implementation* "Clozure CL")
  (DEFVAR *release* (LISP-IMPLEMENTATION-VERSION))
  (DEFVAR *os* (OR #+WINDOWS "Windows" #+LINUX "Linux" #+DARWIN "macOS" #+UNIX "Unix" "Unknown"))
  (DEFCONSTANT COMPILED-SUFFIX (FORMAT NIL "~A" *.FASL-PATHNAME*))
  (DEFCONSTANT NATIVE-PATH "./native/ccl/")
  (DEFCONSTANT BINARY-NAME #+WINDOWS "shen.exe" #-WINDOWS "shen"))

#+ECL
(PROGN
  (DEFVAR *object-files* NIL)
  (DEFCONSTANT COMPILED-SUFFIX ".fas")
  (DEFCONSTANT OBJECT-SUFFIX #+(OR :WIN32 :MINGW32) ".obj" #-(OR :WIN32 :MINGW32) ".o")
  (DEFCONSTANT NATIVE-PATH "./native/ecl/")
  (DEFCONSTANT BINARY-NAME #+(OR :WIN32 :MINGW32) "shen.exe" #-(OR :WIN32 :MINGW32) "shen")
  (EXT:INSTALL-C-COMPILER)
  (SETQ COMPILER::*COMPILE-VERBOSE* NIL)
  (SETQ COMPILER::*SUPPRESS-COMPILER-MESSAGES* NIL))

#+SBCL
(PROGN
  (DEFVAR *implementation* "SBCL")
  (DEFVAR *release* (LISP-IMPLEMENTATION-VERSION))
  (DEFVAR *os* (OR #+WIN32 "Windows" #+LINUX "Linux" #+DARWIN "macOS" #+UNIX "Unix" "Unknown"))
  (DEFCONSTANT COMPILED-SUFFIX ".fasl")
  (DEFCONSTANT NATIVE-PATH "./native/sbcl/")
  (DEFCONSTANT BINARY-NAME #+WIN32 "shen.exe" #-WIN32 "shen")
  (DECLAIM (SB-EXT:MUFFLE-CONDITIONS SB-EXT:COMPILER-NOTE))
  (SETF SB-EXT:*MUFFLED-WARNINGS* T))

;
; Shared KL-Loading Procedure
;

(DEFUN import-lsp (File)
  (LET ((LspFile (FORMAT NIL "~A.lsp" File))
        (FasFile (FORMAT NIL "~A~A~A" NATIVE-PATH File COMPILED-SUFFIX)))
    #-ECL
    (COMPILE-FILE LspFile :OUTPUT-FILE FasFile)
    #+ECL
    (LET ((ObjFile (FORMAT NIL "~A~A~A" NATIVE-PATH File OBJECT-SUFFIX)))
      (COMPILE-FILE LspFile :OUTPUT-FILE ObjFile :SYSTEM-P T)
      (PUSH ObjFile *object-files*)
      (C:BUILD-FASL FasFile :LISP-FILES (LIST ObjFile)))
    (LOAD FasFile)))

(DEFUN import-kl (File)
  (LET ((KlFile  (FORMAT NIL "~A~A.kl" KLAMBDA-PATH File))
        (LspFile (FORMAT NIL "~A~A.lsp" NATIVE-PATH File))
        (FasFile (FORMAT NIL "~A~A~A" NATIVE-PATH File COMPILED-SUFFIX)))
    (write-lsp-file LspFile (translate-kl (read-kl-file KlFile)))
    #-ECL
    (COMPILE-FILE LspFile)
    #+ECL
    (LET ((ObjFile (FORMAT NIL "~A~A~A" NATIVE-PATH File OBJECT-SUFFIX)))
      (COMPILE-FILE LspFile :OUTPUT-FILE ObjFile :SYSTEM-P T)
      (PUSH ObjFile *object-files*)
      (C:BUILD-FASL FasFile :LISP-FILES (LIST ObjFile)))
    (LOAD FasFile)))

(DEFUN read-kl-file (File)
  (WITH-OPEN-FILE
    (In File :DIRECTION :INPUT)
    (LET ((CleanedCode (clean-kl (READ-CHAR In NIL NIL) In NIL NIL)))
      (READ-FROM-STRING (FORMAT NIL "(~A)" (COERCE CleanedCode 'STRING))))))

(DEFUN clean-kl (Char In Chars InsideQuote)
  (IF (NULL Char)
    (REVERSE Chars)
    (clean-kl
      (READ-CHAR In NIL NIL)
      In
      (IF (AND (NOT InsideQuote) (MEMBER Char '(#\: #\; #\,) :TEST 'CHAR-EQUAL))
        (LIST* #\| Char #\| Chars)
        (CONS Char Chars))
      (IF (CHAR-EQUAL Char #\")
        (NOT InsideQuote)
        InsideQuote))))

(DEFUN translate-kl (KlCode)
  (MAPCAR #'(LAMBDA (X) (shen.kl-to-lisp NIL X)) KlCode))

(DEFUN write-lsp-file (File Code)
  (WITH-OPEN-FILE
{'fraction_non_alphanumeric': 0.14504470230169297, 'fraction_numerical': 0.007989347536617843, 'mean_word_length': 4.033508855911919, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '45151', 'n_tokens_mistral': 4064, 'n_tokens_neox': 3848, 'n_words': 1158}