<commit_before>;;;; -*- mode:lisp;coding:utf-8 -*-
;;;;**************************************************************************
;;;;FILE:               manifest.lisp
;;;;LANGUAGE:           Common-Lisp
;;;;SYSTEM:             Common-Lisp
;;;;USER-INTERFACE:     NONE
;;;;DESCRIPTION
;;;;
;;;;    Check the licenses of the dependencies, write a manifest.
;;;;
;;;;AUTHORS
;;;;    <PJB> <NAME> <<EMAIL>>
;;;;MODIFICATIONS
;;;;    2012-03-14 <PJB> Extracted from generate-cli.lisp
;;;;    2014-02-21 <PJB> Exported print-manifest.
;;;;BUGS
;;;;LEGAL
;;;;    AGPL3
;;;;
;;;;    Copyright <NAME> 2012 - 2016
;;;;
;;;;    This program is free software: you can redistribute it and/or modify
;;;;    it under the terms of the GNU Affero General Public License as published by
;;;;    the Free Software Foundation, either version 3 of the License, or
;;;;    (at your option) any later version.
;;;;
;;;;    This program is distributed in the hope that it will be useful,
;;;;    but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;;    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;;;    GNU Affero General Public License for more details.
;;;;
;;;;    You should have received a copy of the GNU Affero General Public License
;;;;    along with this program.  If not, see http://www.gnu.org/licenses/
;;;;**************************************************************************
(eval-when (:compile-toplevel :load-toplevel :execute)
  (setf *readtable* (copy-readtable nil)))
(in-package "COMMON-LISP-USER")
(declaim (declaration also-use-packages))
(declaim (also-use-packages "ASDF"))
(defpackage "COM.INFORMATIMAGO.TOOLS.MANIFEST"
  (:use "COMMON-LISP"
        "COM.INFORMATIMAGO.COMMON-LISP.CESARUM.STRING"
        "COM.INFORMATIMAGO.COMMON-LISP.CESARUM.UTILITY"
        "COM.INFORMATIMAGO.COMMON-LISP.CESARUM.VERSION"
        "COM.INFORMATIMAGO.COMMON-LISP.CESARUM.FILE"
        "SPLIT-SEQUENCE")
  (:export "ASDF-SYSTEM-NAME"
           "ASDF-SYSTEM-LICENSE"
           "SYSTEM-DEPENDS-ON"
           "SYSTEM-DEPENDS-ON/RECURSIVE"
           "LISP-IMPLEMENTATION-TYPE-KEYWORD"
           "VERSION"
           "MACHINE-TYPE-KEYWORD"
           "DISTRIBUTION"
           "EXECUTABLE-NAME"
           "EXECUTABLE-FILENAME"
           "PRINT-MANIFEST"
           "WRITE-MANIFEST"
           "PRINT-BUG-REPORT-INFO"))
(in-package "COM.INFORMATIMAGO.TOOLS.MANIFEST")


(defparameter *system-licenses*
  '(("cl-ppcre"       . "BSD-2")
    ("split-sequence" . :unknown)
    ("terminfo"       . "MIT")
    ("closer-mop"     . "MIT")))

(defun asdf-system-name (system)
  (slot-value system 'asdf::name))

(defun asdf-system-license (system-name)
  (let ((system  (asdf:find-system system-name)))
    (or (cdr (assoc system-name *system-licenses* :test 'string-equal))
        (and (slot-boundp system 'asdf::licence)
             (slot-value system 'asdf::licence))
        :unknown)))

#+mocl
(defun system-depends-on (system)
  '())

#-mocl
(defun system-depends-on (system)
  (delete (string-downcase system)
          (delete-duplicates
           (mapcar (lambda (dependency)
                     (etypecase dependency
                       (asdf:system (asdf:component-name dependency))
                       (string      dependency)
                       (symbol      (string-downcase dependency))
                       (cons        ; (:version "asdf" "3.1.2")
                        (second dependency))))
                   (asdf:system-depends-on (asdf:find-system system)))
           :test 'string=)
          :test 'string=))

(defun system-depends-on/recursive (system)
  (delete-duplicates
   (transitive-closure
    (function system-depends-on)
    (list (string-downcase system)))
   :test 'string=))


;; kuiper                Linux kuiper 2.6.38-gentoo-r6-pjb-c9 #2 SMP Wed Jul 13 00:23:08 CEST 2011 x86_64 Intel(R) Core(TM) i7 CPU 950 @ 3.07GHz GenuineIntel GNU/Linux
;; hubble                Linux hubble 2.6.34-gentoo-r1-d3 #5 SMP PREEMPT Mon Sep 6 13:17:41 CEST 2010 i686 QEMU Virtual CPU version 0.13.0 GenuineIntel GNU/Linux
;; voyager               Linux voyager.informatimago.com 2.6.18-6-k7 #1 SMP Mon Oct 13 16:52:47 UTC 2008 i686 GNU/Linux
;; triton   10.5.8       Darwin triton.lan.informatimago.com 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc PowerBook6,8 Darwin
;; neuron   10.5.8       Darwin neuron.intergruas.com 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:55:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_I386 i386
;; galatea               Darwin galatea.lan.informatimago.com 11.3.0 Darwin Kernel Version 11.3.0: Thu Jan 12 18:48:32 PST 2012; root:xnu-1699.24.23~1/RELEASE_I386 i386
;; galatea  10.7.5       Darwin galatea.lan.informatimago.com 11.4.2 Darwin Kernel Version 11.4.2: Thu Aug 23 16:26:45 PDT 2012; root:xnu-1699.32.7~1/RELEASE_I386 i386
;; despina  10.8         Darwin despina.home 12.5.0 Darwin Kernel Version 12.5.0: Sun Sep 29 13:33:47 PDT 2013; root:xnu-2050.48.12~1/RELEASE_X86_64 x86_64
;; larissa  10.9.2       Darwin larissa.home 13.1.0 Darwin Kernel Version 13.1.0: Thu Jan 16 19:40:37 PST 2014; root:xnu-2422.90.20~2/RELEASE_X86_64 x86_64 i386 MacBookAir6,2 Darwin



(defun shell-command-to-string (command &rest arguments)
  "Execute the COMMAND with asdf:run-shell-command and returns its
stdout in a string (going thru a file)."
  (let ((*default-pathname-defaults* #P"")
        (path (format nil "~:@(out-~36,8,'0R.txt~)" (random (expt 2 32)))))
    (unwind-protect
         (when (zerop (asdf:run-shell-command (format nil "~? > ~S" command arguments path)))
           (with-output-to-string (out)
             (with-open-file (file path)
               (loop
                 :for line = (read-line file nil nil)
                 :while line :do (write-line line out)))))
      (ignore-errors (delete-file path)))))



(defun prepare-options (options)
  (mapcar (lambda (option)<commit_msg>shell-command-to-string: Store temporary file in TMP: or in (USER-HOMEDIR-PATHNAME).<commit_after>;;;; -*- mode:lisp;coding:utf-8 -*-
;;;;**************************************************************************
;;;;FILE:               manifest.lisp
;;;;LANGUAGE:           Common-Lisp
;;;;SYSTEM:             Common-Lisp
;;;;USER-INTERFACE:     NONE
;;;;DESCRIPTION
;;;;
;;;;    Check the licenses of the dependencies, write a manifest.
;;;;
;;;;AUTHORS
;;;;    <PJB> <NAME> <<EMAIL>>
;;;;MODIFICATIONS
;;;;    2012-03-14 <PJB> Extracted from generate-cli.lisp
;;;;    2014-02-21 <PJB> Exported print-manifest.
;;;;BUGS
;;;;LEGAL
;;;;    AGPL3
;;;;
;;;;    Copyright <NAME> 2012 - 2016
;;;;
;;;;    This program is free software: you can redistribute it and/or modify
;;;;    it under the terms of the GNU Affero General Public License as published by
;;;;    the Free Software Foundation, either version 3 of the License, or
;;;;    (at your option) any later version.
;;;;
;;;;    This program is distributed in the hope that it will be useful,
;;;;    but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;;    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;;;    GNU Affero General Public License for more details.
;;;;
;;;;    You should have received a copy of the GNU Affero General Public License
;;;;    along with this program.  If not, see http://www.gnu.org/licenses/
;;;;**************************************************************************
(eval-when (:compile-toplevel :load-toplevel :execute)
  (setf *readtable* (copy-readtable nil)))
(in-package "COMMON-LISP-USER")
(declaim (declaration also-use-packages))
(declaim (also-use-packages "ASDF"))
(defpackage "COM.INFORMATIMAGO.TOOLS.MANIFEST"
  (:use "COMMON-LISP"
        "COM.INFORMATIMAGO.COMMON-LISP.CESARUM.STRING"
        "COM.INFORMATIMAGO.COMMON-LISP.CESARUM.UTILITY"
        "COM.INFORMATIMAGO.COMMON-LISP.CESARUM.VERSION"
        "COM.INFORMATIMAGO.COMMON-LISP.CESARUM.FILE"
        "SPLIT-SEQUENCE")
  (:export "ASDF-SYSTEM-NAME"
           "ASDF-SYSTEM-LICENSE"
           "SYSTEM-DEPENDS-ON"
           "SYSTEM-DEPENDS-ON/RECURSIVE"
           "LISP-IMPLEMENTATION-TYPE-KEYWORD"
           "VERSION"
           "MACHINE-TYPE-KEYWORD"
           "DISTRIBUTION"
           "EXECUTABLE-NAME"
           "EXECUTABLE-FILENAME"
           "PRINT-MANIFEST"
           "WRITE-MANIFEST"
           "PRINT-BUG-REPORT-INFO"))
(in-package "COM.INFORMATIMAGO.TOOLS.MANIFEST")


(defparameter *system-licenses*
  '(("cl-ppcre"       . "BSD-2")
    ("split-sequence" . :unknown)
    ("terminfo"       . "MIT")
    ("closer-mop"     . "MIT")))

(defun asdf-system-name (system)
  (slot-value system 'asdf::name))

(defun asdf-system-license (system-name)
  (let ((system  (asdf:find-system system-name)))
    (or (cdr (assoc system-name *system-licenses* :test 'string-equal))
        (and (slot-boundp system 'asdf::licence)
             (slot-value system 'asdf::licence))
        :unknown)))

#+mocl
(defun system-depends-on (system)
  '())

#-mocl
(defun system-depends-on (system)
  (delete (string-downcase system)
          (delete-duplicates
           (mapcar (lambda (dependency)
                     (etypecase dependency
                       (asdf:system (asdf:component-name dependency))
                       (string      dependency)
                       (symbol      (string-downcase dependency))
                       (cons        ; (:version "asdf" "3.1.2")
                        (second dependency))))
                   (asdf:system-depends-on (asdf:find-system system)))
           :test 'string=)
          :test 'string=))

(defun system-depends-on/recursive (system)
  (delete-duplicates
   (transitive-closure
    (function system-depends-on)
    (list (string-downcase system)))
   :test 'string=))


;; kuiper                Linux kuiper 2.6.38-gentoo-r6-pjb-c9 #2 SMP Wed Jul 13 00:23:08 CEST 2011 x86_64 Intel(R) Core(TM) i7 CPU 950 @ 3.07GHz GenuineIntel GNU/Linux
;; hubble                Linux hubble 2.6.34-gentoo-r1-d3 #5 SMP PREEMPT Mon Sep 6 13:17:41 CEST 2010 i686 QEMU Virtual CPU version 0.13.0 GenuineIntel GNU/Linux
;; voyager               Linux voyager.informatimago.com 2.6.18-6-k7 #1 SMP Mon Oct 13 16:52:47 UTC 2008 i686 GNU/Linux
;; triton   10.5.8       Darwin triton.lan.informatimago.com 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc PowerBook6,8 Darwin
;; neuron   10.5.8       Darwin neuron.intergruas.com 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:55:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_I386 i386
;; galatea               Darwin galatea.lan.informatimago.com 11.3.0 Darwin Kernel Version 11.3.0: Thu Jan 12 18:48:32 PST 2012; root:xnu-1699.24.23~1/RELEASE_I386 i386
;; galatea  10.7.5       Darwin galatea.lan.informatimago.com 11.4.2 Darwin Kernel Version 11.4.2: Thu Aug 23 16:26:45 PDT 2012; root:xnu-1699.32.7~1/RELEASE_I386 i386
;; despina  10.8         Darwin despina.home 12.5.0 Darwin Kernel Version 12.5.0: Sun Sep 29 13:33:47 PDT 2013; root:xnu-2050.48.12~1/RELEASE_X86_64 x86_64
;; larissa  10.9.2       Darwin larissa.home 13.1.0 Darwin Kernel Version 13.1.0: Thu Jan 16 19:40:37 PST 2014; root:xnu-2422.90.20~2/RELEASE_X86_64 x86_64 i386 MacBookAir6,2 Darwin



(defun shell-command-to-string (command &rest arguments)
  "Execute the COMMAND with asdf:run-shell-command and returns its
stdout in a string (going thru a file)."
  (let* ((*default-pathname-defaults* #P"")
         (name (format nil "~:@(OUT-~36,8,'0R~)" (random (expt 2 32))))
         (path (namestring (translate-logical-pathname
                            (if (ignore-errors (logical-pathname-translations "TMP"))
                                (make-pathname :host "TMP" :directory '(:absolute)
                                               :name name :type "TXT")
                                (merge-pathnames (user-homedir-pathname)
                                                 (make-pathname :name name :type "TXT")))))))
    (unwind-protect
         (when (zerop (asdf:run-shell-command (format nil "~? > ~S" command arguments path)))
           (with-output-to-string (out)
             (with-open-file (file path)
               (loop
                 :for line = (read-line file nil nil)
                 :while line :do (write-line line out)))))
      (ignore-errors (delete-file path)))))


(defun prepare-options (options)
  (mapcar (lambda (option)
{'fraction_non_alphanumeric': 0.15908908668110014, 'fraction_numerical': 0.05484724560981477, 'mean_word_length': 2.642523364485981, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1160570', 'n_tokens_mistral': 4684, 'n_tokens_neox': 4053, 'n_words': 1040}