<commit_before>(defpackage #:test-grid (:use :cl))

(in-package #:test-grid)

(defgeneric libtest (library-name)
  (:documentation "Define a method for this function
with LIBRARY-NAME eql-specialized for for every library added 
to the test grid.

The method should run test suite and return the resulting
status. Status is one of three values: 
:OK - all tests passed,
:FAIL - some test failed,
:NO-RESOURCE - test suite can not be run because some required 
resource is absent in the environment. For example, CFFI library
test suite needs a small C library compiled to DLL. User must
do it manually. In case the DLL is absent, the LIBTEST method
for CFFI returns :NO-RESOURCE.

For convenience, T may be returned instead of :OK and NIL instead of :FAIL."))

(defun normalize-status (status)
  "Normilzies test resul status - converts T to :OK and NIL to :FAIL."
  (case status
    ((t :ok) :ok)
    ((nil :fail) :fail)
    (otherwise status)))

(defparameter *all-libs* '(:alexandria :babel :trivial-features :cffi 
                           :cl-ppcre :usocket :flexi-streams :bordeaux-threads
                           :cl-base64 :trivial-backtrace :puri :anaphora
                           :parenscript :trivial-garbage :iterate :metabang-bind
                           :cl-json :cl-containers :metatilities-base :cl-cont
                           :moptilities :trivial-timeout :metatilities)
  "All the libraries currently supported by the test-grid.")

(defun clean-rt ()
  "Helper function to assist running test suites created using the RT 
test framework. The problem is that RT uses global storage for all
the tests; in result if we previously loaded any test system,
after loading another test system the global test RT test suite
contains the tests of _both_ libraries."
  (let ((rem-all-tests (and (find-package '#:rt)
                            (find-symbol (symbol-name '#:rem-all-tests) '#:rt))))
    (when rem-all-tests (funcall rem-all-tests))))
  
(defmethod libtest ((library-name (eql :alexandria)))

  ;; The test framework used: rt.
  (clean-rt)
  (asdf:clear-system :alexandria-tests)

  (quicklisp:quickload :alexandria-tests)

  (flet (
         ;; the run-tests local function is copy/pasted 
         ;; from alexandria-tests.asd         
         (run-tests (&rest args)
           (apply (intern (string '#:run-tests) '#:alexandria-tests) args)))

    (let ((a (run-tests :compiled nil))
          (b (run-tests :compiled t)))
      (and a b))))

(defmethod libtest ((library-name (eql :babel)))
  
  ;; The test framework used: stefil.
  
  (quicklisp:quickload :babel-tests)

  (let* ((*debug-io* *standard-output*)
         (result (funcall (intern (string '#:run) '#:babel-tests)))
         (failures (funcall (intern (string '#:failure-descriptions-of) '#:hu.dwim.stefil)
                            result))
         (failed-p (> (length failures) 0)))

    (when failed-p
      (format t "~&~%-------------------------------------------------------------------------------------------------~%")
      (format t "The test result details (printed by cl-test-grid in addition to the standard Stefil output above)~%")
      (format t "~%(DESCRIBE RESULT):~%")
      (describe result)
      (format t "~&~%Individual failures:~%")
      (map 'nil
           (lambda (descr) (format t "~A~%" descr))
           failures))
    (not failed-p)))

(defmethod libtest ((library-name (eql :trivial-features)))
  
  ;; The test framework used: rt.
  (clean-rt)
  (asdf:clear-system :trivial-features-tests)

  (quicklisp:quickload :trivial-features-tests)
  
  ;; copy/past from trivial-features-tests.asd
  (let ((*package* (find-package 'trivial-features-tests)))
    (funcall (find-symbol (symbol-name '#:do-tests)))))

(defmethod libtest ((library-name (eql :cffi)))

  ;; The test framework used: rt.
  (clean-rt)
  (asdf:clear-system :cffi-tests)

  ;; CFFI tests work with a small test C library.
  ;; The user is expected to compile the library manually.
  ;; If the library is not available, CFFI tests
  ;; signal cffi:load-foreign-library-error.
  (handler-bind ((error #'(lambda (condition)
                            ;; Check if the error is a
                            ;; cffi:load-foreign-library-error.
                            ;; Take into account that it might
                            ;; be some other error which prevents<commit_msg>closes #2 - load cffi-grovel explicitly before loading trivial-features-tests<commit_after>(defpackage #:test-grid (:use :cl))

(in-package #:test-grid)

(defgeneric libtest (library-name)
  (:documentation "Define a method for this function
with LIBRARY-NAME eql-specialized for for every library added 
to the test grid.

The method should run test suite and return the resulting
status. Status is one of three values: 
:OK - all tests passed,
:FAIL - some test failed,
:NO-RESOURCE - test suite can not be run because some required 
resource is absent in the environment. For example, CFFI library
test suite needs a small C library compiled to DLL. User must
do it manually. In case the DLL is absent, the LIBTEST method
for CFFI returns :NO-RESOURCE.

For convenience, T may be returned instead of :OK and NIL instead of :FAIL."))

(defun normalize-status (status)
  "Normilzies test resul status - converts T to :OK and NIL to :FAIL."
  (case status
    ((t :ok) :ok)
    ((nil :fail) :fail)
    (otherwise status)))

(defparameter *all-libs* '(:alexandria :babel :trivial-features :cffi 
                           :cl-ppcre :usocket :flexi-streams :bordeaux-threads
                           :cl-base64 :trivial-backtrace :puri :anaphora
                           :parenscript :trivial-garbage :iterate :metabang-bind
                           :cl-json :cl-containers :metatilities-base :cl-cont
                           :moptilities :trivial-timeout :metatilities)
  "All the libraries currently supported by the test-grid.")

(defun clean-rt ()
  "Helper function to assist running test suites created using the RT 
test framework. The problem is that RT uses global storage for all
the tests; in result if we previously loaded any test system,
after loading another test system the global test RT test suite
contains the tests of _both_ libraries."
  (let ((rem-all-tests (and (find-package '#:rt)
                            (find-symbol (symbol-name '#:rem-all-tests) '#:rt))))
    (when rem-all-tests (funcall rem-all-tests))))
  
(defmethod libtest ((library-name (eql :alexandria)))

  ;; The test framework used: rt.
  (clean-rt)
  (asdf:clear-system :alexandria-tests)

  (quicklisp:quickload :alexandria-tests)

  (flet (
         ;; the run-tests local function is copy/pasted 
         ;; from alexandria-tests.asd         
         (run-tests (&rest args)
           (apply (intern (string '#:run-tests) '#:alexandria-tests) args)))

    (let ((a (run-tests :compiled nil))
          (b (run-tests :compiled t)))
      (and a b))))

(defmethod libtest ((library-name (eql :babel)))
  
  ;; The test framework used: stefil.
  
  (quicklisp:quickload :babel-tests)

  (let* ((*debug-io* *standard-output*)
         (result (funcall (intern (string '#:run) '#:babel-tests)))
         (failures (funcall (intern (string '#:failure-descriptions-of) '#:hu.dwim.stefil)
                            result))
         (failed-p (> (length failures) 0)))

    (when failed-p
      (format t "~&~%-------------------------------------------------------------------------------------------------~%")
      (format t "The test result details (printed by cl-test-grid in addition to the standard Stefil output above)~%")
      (format t "~%(DESCRIBE RESULT):~%")
      (describe result)
      (format t "~&~%Individual failures:~%")
      (map 'nil
           (lambda (descr) (format t "~A~%" descr))
           failures))
    (not failed-p)))

(defmethod libtest ((library-name (eql :trivial-features)))
  
  ;; The test framework used: rt.
  (clean-rt)
  (asdf:clear-system :trivial-features-tests)

  ;; Load cffi-grovel which is used in trivial-features-tests.asd,
  ;; but not in the asdf:defsystem macro (issue #2). 
  (quicklisp:quickload :cffi-grovel)

  (quicklisp:quickload :trivial-features-tests)
  
  ;; copy/past from trivial-features-tests.asd
  (let ((*package* (find-package 'trivial-features-tests)))
    (funcall (find-symbol (symbol-name '#:do-tests)))))

(defmethod libtest ((library-name (eql :cffi)))

  ;; The test framework used: rt.
  (clean-rt)
  (asdf:clear-system :cffi-tests)

  ;; CFFI tests work with a small test C library.
  ;; The user is expected to compile the library manually.
  ;; If the library is not available, CFFI tests
  ;; signal cffi:load-foreign-library-error.
  (handler-bind ((error #'(lambda (condition)
                            ;; Check if the error is a
                            ;; cffi:load-foreign-library-error.
                            ;; Take into account that it might
                            ;; be some other error which prevents
{'fraction_non_alphanumeric': 0.13875013875013875, 'fraction_numerical': 0.000888000888000888, 'mean_word_length': 2.5058365758754864, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '454630', 'n_tokens_mistral': 2790, 'n_tokens_neox': 2637, 'n_words': 919}