<commit_before>(in-package #:config-checker)

(define-condition config-check-failure (error) ())

(defun each-concept-has-a-directory (config)
  (let* ((concepts (mapcar #'(lambda (h) (gethash "slug" h)) (gethash "concepts" config)))
         (bad-concepts
           (remove-if #'(lambda (c) (probe-file (merge-pathnames c "./concepts/"))) concepts)))
    (when bad-concepts
      (dolist (c bad-concepts)
        (format *error-output* "Concept: ~S does not have a directory~&" c))
      (error 'config-check-failure))))

(defun exercise-concepts-are-in-concept-list (config)
  (let* ((exercises (gethash "exercises" config))
         (concepts (sort (copy-seq (mapcar #'(lambda (h) (gethash "slug" h))
                                           (gethash "concepts" config)))
                         #'string<))
         (concept-exercises  (gethash "concept" exercises))
         (practice-exercises (gethash "practice" exercises))

         (exercise-concepts
           (delete-duplicates
            (sort
             (append (apply #'append (mapcar #'(lambda (h) (gethash "concepts" h)) concept-exercises))
                     (apply #'append (mapcar #'(lambda (h) (gethash "practices" h)) practice-exercises))
                     (apply #'append (mapcar #'(lambda (h) (gethash "prerequisites" h)) practice-exercises)))
             #'string<)
            :test #'string=)))

    (let ((unlisted-concepts (set-difference exercise-concepts concepts :test #'string=)))
      (when unlisted-concepts
        (format *error-output* "Concepts from exercises: ~S not in the concept list~&"
                unlisted-concepts)
        (error 'config-check-failure)))))

(defparameter *checkers* (list #'each-concept-has-a-directory
                               #'exercise-concepts-are-in-concept-list))

(defun check-config ()
  (let ((config (yason:parse (truename "./config.json"))))
    (dolist (checker *checkers*)
      (funcall checker config))))

(defun ci-check-config ()
  "Version of CHECK-CONFIG for use in CI processes. Will exit with an error code."
  (handler-case (check-config)
    (error () (uiop:quit -1))))
<commit_msg>Add checker to ensure concept directories are for concepts in list.<commit_after>(in-package #:config-checker)

(define-condition config-check-failure (error) ())

(defun each-concept-has-a-directory (config)
  (let* ((concepts (mapcar #'(lambda (h) (gethash "slug" h)) (gethash "concepts" config)))
         (bad-concepts
           (remove-if #'(lambda (c) (probe-file (merge-pathnames c "./concepts/"))) concepts)))
    (when bad-concepts
      (dolist (c bad-concepts)
        (format *error-output* "Concept: ~S does not have a directory~&" c))
      (error 'config-check-failure))))

(defun each-concept-directory-is-a-concept (config)
  (let ((concepts (sort (copy-seq (mapcar #'(lambda (h) (gethash "slug" h))
                                          (gethash "concepts" config)))
                        #'string<))
        (directories (sort (apply #'append
                                  (mapcar #'last
                                          (mapcar #'pathname-directory
                                                  (directory "./concepts/*/"))))
                           #'string<)))
    (let ((orphan-directories (set-difference directories concepts :test #'string=)))
      (when orphan-directories
        (format *error-output* "Concept Directories: ~S are not in concept list~&"
                orphan-directories)
        (error 'config-check-failure)))))

(defun exercise-concepts-are-in-concept-list (config)
  (let* ((exercises (gethash "exercises" config))
         (concepts (sort (copy-seq (mapcar #'(lambda (h) (gethash "slug" h))
                                           (gethash "concepts" config)))
                         #'string<))
         (concept-exercises  (gethash "concept" exercises))
         (practice-exercises (gethash "practice" exercises))

         (exercise-concepts
           (delete-duplicates
            (sort
             (append (apply #'append (mapcar #'(lambda (h) (gethash "concepts" h)) concept-exercises))
                     (apply #'append (mapcar #'(lambda (h) (gethash "practices" h)) practice-exercises))
                     (apply #'append (mapcar #'(lambda (h) (gethash "prerequisites" h)) practice-exercises)))
             #'string<)
            :test #'string=)))

    (let ((unlisted-concepts (set-difference exercise-concepts concepts :test #'string=)))
      (when unlisted-concepts
        (format *error-output* "Concepts from exercises: ~S not in the concept list~&"
                unlisted-concepts)
        (error 'config-check-failure)))))

(defparameter *checkers* (list #'each-concept-has-a-directory
                               #'each-concept-directory-is-a-concept
                               #'exercise-concepts-are-in-concept-list))

(defun check-config ()
  (let ((config (yason:parse (truename "./config.json"))))
    (dolist (checker *checkers*)
      (funcall checker config))))

(defun ci-check-config ()
  "Version of CHECK-CONFIG for use in CI processes. Will exit with an error code."
  (handler-case (check-config)
    (error () (uiop:quit -1))))

{'fraction_non_alphanumeric': 0.14861751152073732, 'fraction_numerical': 0.00038402457757296467, 'mean_word_length': 2.3031071655041218, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5189582', 'n_tokens_mistral': 1625, 'n_tokens_neox': 1552, 'n_words': 401}