<commit_before>;; -*- lisp -*-

(in-package :ucw)

;;;; * Compiling TAL from .tal files

(defclass tal-generator () ())

(defgeneric load-tal (generator name))

(defgeneric template-truename (generator name))


(defclass file-system-generator (tal-generator)
  ((root-directories :initarg :root-directories :type list
		     :accessor root-directories)))

(defclass caching-file-system-generator (file-system-generator)
  ((cache :accessor cache :initform (make-hash-table :test 'equal))))

(defmethod shared-initialize :after ((fsg file-system-generator)
				     slot-names
				     &key cachep &allow-other-keys)
  (when cachep
    (setf (cache fsg) (make-hash-table :test 'equal))))


(defparameter *tal-templates* (make-hash-table :test 'equal))

(defmethod template-truename ((generator file-system-generator) (name string))
  (template-truename generator (pathname name)))

(defmethod template-truename ((generator file-system-generator) (name pathname))
  (aif (probe-file name)
       it
       (dolist (root (root-directories generator))
	 (when-bind truename (probe-file (merge-pathnames name root))
	   (return-from template-truename truename)))))

(defmethod load-tal ((generator file-system-generator) (name string))
  (load-tal generator (pathname name)))

(defmethod load-tal ((generator file-system-generator) (name pathname))<commit_msg>Introducing and using find-file-in-directories that finds in the set of roots, and guards against shenanigans.<commit_after>;; -*- lisp -*-

(in-package :ucw)

;;;; * Compiling TAL from .tal files

(defclass tal-generator () ())

(defgeneric load-tal (generator name))

(defgeneric template-truename (generator name))


(defclass file-system-generator (tal-generator)
  ((root-directories :initarg :root-directories :type list
		     :accessor root-directories)))

(defclass caching-file-system-generator (file-system-generator)
  ((cache :accessor cache :initform (make-hash-table :test 'equal))))

(defmethod shared-initialize :after ((fsg file-system-generator)
				     slot-names
				     &key cachep &allow-other-keys)
  (when cachep
    (setf (cache fsg) (make-hash-table :test 'equal))))


(defparameter *tal-templates* (make-hash-table :test 'equal))

(defmethod template-truename ((generator file-system-generator) name)
  (find-file-in-directories name (root-directories generator)))

(defmethod load-tal ((generator file-system-generator) (name string))
  (load-tal generator (pathname name)))

(defmethod load-tal ((generator file-system-generator) (name pathname))
{'fraction_non_alphanumeric': 0.1405448085274378, 'fraction_numerical': 0.0, 'mean_word_length': 4.569230769230769, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1146801', 'n_tokens_mistral': 810, 'n_tokens_neox': 778, 'n_words': 209}