<commit_before>(in-package :com.gigamonkeys.prose-diff)

(defvar *interned-text* (make-hash-table :test #'equal))

(defparameter *token-scanner* (cl-ppcre:create-scanner "(\\w+|\\s+|[\\W\\S])"))

(defclass propertied-text ()
  ((text :initarg :text :accessor text)
   (properties :initarg :properties :accessor properties)))

(defun clear-interned-text ()
  (setf *interned-text* (make-hash-table :test #'equal)))

(defun intern-text (text &optional properties)
  (let ((key (cons text properties)))
    (let ((existing (gethash key *interned-text*)))
      (or
       existing
       (setf
        (gethash key *interned-text*)
        (make-instance 'propertied-text
          :text text 
          :properties properties))))))

(defun add-property (text new-prop)
  (intern-text (text text) (cons new-prop (properties text))))

(defun has-property-p (prop text)
  (member prop (properties text)))

(defmethod print-object ((object propertied-text) stream)
  (when *print-readably*
    (error 'print-not-readable object))
  (format stream "\"~a\"~@[[~{~(~a~)~^:~}]~]" (text object) (properties object)))

(defun textify-markup (markup-list)
  "Convert a list of Markup sexps into a vector of interned
propertied-text objects."
  (let ((v (make-array 10 :adjustable t :fill-pointer 0)))
    (loop for markup in markup-list do (vector-push-extend* (%textify-markup markup ()) v))
    v))
<commit_msg>Adding REMOVE-PROPERTIES function<commit_after>(in-package :com.gigamonkeys.prose-diff)

(defvar *interned-text* (make-hash-table :test #'equal))

(defparameter *token-scanner* (cl-ppcre:create-scanner "(\\w+|\\s+|[\\W\\S])"))

(defclass propertied-text ()
  ((text :initarg :text :accessor text)
   (properties :initarg :properties :accessor properties)))

(defun clear-interned-text ()
  (setf *interned-text* (make-hash-table :test #'equal)))

(defun intern-text (text &optional properties)
  (let ((key (cons text properties)))
    (let ((existing (gethash key *interned-text*)))
      (or
       existing
       (setf
        (gethash key *interned-text*)
        (make-instance 'propertied-text
          :text text 
          :properties properties))))))

(defun add-property (text new-prop)
  (intern-text (text text) (cons new-prop (properties text))))

(defun remove-properties (text props)
  (intern-text (text text) (set-difference (properties text) props)))

(defun has-property-p (prop text)
  (member prop (properties text)))

(defmethod print-object ((object propertied-text) stream)
  (when *print-readably*
    (error 'print-not-readable object))
  (format stream "\"~a\"~@[[~{~(~a~)~^:~}]~]" (text object) (properties object)))

(defun textify-markup (markup-list)
  "Convert a list of Markup sexps into a vector of interned
propertied-text objects."
  (let ((v (make-array 10 :adjustable t :fill-pointer 0)))
    (loop for markup in markup-list do (vector-push-extend* (%textify-markup markup ()) v))
    v))

{'fraction_non_alphanumeric': 0.16797541823147832, 'fraction_numerical': 0.0020484807101399795, 'mean_word_length': 3.9409780775716694, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3647933', 'n_tokens_mistral': 1014, 'n_tokens_neox': 987, 'n_words': 269}