<commit_before>(in-package :mu-support)

;;; helpers
(defun s+ (&rest strings)
  "Concatenates a set of strings"
  (apply #'concatenate 'string "" strings))

(defun string-replace (str from to)
  "replaces <from> with <to> in <str> for every regex occurence
  of <from>."
  (cl-ppcre:regex-replace-all from str to))

(defun clean-url (url)
  "Cleans the supplied URL."
  (string-replace (string-replace url "<" "&lt;")
                  ">" "&gt;"))

(defun clean-string (string)
  "Cleans the supplied string"
  (string-replace string "\"" "\\\""))

(defun make-uuid ()
  "Creates a new UUID"
  (mongoid:oid-str (mongoid:oid)))<commit_msg>In clean-string \ should be escaped before "<commit_after>(in-package :mu-support)

;;; helpers
(defun s+ (&rest strings)
  "Concatenates a set of strings"
  (apply #'concatenate 'string "" strings))

(defun string-replace (str from to)
  "replaces <from> with <to> in <str> for every regex occurence
  of <from>."
  (cl-ppcre:regex-replace-all from str to))

(defun clean-url (url)
  "Cleans the supplied URL."
  (string-replace (string-replace url "<" "&lt;")
                  ">" "&gt;"))

(defun clean-string (string)
  "Cleans the supplied string"
  (string-replace
   (string-replace string "\\" "\\\\\\\\")
   "\"" "\\\""))

(defun make-uuid ()
  "Creates a new UUID"
  (mongoid:oid-str (mongoid:oid)))
{'fraction_non_alphanumeric': 0.17777777777777778, 'fraction_numerical': 0.0, 'mean_word_length': 3.4735099337748343, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6132017', 'n_tokens_mistral': 494, 'n_tokens_neox': 482, 'n_words': 138}