<commit_before>(load "./package.lisp")
(load "./toolsbox.lisp")
(in-package #:anomaly-detection)


;; Can use toolsbox:points-average
(defun standard-deviation (numlist)
  "return the value include the average and standard deviation of the numlist"
  (let* ((sum (apply #'+ numlist))
         (len (length numlist))
         (aver (/ sum len))
         (SD (sqrt (loop for i in numlist sum
                       (expt (- i aver) 2)))))
    (list aver SD)))




(defun anomaly-detection (matrix)
  "as usual, the input var is matrix include every attributions of sample and get the average and the standard deviation"
  (let* ((matrixDim (array-dimensions matrix))
         (rowNum (nth 0 matrixDim))
         (colNum (nth 1 matrixDim))
         (result '()))
    (do* ((colIndex 0 (1+ colIndex))
          (thisCol (loop for i from 0 to (1- rowNum) collect
                        (aref matrix i colIndex))
                   (loop for i from 0 to (1- rowNum) collect
                        (aref matrix i colIndex))))
         ((= colIndex (1- colNum))
          (aappend result (standard-deviation thisCol)))  ;last time
      ;(print thisCol) (print colIndex) (print (standard-deviation thisCol))
      (aappend result (standard-deviation thisCol)))
    result))
      
<commit_msg>add new tool marco<commit_after>(load "./package.lisp")
(load "./toolsbox.lisp")
(in-package #:anomaly-detection)


(defun standard-deviation (numlist)
  "return the value include the average and standard deviation of the numlist"
  (let* ((sum (apply #'+ numlist))
         (len (length numlist))
         (aver (/ sum len))
         (SD (sqrt (loop for i in numlist sum
                       (expt (- i aver) 2)))))
    (list aver SD)))

(defmacro array-slice-col (m i)
  (with-gensyms (mm ii)
    `(let* ((,mm ,m)
            (,ii ,i)
            (dimens (array-dimensions ,mm))
            (rowNum (nth 0 dimens)))
       (loop for i from 0 to (1- rowNum) collect
            (aref ,mm i ,ii)))))


(defun anomaly-detection (matrix)
  "as usual, the input var is matrix include every attributions of sample and get the average and the standard deviation"
  (let* ((matrixDim (array-dimensions matrix))
         (colNum (nth 1 matrixDim))
         (result '()))
    (do* ((colIndex 0 (1+ colIndex))
          (thisCol (array-slice-col matrix colIndex)
                   (array-slice-col matrix colIndex)))
         ((= colIndex (1- colNum))
          (aappend result (standard-deviation thisCol)))  ;last time
      ;(print thisCol) (print colIndex) (print (standard-deviation thisCol))
      (aappend result (standard-deviation thisCol)))
    result))
      

{'fraction_non_alphanumeric': 0.12745098039215685, 'fraction_numerical': 0.006787330316742082, 'mean_word_length': 2.388250319284802, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5410778', 'n_tokens_mistral': 827, 'n_tokens_neox': 787, 'n_words': 284}