<commit_before>;(require :org.ancar.CLUnit "CLUnit")
;(require :com.rrette.finenight "finenight")

(load "CLUnit.lisp")

(load "finenight")

;(in-package :cl-user)

;(import 'org.ancar.CLUnit::deftest)

(in-package :com.rrette.finenight.iadfa)

(defparameter *my-iadfa* (debug-gen-iadfa '("addendice"
					    "append" "appendice"
					    "bappend" "bappendice"
					    "cappend" "cappendice"
					    "mormont")))

(defmacro test-equivalence (words)
  (with-syms (w iadfa output)
    `(let* ((,w ,words)
            (,iadfa (debug-gen-iadfa ,w))
	    (,output nil))
       (setf ,output (extract-words (iadfa-fsa ,iadfa)))
       (format t "input:~%~S~%output:~%~S~%" ,w ,output)
       (equal ,w ,output))))

(defun detect-problems (words)
  (let ((iadfa (build-iadfa))
	(words-to-be-checked nil))
    (dolist (word words)
      (setf words-to-be-checked (nconc words-to-be-checked (list word)))
      (handle-word iadfa (concatenate 'list word))
      (when (not (equal words-to-be-checked 
			(extract-words (iadfa-fsa iadfa))))
	(return)))
    ;; We got the first entry that trigger the problem.
    ;; we need now to see which entry is needed to start
    ;; the problem
    words-to-be-checked))


(defun detect-first-starting-problematic-word (words-to-be-checked)
  (let ((wtbc (cdr words-to-be-checked))
	(last-word (car words-to-be-checked)))
    (do ((iadfa (gen-iadfa wtbc) (gen-iadfa wtbc)))
	((null wtbc))
      (if (equal wtbc
		 (extract-words (iadfa-fsa iadfa)))
	  (return (cons last-word wtbc)))
      (setf last-word (car wtbc))
      (setf wtbc (cdr wtbc)))))
	     

(defun detect-problems-from-file (filename)
  (let ((words-to-be-checked nil))
    (let ((iadfa (build-iadfa)))
      (for-each-line-in-file (word filename)
	(setf words-to-be-checked (nconc words-to-be-checked (list word)))
	(handle-word iadfa (concatenate 'list word))
	(when (not (equal words-to-be-checked 
			  (extract-words (iadfa-fsa iadfa))))
	  (return))
	nil))
    ;; We got the first entry that trigger the problem.
    ;; we need now to see which entry is needed to start
    ;; the problem
    (detect-first-starting-problematic-word words-to-be-checked)))


(detect-problems-from-file "../../data/com.zone.sorted.small")
      
(defun iadfa-non-branch-suffix ()
  "This tests that the output of the iadfa isn't screwed up 
by the prefix 0--0 of 0--0--0 because the delete branch <commit_msg>comleted the detect-problems-from-file<commit_after>;(require :org.ancar.CLUnit "CLUnit")
;(require :com.rrette.finenight "finenight")

(load "CLUnit.lisp")

(load "finenight")

;(in-package :cl-user)

;(import 'org.ancar.CLUnit::deftest)

(in-package :com.rrette.finenight.iadfa)

(defparameter *my-iadfa* (debug-gen-iadfa '("addendice"
					    "append" "appendice"
					    "bappend" "bappendice"
					    "cappend" "cappendice"
					    "mormont")))

(defmacro test-equivalence (words)
  (with-syms (w iadfa output)
    `(let* ((,w ,words)
            (,iadfa (debug-gen-iadfa ,w))
	    (,output nil))
       (setf ,output (extract-words (iadfa-fsa ,iadfa)))
       (format t "input:~%~S~%output:~%~S~%" ,w ,output)
       (equal ,w ,output))))

(defun detect-problems (words)
  (let ((iadfa (build-iadfa))
	(words-to-be-checked nil))
    (dolist (word words)
      (setf words-to-be-checked (nconc words-to-be-checked (list word)))
      (handle-word iadfa (concatenate 'list word))
      (when (not (equal words-to-be-checked 
			(extract-words (iadfa-fsa iadfa))))
	(return)))
    ;; We got the first entry that trigger the problem.
    ;; we need now to see which entry is needed to start
    ;; the problem
    words-to-be-checked))


(defun detect-first-starting-problematic-word (words-to-be-checked)
  (let ((wtbc (cdr words-to-be-checked))
	(last-word (car words-to-be-checked)))
    (do ((iadfa (gen-iadfa wtbc) (gen-iadfa wtbc)))
	((null wtbc))
      (if (equal wtbc
		 (extract-words (iadfa-fsa iadfa)))
	  (return (cons last-word wtbc)))
      (setf last-word (car wtbc))
      (setf wtbc (cdr wtbc)))))

(defun filter-non-problematic-words (words-to-be-checked)
  (let ((problematics-words (list (car words-to-be-checked)))
	(last-word (cadr words-to-be-checked))
	(words-to-be-checked (cddr words-to-be-checked)))
    (do ((iadfa (gen-iadfa (append problematics-words words-to-be-checked))
		(gen-iadfa (append problematics-words words-to-be-checked))))
	((null words-to-be-checked))
      (if (equal (append problematics-words words-to-be-checked)
		 (extract-words (iadfa-fsa iadfa)))
	  (setf problematics-words (nconc problematics-words (list last-word))))
      (setf last-word (car words-to-be-checked))
      (setf words-to-be-checked (cdr words-to-be-checked)))
    problematics-words))
	     

(defun detect-problems-from-file (filename)
  (let ((words-to-be-checked nil))
    (let ((iadfa (build-iadfa)))
      (for-each-line-in-file (word filename)
	(setf words-to-be-checked (nconc words-to-be-checked (list word)))
	(handle-word iadfa (concatenate 'list word))
	(when (not (equal words-to-be-checked 
			  (extract-words (iadfa-fsa iadfa))))
	  (return))
	nil))
    ;; We got the first entry that trigger the problem.
    ;; we need now to see which entry is needed to start
    ;; the problem
    (setf words-to-be-checked 
	  (detect-first-starting-problematic-word words-to-be-checked))
    (setf words-to-be-checked
	  (filter-non-problematic-words words-to-be-checked))
    words-to-be-checked))
  


(detect-problems-from-file "../../data/com.zone.sorted.small")
      
(defun iadfa-non-branch-suffix ()
  "This tests that the output of the iadfa isn't screwed up 
by the prefix 0--0 of 0--0--0 because the delete branch 
{'fraction_non_alphanumeric': 0.16386777443874845, 'fraction_numerical': 0.00176772140710624, 'mean_word_length': 3.2160953800298064, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6862752', 'n_tokens_mistral': 2259, 'n_tokens_neox': 2116, 'n_words': 508}