<commit_before>(cl:in-package #:cleavir-equivalent-lexical-locations)

(defun set-equality (set1 set2 test)
  (and (null (set-difference set1 set2 :test test))
       (null (set-difference set2 set1 :test test))))

(defun class-equality (class1 class2)
  (set-equality class1 class2 #'eq))

(defun partition-equality (partition1 partition2)
  (set-equality partition1 partition2 #'class-equality))

(defun update-for-meet (partition defined used)
  (let* ((dclass (find defined partition :test #'member))
	 (intermediate (cond ((null dclass)
			      partition)
			     ((= (length dclass) 2)
			      (remove dclass partition :test #'eq))
			     (t
			      (cons (remove defined dclass :test #'eq)
				    (remove dclass partition :test #'eq)))))
	 (uclass (find used intermediate :test #'member)))
    (if (null uclass)
	(cons (list defined used) intermediate)
	(cons (cons defined uclass)
	      (remove uclass intermediate :test #'eq)))))<commit_msg>New function for computing a new partition on JOIN.<commit_after>(cl:in-package #:cleavir-equivalent-lexical-locations)

(defun set-equality (set1 set2 test)
  (and (null (set-difference set1 set2 :test test))
       (null (set-difference set2 set1 :test test))))

(defun class-equality (class1 class2)
  (set-equality class1 class2 #'eq))

(defun partition-equality (partition1 partition2)
  (set-equality partition1 partition2 #'class-equality))

(defun update-for-meet (partition defined used)
  (let* ((dclass (find defined partition :test #'member))
	 (intermediate (cond ((null dclass)
			      partition)
			     ((= (length dclass) 2)
			      (remove dclass partition :test #'eq))
			     (t
			      (cons (remove defined dclass :test #'eq)
				    (remove dclass partition :test #'eq)))))
	 (uclass (find used intermediate :test #'member)))
    (if (null uclass)
	(cons (list defined used) intermediate)
	(cons (cons defined uclass)
	      (remove uclass intermediate :test #'eq)))))

(defun update-for-join (partition1 partition2)
  (let* ((locations1 (reduce #'append partition1 :from-end t))
	 (locations2 (reduce #'append partition2 :from-end t))
	 (common (intersection locations1 locations2 :test #'eq))
	 (p1 (loop for class in partition1
		   for stripped = (intersection class common :test #'eq)
		   when (> (length stripped) 1)
		     collect stripped))
	 (p2 (loop for class in partition2
		   for stripped = (intersection class common :test #'eq)
		   when (> (length stripped) 1)
		     collect stripped))
	 (result '()))
    (loop until (null common)
	  do (let* ((location (first common))
		    (class1 (find location p1 :test #'member))
		    (class2 (find location p2 :test #'member))
		    (intersection (intersection class1 class2 :test #'eq))
		    (diff1 (set-difference class1 intersection :test #'eq))
		    (diff2 (set-difference class2 intersection :test #'eq)))
	       (setf common (set-difference common intersection :test #'eq))
	       (setf p1 (remove class1 p1 :test #'eq))
	       (setf p2 (remove class2 p2 :test #'eq))
	       (when (> (length intersection) 1)
		 (push intersection result))
	       (when (> (length diff1) 1)
		 (push diff1 p1))
	       (when (> (length diff2) 1)
		 (push diff2 p2))))
    result))
{'fraction_non_alphanumeric': 0.13202858030444237, 'fraction_numerical': 0.021435228331780055, 'mean_word_length': 3.5480225988700567, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1121490', 'n_tokens_mistral': 1108, 'n_tokens_neox': 993, 'n_words': 343}