<commit_before>(cl:in-package #:common-lisp-user)

(defpackage #:cleavir-test-generate-ast
  (:use #:common-lisp))

(in-package #:cleavir-test-generate-ast)

(defgeneric same-p (ast1 ast2))

(defvar *table*)

(defun same-ast-p (ast1 ast2)
  (let ((*table* (make-hash-table :test #'equal)))
    (same-p ast1 ast2)))

(defmethod same-p :around (ast1 ast2)
  (cond ((not (eq (class-of ast1) (class-of ast2)))
	 nil)
	((gethash (cons ast1 ast2) *table*)
	 t)
	(t
	 (setf (gethash (cons ast1 ast2) *table*) t)
	 (call-next-method ast1 ast2))))

(defmethod same-p ((ast1 cleavir-ast:constant-ast) ast2)
  (equalp (cleavir-ast:value ast1) (cleavir-ast:value ast2)))

(defmethod same-p ((ast1 cleavir-ast:lexical-ast) ast2)
  t)

(defmethod same-p ((ast1 cleavir-ast:symbol-value-ast) ast2)
  (eq (cleavir-ast:symbol ast1) (cleavir-ast:symbol ast2)))

(defmethod same-p ((ast1 cleavir-ast:set-symbol-value-ast) ast2)
  (and (eq (cleavir-ast:symbol ast1) (cleavir-ast:symbol ast2))
       (same-p (cleavir-ast:value-ast ast1) (cleavir-ast:value-ast ast2))))

(defmethod same-p ((ast1 cleavir-ast:fdefinition-ast) ast2)
  (equal (cleavir-ast:name ast1) (cleavir-ast:name ast2)))

(defmethod same-p ((ast1 cleavir-ast:call-ast) ast2)
  (and (same-p (cleavir-ast:callee-ast ast1) (cleavir-ast:callee-ast ast2))
       (every #'same-p
	      (cleavir-ast:argument-asts ast1)
	      (cleavir-ast:argument-asts ast2))))

(defmethod same-p ((ast1 cleavir-ast:function-ast) ast2)
  (flet ((same-lambda-list-entry (e1 e2)
	   (or (and (eq e1 '&optional)
		    (eq e2 '&optional))
	       (and (eq e1 '&key)
		    (eq e2 '&key))
	       (and (eq e1 '&allow-other-keys)
		    (eq e2 '&allow-other-keys))
	       (and (listp e1)
		    (listp e2)
		    (= (length e1) (length e2))
		    (every #'same-p e1 e2))
	       (same-p e1 e2))))
    (and (every #'same-lambda-list-entry
		(cleavir-ast:lambda-list ast1)
		(cleavir-ast:lambda-list ast2))
	 (same-p (cleavir-ast:body-ast ast1)
		 (cleavir-ast:body-ast ast2)))))

(defmethod same-p ((ast1 cleavir-ast:progn-ast) ast2)
  (every #'same-p
	 (cleavir-ast:form-asts ast1)
	 (cleavir-ast:form-asts ast2)))

(defmethod same-p ((ast1 cleavir-ast:block-ast) ast2)
  (same-p (cleavir-ast:body-ast ast1) (cleavir-ast:body-ast ast2)))

(defmethod same-p ((ast1 cleavir-ast:return-from-ast) ast2)
  (and (same-p (cleavir-ast:block-ast ast1) (cleavir-ast:block-ast ast2))
       (same-p (cleavir-ast:form-ast ast1) (cleavir-ast:form-ast ast1))))

(defmethod same-p ((ast1 cleavir-ast:setq-ast) ast2)
  (and (same-p (cleavir-ast:lhs-ast ast1) (cleavir-ast:lhs-ast ast2))
       (same-p (cleavir-ast:value-ast ast1) (cleavir-ast:value-ast ast2))))

(defmethod same-p ((ast1 cleavir-ast:tag-ast) ast2)
  (eq (cleavir-ast:name ast1) (cleavir-ast:name ast2)))

(defmethod same-p ((ast1 cleavir-ast:tagbody-ast) ast2)
  (every #'same-p
	 (cleavir-ast:item-asts ast1)
	 (cleavir-ast:item-asts ast2)))

(defmethod same-p ((ast1 cleavir-ast:go-ast) ast2)
  (same-p (cleavir-ast:tag-ast ast1) (cleavir-ast:tag-ast ast2)))
<commit_msg>Better version of SAME-P for FUNCTION-AST.<commit_after>(cl:in-package #:common-lisp-user)

(defpackage #:cleavir-test-generate-ast
  (:use #:common-lisp))

(in-package #:cleavir-test-generate-ast)

(defgeneric same-p (ast1 ast2))

(defvar *table*)

(defun same-ast-p (ast1 ast2)
  (let ((*table* (make-hash-table :test #'equal)))
    (same-p ast1 ast2)))

(defmethod same-p :around (ast1 ast2)
  (cond ((not (eq (class-of ast1) (class-of ast2)))
	 nil)
	((gethash (cons ast1 ast2) *table*)
	 t)
	(t
	 (setf (gethash (cons ast1 ast2) *table*) t)
	 (call-next-method ast1 ast2))))

(defmethod same-p ((ast1 cleavir-ast:constant-ast) ast2)
  (equalp (cleavir-ast:value ast1) (cleavir-ast:value ast2)))

(defmethod same-p ((ast1 cleavir-ast:lexical-ast) ast2)
  t)

(defmethod same-p ((ast1 cleavir-ast:symbol-value-ast) ast2)
  (eq (cleavir-ast:symbol ast1) (cleavir-ast:symbol ast2)))

(defmethod same-p ((ast1 cleavir-ast:set-symbol-value-ast) ast2)
  (and (eq (cleavir-ast:symbol ast1) (cleavir-ast:symbol ast2))
       (same-p (cleavir-ast:value-ast ast1) (cleavir-ast:value-ast ast2))))

(defmethod same-p ((ast1 cleavir-ast:fdefinition-ast) ast2)
  (equal (cleavir-ast:name ast1) (cleavir-ast:name ast2)))

(defmethod same-p ((ast1 cleavir-ast:call-ast) ast2)
  (and (same-p (cleavir-ast:callee-ast ast1) (cleavir-ast:callee-ast ast2))
       (every #'same-p
	      (cleavir-ast:argument-asts ast1)
	      (cleavir-ast:argument-asts ast2))))

(defmethod same-p ((ast1 cleavir-ast:function-ast) ast2)
  (flet ((same-lambda-list-entry (e1 e2)
	   (cond ((and (eq e1 e2)
		       (member e1 lambda-list-keywords)
		       (member e2 lambda-list-keywords))
		  t)
		 ((or (member e1 lambda-list-keywords)
		      (member e2 lambda-list-keywords))
		  nil)
		 ((and (listp e1)
		       (listp e2)
		       (= (length e1) (length e2))
		       (every #'same-p e1 e2))
		  t)
		 (t
		  (same-p e1 e2)))))
    (and (every #'same-lambda-list-entry
		(cleavir-ast:lambda-list ast1)
		(cleavir-ast:lambda-list ast2))
	 (same-p (cleavir-ast:body-ast ast1)
		 (cleavir-ast:body-ast ast2)))))

(defmethod same-p ((ast1 cleavir-ast:progn-ast) ast2)
  (every #'same-p
	 (cleavir-ast:form-asts ast1)
	 (cleavir-ast:form-asts ast2)))

(defmethod same-p ((ast1 cleavir-ast:block-ast) ast2)
  (same-p (cleavir-ast:body-ast ast1) (cleavir-ast:body-ast ast2)))

(defmethod same-p ((ast1 cleavir-ast:return-from-ast) ast2)
  (and (same-p (cleavir-ast:block-ast ast1) (cleavir-ast:block-ast ast2))
       (same-p (cleavir-ast:form-ast ast1) (cleavir-ast:form-ast ast1))))

(defmethod same-p ((ast1 cleavir-ast:setq-ast) ast2)
  (and (same-p (cleavir-ast:lhs-ast ast1) (cleavir-ast:lhs-ast ast2))
       (same-p (cleavir-ast:value-ast ast1) (cleavir-ast:value-ast ast2))))

(defmethod same-p ((ast1 cleavir-ast:tag-ast) ast2)
  (eq (cleavir-ast:name ast1) (cleavir-ast:name ast2)))

(defmethod same-p ((ast1 cleavir-ast:tagbody-ast) ast2)
  (every #'same-p
	 (cleavir-ast:item-asts ast1)
	 (cleavir-ast:item-asts ast2)))

(defmethod same-p ((ast1 cleavir-ast:go-ast) ast2)
  (same-p (cleavir-ast:tag-ast ast1) (cleavir-ast:tag-ast ast2)))

{'fraction_non_alphanumeric': 0.18097711410485312, 'fraction_numerical': 0.03116377211491641, 'mean_word_length': 3.8329411764705883, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4732532', 'n_tokens_mistral': 2746, 'n_tokens_neox': 2563, 'n_words': 541}