<commit_before>;;;;; TRE tree processor transpiler
;;;;; Copyright (c) 2008 <NAME> <<EMAIL>>
;;;;;
;;;;; Identifier style conversion

(defun transpiler-translate-symbol (tr from to)
  (acons! from to (transpiler-symbol-translations tr)))

(defun transpiler-special-char? (tr x)
  (not (funcall (transpiler-identifier-char? tr) x)))

(defun transpiler-symbol-string (tr s)
  (with (encapsulate-char
		   (fn string-list (string-concat "T" (format nil "~A" (char-code _))))
				
		 convert-camel
		   (fn (when _
			     (with (c (char-downcase _.))
			       (if (and (in=? c #\* #\-)
							._)
					   (cons (char-upcase (cadr _))
							 (convert-camel (cddr _)))
					   (cons c (convert-camel ._))))))

		 convert-special2
		   (fn (when _
			     (with (c _.)
				   (if (transpiler-special-char? tr c)
					   (append (encapsulate-char c)
							   (convert-special2 ._))
					   (cons c (convert-special2 ._))))))

		 convert-special
		   (fn (when _
			     (with (c _.)
				   ; Encapsulate initial char if it's a digit.
				   (if (digit-char-p c)
					   (append (encapsulate-char c)
							   (convert-special2 ._))
					   (convert-special2 _)))))

		 str (string s)
	     l (length str))

	(if (or (stringp s)
			(numberp s))
		str
        (list-string
	      (convert-special
            (if (and (< 2 (length str)) ; Make *GLOBAL* upcase.
			         (= (elt str 0) #\*)
			         (= (elt str (1- l)) #\*))
		        (remove-if (fn = _ #\-)
					       (string-list (string-upcase (subseq str 1 (1- l)))))
    	        (convert-camel (string-list str))))))))

(defun transpiler-to-string (tr x)
  (maptree #'((e)
				(if
				  (consp e)
					(if (eq e. '%transpiler-string)
					    (string-concat "\"" (cadr e) "\"")
						(if (in? e. '%transpiler-native '%no-expex)
							(transpiler-to-string tr .e)
							e))
				  (stringp e)<commit_msg>Split identifier at dots before converting.<commit_after>;;;;; TRE transpiler
;;;;; Copyright (c) 2008-2009 <NAME> <<EMAIL>>
;;;;;
;;;;; Identifier style conversion

(defun transpiler-translate-symbol (tr from to)
  (acons! from to (transpiler-symbol-translations tr)))

(defun transpiler-special-char? (tr x)
  (not (funcall (transpiler-identifier-char? tr) x)))

(defun transpiler-symbol-string-r (tr s)
  (with (encapsulate-char
		   (fn string-list (string-concat "T" (format nil "~A" (char-code _))))
				
		 convert-camel
		   (fn (when _
			     (with (c (char-downcase _.))
			       (if (and (in=? c #\* #\-)
							._)
					   (cons (char-upcase (cadr _))
							 (convert-camel (cddr _)))
					   (cons c (convert-camel ._))))))

		 convert-special2
		   (fn (when _
			     (with (c _.)
				   (if (transpiler-special-char? tr c)
					   (append (encapsulate-char c)
							   (convert-special2 ._))
					   (cons c (convert-special2 ._))))))

		 convert-special
		   (fn (when _
			     (with (c _.)
				   ; Encapsulate initial char if it's a digit.
				   (if (digit-char-p c)
					   (append (encapsulate-char c)
							   (convert-special2 ._))
					   (convert-special2 _))))))
	(if (or (stringp s)
			(numberp s))
		(string s)
		(with (str (string s)
	     	   l (length str))
          (list-string
	        (convert-special
              (if (and (< 2 (length str)) ; Make *GLOBAL* upcase.
			           (= (elt str 0) #\*)
			           (= (elt str (1- l)) #\*))
		          (remove-if (fn = _ #\-)
					         (string-list (string-upcase (subseq str 1 (1- l)))))
    	          (convert-camel (string-list str)))))))))

(defun transpiler-symbol-string (tr s)
  (let sl (string-list (string s))
    (if (position #\. sl)
	    (apply #'string-concat
			   (pad (mapcar (fn transpiler-symbol-string-r tr (make-symbol (list-string _)))
			  				(split #\. sl))
					"."))
	    (transpiler-symbol-string-r tr s))))

(defun transpiler-to-string (tr x)
  (maptree #'((e)
				(if
				  (consp e)
					(if (eq e. '%transpiler-string)
						(funcall (transpiler-gen-string tr) tr (cadr e))
						(if (in? e. '%transpiler-native '%no-expex)
							(transpiler-to-string tr .e)
							e))
				  (stringp e)
{'fraction_non_alphanumeric': 0.17373192844890958, 'fraction_numerical': 0.0078412153883852, 'mean_word_length': 2.080754716981132, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1331290', 'n_tokens_mistral': 1824, 'n_tokens_neox': 1451, 'n_words': 371}