<commit_before>(ns mpa.cst.parser.parser)

(import '(mpa.cst.parser MpaParser
                         MpaStart)
        '(java.io StringReader))

(MpaParser. (StringReader. ""))

(defn- cst2list
  [cst]
  (case (.jjtGetNumChildren cst)
    0 (case (.getLexeme cst) ; Boolean | Atom
        "TRUE" 'true
        "FALSE" 'false
        (symbol (.getLexeme cst)))
    1 (if (= (class cst) mpa.cst.parser.MpaNotExpr) ; NOT | hierarchic descent
        (list 'not (cst2list (.jjtGetChild cst 0)))
        (cst2list (.jjtGetChild cst 0)))
    2 (let [op-nr (.getOp cst)
            AND (MpaParser/AND)
						NAND (MpaParser/NAND)
						OR (MpaParser/OR)
						NOR (MpaParser/NOR)
						IF (MpaParser/IF)
						NIF (MpaParser/NIF)
						IMPL (MpaParser/IMPL)
						NIMPL (MpaParser/NIMPL)
						IFF (MpaParser/IFF)
						XOR (MpaParser/XOR)
            op (cond
                 (= op-nr AND) 'and
                 (= op-nr NAND) 'nand
                 (= op-nr OR) 'or
                 (= op-nr NOR) 'nor
                 (= op-nr IF) 'cimpl
                 (= op-nr NIF) 'ncimpl
                 (= op-nr IMPL) 'impl
                 (= op-nr NIMPL) 'nimpl
                 (= op-nr IFF) 'equiv
                 (= op-nr XOR) 'xor)]
        (list op (cst2list (.jjtGetChild cst 0)) (cst2list (.jjtGetChild cst 1))))))

(defn javaCCparse
  [formula]
  (MpaParser/ReInit (StringReader. formula))
  (let [cst (MpaParser/Start)]
    (cst2list cst)))
<commit_msg>(parse) now has metadata :constants?, indicating if bools are in formula<commit_after>(ns mpa.cst.parser.parser)

(import '(mpa.cst.parser MpaParser
                         MpaStart)
        '(java.io StringReader))

(MpaParser. (StringReader. ""))

(defn- cst2list
  [cst *const?*]
  (case (.jjtGetNumChildren cst)
    0 (case (.getLexeme cst) ; Boolean | Atom
        "TRUE" (do (reset! *const?* true) 'true)
        "FALSE" (do (reset! *const?* true) 'false)
        (symbol (.getLexeme cst)))
    1 (if (= (class cst) mpa.cst.parser.MpaNotExpr) ; NOT | hierarchic descent
        (list 'not (cst2list (.jjtGetChild cst 0) *const?*))
        (cst2list (.jjtGetChild cst 0) *const?*))
    2 (let [op-nr (.getOp cst)
            AND (MpaParser/AND)
						NAND (MpaParser/NAND)
						OR (MpaParser/OR)
						NOR (MpaParser/NOR)
						IF (MpaParser/IF)
						NIF (MpaParser/NIF)
						IMPL (MpaParser/IMPL)
						NIMPL (MpaParser/NIMPL)
						IFF (MpaParser/IFF)
						XOR (MpaParser/XOR)
            op (cond
                 (= op-nr AND) 'and
                 (= op-nr NAND) 'nand
                 (= op-nr OR) 'or
                 (= op-nr NOR) 'nor
                 (= op-nr IF) 'cimpl
                 (= op-nr NIF) 'ncimpl
                 (= op-nr IMPL) 'impl
                 (= op-nr NIMPL) 'nimpl
                 (= op-nr IFF) 'equiv
                 (= op-nr XOR) 'xor)]
        (list op (cst2list (.jjtGetChild cst 0) *const?*) (cst2list (.jjtGetChild cst 1) *const?*)))))

(defn javaCCparse
  [formula]
  (MpaParser/ReInit (StringReader. formula))
  (let [cst (MpaParser/Start)
        *const?* (atom false)
        parsed (cst2list cst *const?*)]
    (with-meta parsed {:constants? @*const?*})))


{'fraction_non_alphanumeric': 0.1442550505050505, 'fraction_numerical': 0.008207070707070708, 'mean_word_length': 1.7822651448639157, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5691822', 'n_tokens_mistral': 1317, 'n_tokens_neox': 1177, 'n_words': 282}