<commit_before>;; Find the immediate subparts of expressions and patterns, for
;; source-code walking.
;; XXX duplication of terp/parse.scm

(let none '(() ()))

;; Return a pair `(,exprs ,patts) of the immediate subexpressions and
;; subpatterns of expr.
(to (expr-subparts expr)
  (match (macroexpand-outer-expr expr)
    ((: symbol?)                    none)
    ((: self-evaluating?)           none)
    (`(quote ,_)                    none)
    ((: term?)                      `(,expr.arguments ()))
    (`(let ,p ,e)                   `((,e) (,p)))
    (`(make ,(: symbol?) ,@clauses) (clauses-subparts clauses))
    (`(make ,@clauses)              (clauses-subparts clauses))
    (`(do ,@es)                     `(,es ()))
    (`(call ,e1 ,e2)                `((,e1 ,e2) ()))
    (`(,e1 ,(: cue?) ,@es)          `((,e1 ,@es) ()))
    (`(,e1 ,@es)                    `((,e1 ,@es) ()))))

(to (clauses-subparts clauses)
  (match (transpose (each clause-subparts clauses))
    (() none)
    (`(,es-lists ,ps-lists) `(,(call chain es-lists)
                              ,(call chain ps-lists)))))

(to (clause-subparts `(,p ,@es))
  `(,es (,p)))

(to (macroexpand-outer-expr expr)
  (match (maybe-macroexpand-expr expr)
    (#no expr)
    (expanded
     ;; Keep expanding the outermost expr until we get core
     ;; syntax. But N.B. we leave subexpressions unexpanded.
     (macroexpand-outer-expr expanded))))

;; Return a pair `(,exprs ,patts) of the immediate subexpressions and
;; subpatterns of patt.
(to (patt-subparts patt)<commit_msg>fix forgot to macroexpand<commit_after>;; Find the immediate subparts of expressions and patterns, for
;; source-code walking.
;; XXX duplication of terp/parse.scm

(let none '(() ()))

;; Return a pair `(,exprs ,patts) of the immediate subexpressions and
;; subpatterns of expr.
(to (expr-subparts expr)
  (match (macroexpand-outer-expr expr)
    ((: symbol?)                    none)
    ((: self-evaluating?)           none)
    (`(quote ,_)                    none)
    ((: t term?)                    `(,t.arguments ()))
    (`(let ,p ,e)                   `((,e) (,p)))
    (`(make ,(: symbol?) ,@clauses) (clauses-subparts clauses))
    (`(make ,@clauses)              (clauses-subparts clauses))
    (`(do ,@es)                     `(,es ()))
    (`(call ,e1 ,e2)                `((,e1 ,e2) ()))
    (`(,e1 ,(: cue?) ,@es)          `((,e1 ,@es) ()))
    (`(,e1 ,@es)                    `((,e1 ,@es) ()))))

(to (clauses-subparts clauses)
  (match (transpose (each clause-subparts clauses))
    (() none)
    (`(,es-lists ,ps-lists) `(,(call chain es-lists)
                              ,(call chain ps-lists)))))

(to (clause-subparts `(,p ,@es))
  `(,es (,p)))

(to (macroexpand-outer-expr expr)
  (match (maybe-macroexpand-expr expr)
    (#no expr)
    (expanded
     ;; Keep expanding the outermost expr until we get core
     ;; syntax. But N.B. we leave subexpressions unexpanded.
     (macroexpand-outer-expr expanded))))

;; Return a pair `(,exprs ,patts) of the immediate subexpressions and
;; subpatterns of patt.
(to (patt-subparts patt)
{'fraction_non_alphanumeric': 0.18304866258459554, 'fraction_numerical': 0.00515630035449565, 'mean_word_length': 2.250261780104712, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1853330', 'n_tokens_mistral': 1098, 'n_tokens_neox': 1030, 'n_words': 287}