<commit_before>;;; Hy AST walker
;; Copyright 2018 the authors.
;; This file is part of Hy, which is free software licensed under the Expat
;; license. See the LICENSE.

(import [hy [HyExpression HyDict]]
        [functools [partial]]
        [collections [OrderedDict]]
        [hy.macros [macroexpand :as mexpand]]
        [hy.compiler [HyASTCompiler]])

(defn walk [inner outer form]
  "Traverses form, an arbitrary data structure. Applies inner to each
  element of form, building up a data structure of the same type.
  Applies outer to the result."
  (cond
   [(instance? HyExpression form)
    (outer (HyExpression (map inner form)))]
   [(instance? HyDict form)
    (HyDict (outer (HyExpression (map inner form))))]
   [(instance? list form)
    ((type form) (outer (HyExpression (map inner form))))]
   [(coll? form)
    (walk inner outer (list form))]
   [True (outer form)]))

(defn postwalk [f form]
  "Performs depth-first, post-order traversal of form. Calls f on each
  sub-form, uses f's return value in place of the original."
  (walk (partial postwalk f) f form))

(defn prewalk [f form]
  "Performs depth-first, pre-order traversal of form. Calls f on each
  sub-form, uses f's return value in place of the original."
  (walk (partial prewalk f) identity (f form)))

;; TODO: move to hy.core?
(defn call? [form]
  "Checks whether form is a non-empty HyExpression"
  (and (instance? HyExpression form)
       form))

(defn macroexpand-all [form &optional module-name]
  "Recursively performs all possible macroexpansions in form."
  (setv module-name (or module-name (calling-module-name))
        quote-level [0])  ; TODO: make nonlocal after dropping Python2
  (defn traverse [form]
    (walk expand identity form))
  (defn expand [form]
    ;; manages quote levels
    (defn +quote [&optional [x 1]]
      (setv head (first form))
      (+= (get quote-level 0) x)
      (when (neg? (get quote-level 0))
        (raise (TypeError "unquote outside of quasiquote")))
      (setv res (traverse (cut form 1)))
      (-= (get quote-level 0) x)
      `(~head ~@res))
    (if (call? form)
        (cond [(get quote-level 0)
               (cond [(in (first form) '[unquote unquote-splice])
                      (+quote -1)]
                     [(= (first form) 'quasiquote) (+quote)]
                     [True (traverse form)])]
              [(= (first form) 'quote) form]
              [(= (first form) 'quasiquote) (+quote)]
              [True (traverse (mexpand form (HyASTCompiler module-name)))])
        (if (coll? form)
            (traverse form)
            form)))
  (expand form))

;; TODO: move to hy.extra.reserved?
(import hy)
(setv special-forms (list-comp k
                               [k (.keys hy.compiler._compile-table)]
                               (isinstance k hy._compat.string-types)))


(defn lambda-list [form]
  "
splits a fn argument list into sections based on &-headers.

returns an OrderedDict mapping headers to sublists.
Arguments without a header are under None.
"
  (setv headers '[&optional &rest &kwonly &kwargs]
        sections (OrderedDict [(, None [])])
        header None)
  (for [arg form]
    (if (in arg headers)
      (do (setv header arg)
          (assoc sections header [])
          ;; Don't use a header more than once. It's the compiler's problem.
          (.remove headers header))
      (.append (get sections header) arg)))
  sections)


(defn symbolexpand [form expander
                    &optional
                    [protected (frozenset)]
                    [quote-level 0]]
  (.expand (SymbolExpander form expander protected quote-level)))

(defclass SymbolExpander[]

  (defn __init__ [self form expander protected quote-level]
    (setv self.form form
          self.expander expander
          self.protected protected
          self.quote-level quote-level))

  (defn expand-symbols [self form &optional protected quote-level]
    (if (none? protected)
        (setv protected self.protected))
    (if (none? quote-level)
        (setv quote-level self.quote-level))
    (symbolexpand form self.expander protected quote-level))

  (defn traverse [self form &optional protected quote-level]
    (if (none? protected)
        (setv protected self.protected))
    (if (none? quote-level)
        (setv quote-level self.quote-level))
    (walk (partial symbolexpand
                   :expander self.expander
                   :protected protected
                   :quote-level quote-level)
          identity
          form))

  ;; manages quote levels
  (defn +quote [self &optional [x 1]]
    `(~(self.head) ~@(self.traverse (self.tail)
                                    :quote-level (+ self.quote-level x))))

  (defn handle-dot [self]
    `(. ~@(walk (fn [form]
                  (if (symbol? form)
                      form  ; don't expand attrs
                      (self.expand-symbols form)))
                identity
                (self.tail))))

  (defn head [self]
    (first self.form))

  (defn tail [self]
    (cut self.form 1))

  (defn handle-except [self]
    (setv tail (self.tail))
    ;; protect the "as" name binding the exception
    `(~(self.head) ~@(self.traverse tail (| self.protected
                                            (if (and tail
                                                     (-> tail
                                                         first
                                                         len
                                                         (= 2)))
                                                #{(first (first tail))}
                                                #{})))))
  (defn handle-args-list [self]
    (setv protected #{}
          argslist `[])
    (for [[header section] (-> self (.tail) first lambda-list .items)]
      (if header (.append argslist header))
      (cond [(in header [None '&rest '&kwargs])
             (.update protected (-> section flatten set))
             (.extend argslist section)]
            [(in header '[&optional &kwonly])
             (for [pair section]
               (cond [(coll? pair)
                      (.add protected (first pair))
                      (.append argslist
                               `[~(first pair)
                                 ~(self.expand-symbols (second pair))])]
                     [True
                      (.add protected pair)
                      (.append argslist pair)]))]))
    (, protected argslist))

  (defn handle-fn [self]
    (setv [protected argslist] (self.handle-args-list))
    `(~(self.head) ~argslist
      ~@(self.traverse (cut (self.tail) 1)(| protected self.protected))))

  ;; don't expand symbols in quotations
  (defn handle-quoted [self]
    (if (call? self.form)
        (if (in (self.head) '[unquote unquote-splice]) (self.+quote -1)<commit_msg>Remove lambda list unpacking handling in `let`.<commit_after>;;; Hy AST walker
;; Copyright 2018 the authors.
;; This file is part of Hy, which is free software licensed under the Expat
;; license. See the LICENSE.

(import [hy [HyExpression HyDict]]
        [functools [partial]]
        [collections [OrderedDict]]
        [hy.macros [macroexpand :as mexpand]]
        [hy.compiler [HyASTCompiler]])

(defn walk [inner outer form]
  "Traverses form, an arbitrary data structure. Applies inner to each
  element of form, building up a data structure of the same type.
  Applies outer to the result."
  (cond
   [(instance? HyExpression form)
    (outer (HyExpression (map inner form)))]
   [(instance? HyDict form)
    (HyDict (outer (HyExpression (map inner form))))]
   [(instance? list form)
    ((type form) (outer (HyExpression (map inner form))))]
   [(coll? form)
    (walk inner outer (list form))]
   [True (outer form)]))

(defn postwalk [f form]
  "Performs depth-first, post-order traversal of form. Calls f on each
  sub-form, uses f's return value in place of the original."
  (walk (partial postwalk f) f form))

(defn prewalk [f form]
  "Performs depth-first, pre-order traversal of form. Calls f on each
  sub-form, uses f's return value in place of the original."
  (walk (partial prewalk f) identity (f form)))

;; TODO: move to hy.core?
(defn call? [form]
  "Checks whether form is a non-empty HyExpression"
  (and (instance? HyExpression form)
       form))

(defn macroexpand-all [form &optional module-name]
  "Recursively performs all possible macroexpansions in form."
  (setv module-name (or module-name (calling-module-name))
        quote-level [0])  ; TODO: make nonlocal after dropping Python2
  (defn traverse [form]
    (walk expand identity form))
  (defn expand [form]
    ;; manages quote levels
    (defn +quote [&optional [x 1]]
      (setv head (first form))
      (+= (get quote-level 0) x)
      (when (neg? (get quote-level 0))
        (raise (TypeError "unquote outside of quasiquote")))
      (setv res (traverse (cut form 1)))
      (-= (get quote-level 0) x)
      `(~head ~@res))
    (if (call? form)
        (cond [(get quote-level 0)
               (cond [(in (first form) '[unquote unquote-splice])
                      (+quote -1)]
                     [(= (first form) 'quasiquote) (+quote)]
                     [True (traverse form)])]
              [(= (first form) 'quote) form]
              [(= (first form) 'quasiquote) (+quote)]
              [True (traverse (mexpand form (HyASTCompiler module-name)))])
        (if (coll? form)
            (traverse form)
            form)))
  (expand form))

;; TODO: move to hy.extra.reserved?
(import hy)
(setv special-forms (list-comp k
                               [k (.keys hy.compiler._compile-table)]
                               (isinstance k hy._compat.string-types)))


(defn lambda-list [form]
  "
splits a fn argument list into sections based on &-headers.

returns an OrderedDict mapping headers to sublists.
Arguments without a header are under None.
"
  (setv headers '[&optional &rest &kwonly &kwargs]
        sections (OrderedDict [(, None [])])
        header None)
  (for [arg form]
    (if (in arg headers)
      (do (setv header arg)
          (assoc sections header [])
          ;; Don't use a header more than once. It's the compiler's problem.
          (.remove headers header))
      (.append (get sections header) arg)))
  sections)


(defn symbolexpand [form expander
                    &optional
                    [protected (frozenset)]
                    [quote-level 0]]
  (.expand (SymbolExpander form expander protected quote-level)))

(defclass SymbolExpander[]

  (defn __init__ [self form expander protected quote-level]
    (setv self.form form
          self.expander expander
          self.protected protected
          self.quote-level quote-level))

  (defn expand-symbols [self form &optional protected quote-level]
    (if (none? protected)
        (setv protected self.protected))
    (if (none? quote-level)
        (setv quote-level self.quote-level))
    (symbolexpand form self.expander protected quote-level))

  (defn traverse [self form &optional protected quote-level]
    (if (none? protected)
        (setv protected self.protected))
    (if (none? quote-level)
        (setv quote-level self.quote-level))
    (walk (partial symbolexpand
                   :expander self.expander
                   :protected protected
                   :quote-level quote-level)
          identity
          form))

  ;; manages quote levels
  (defn +quote [self &optional [x 1]]
    `(~(self.head) ~@(self.traverse (self.tail)
                                    :quote-level (+ self.quote-level x))))

  (defn handle-dot [self]
    `(. ~@(walk (fn [form]
                  (if (symbol? form)
                      form  ; don't expand attrs
                      (self.expand-symbols form)))
                identity
                (self.tail))))

  (defn head [self]
    (first self.form))

  (defn tail [self]
    (cut self.form 1))

  (defn handle-except [self]
    (setv tail (self.tail))
    ;; protect the "as" name binding the exception
    `(~(self.head) ~@(self.traverse tail (| self.protected
                                            (if (and tail
                                                     (-> tail
                                                         first
                                                         len
                                                         (= 2)))
                                                #{(first (first tail))}
                                                #{})))))
  (defn handle-args-list [self]
    (setv protected #{}
          argslist `[])
    (for [[header section] (-> self (.tail) first lambda-list .items)]
      (if header (.append argslist header))
      (cond [(in header [None '&rest '&kwargs])
             (.update protected section)
             (.extend argslist section)]
            [(in header '[&optional &kwonly])
             (for [pair section]
               (cond [(coll? pair)
                      (.add protected (first pair))
                      (.append argslist
                               `[~(first pair)
                                 ~(self.expand-symbols (second pair))])]
                     [True
                      (.add protected pair)
                      (.append argslist pair)]))]))
    (, protected argslist))

  (defn handle-fn [self]
    (setv [protected argslist] (self.handle-args-list))
    `(~(self.head) ~argslist
      ~@(self.traverse (cut (self.tail) 1)(| protected self.protected))))

  ;; don't expand symbols in quotations
  (defn handle-quoted [self]
    (if (call? self.form)
        (if (in (self.head) '[unquote unquote-splice]) (self.+quote -1)
{'fraction_non_alphanumeric': 0.11936261969154302, 'fraction_numerical': 0.0027775747386886923, 'mean_word_length': 1.8719563392107472, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5856464', 'n_tokens_mistral': 3970, 'n_tokens_neox': 3781, 'n_words': 1301}