<commit_before>(ns ^:no-doc structural-typing.guts.type-descriptions.flatten
  (:use structural-typing.clojure.core)
  (:require [structural-typing.guts.type-descriptions.includes :as includes]
            [com.rpl.specter :as specter]))

(declare map->flatmap)

;;; Flattening paths


(defprotocol CondensedPath
  (->paths [this]))

(defprotocol PathComponent
  (add-on [this paths]))


(extend-type clojure.lang.Keyword
  PathComponent
  (add-on [this paths]
    (map #(conj % this) paths)))

(extend-type clojure.lang.PersistentVector
  CondensedPath
  (->paths [this]
    (reduce (fn [so-far elt] (add-on elt so-far))
            [[]]
            this)))

(defrecord Fork [alternatives]
  CondensedPath
  (->paths [this] (add-on this [[]]))
  PathComponent
  (add-on [this paths]
    (let [addons (mapcat ->paths alternatives)]
      (for [path paths
            addon addons]
        (into path addon)))))

(defn through-each
  "Use `through-each` to describe a \"forking\" path. This is convenient when two
   parts of a bigger data structure should be built the same way.
   
       (type! :Line {[(through-each :start :end) :location] (includes :Point)})
   
   [[each-of]] is a synonym. I tend to use `each-of` for the end of the path,
   `through-each` for a fork earlier than that."
  [& alternatives]
  (->Fork (map force-vector alternatives)))
(def each-of
  "Use `each-of` to describe a \"forking\" path. This is convenient when two
   parts of a bigger data structure should be built the same way.
   
       (type! :Plat {[:corners (each-of :nw :ne :sw :se)] (includes :GeoPoint)})
   
   [[through-each]] is a synonym. I tend to use `each-of` for the end of the path,
   `through-each` for a fork earlier than that."
  through-each)

(defn paths-of
  "Include all the paths of a type (or a literal map) within a path.
   
        (type! :StrictX (includes :X)
                        (requires (paths-of :X)))
                        
   
   The above example constructs a stricter version of `:X` by insisting 
   all of its paths are required.
"
  [type-signifier-or-map]
  (let [handle-kvs #(->Fork (keys %))]
    (if (map? type-signifier-or-map)
      (handle-kvs (map->flatmap type-signifier-or-map))
      (-> (fn [type-map]
            (handle-kvs ( (includes/includes type-signifier-or-map) type-map)))
          includes/as-type-expander))))

(deftype ALLType []
  CondensedPath 
  (->paths [this] (add-on this [[]]))
  PathComponent
  (add-on [this paths]
    (prn :adding this :onto paths)
    (let [note-index (specter/view (partial map-indexed vector)) ; value [x y] -> [ [0 x] [1 y] ]
                                                                 ; for next step
          prepend-index (specter/collect-one specter/FIRST)  ; stash the index (0 or 1 above) so that
                                                             ; Specter will prepend to final result.
          intermediate-value specter/LAST]  ; Further selectors apply to the original val (x and y)
      (for [path paths]
        (let [x (into path [note-index specter/ALL prepend-index intermediate-value])]
          (prn :result x)
          x)))))

;; TODO: don't know yet if this should reject objects other than Specter-equivalents,
;; or if compilation should.
(extend-type Object
  PathComponent
  (add-on [this paths]
    (map #(conj % this) paths)))

(defn uncondense-path [condensed-path]
  (->paths condensed-path))


   


;;;; Flattening maps

(declare map->flatmap map->pairs)<commit_msg>Remove dead code<commit_after>(ns ^:no-doc structural-typing.guts.type-descriptions.flatten
  (:use structural-typing.clojure.core)
  (:require [structural-typing.guts.type-descriptions.includes :as includes]
            [com.rpl.specter :as specter]))

(declare map->flatmap map->pairs)

;;; Flattening paths


(defprotocol CondensedPath
  (->paths [this]))

(defprotocol PathComponent
  (add-on [this paths]))


(extend-type clojure.lang.Keyword
  PathComponent
  (add-on [this paths]
    (map #(conj % this) paths)))

(extend-type clojure.lang.PersistentVector
  CondensedPath
  (->paths [this]
    (reduce (fn [so-far elt] (add-on elt so-far))
            [[]]
            this)))

(defrecord Fork [alternatives]
  CondensedPath
  (->paths [this] (add-on this [[]]))
  PathComponent
  (add-on [this paths]
    (let [addons (mapcat ->paths alternatives)]
      (for [path paths
            addon addons]
        (into path addon)))))

(defn through-each
  "Use `through-each` to describe a \"forking\" path. This is convenient when two
   parts of a bigger data structure should be built the same way.
   
       (type! :Line {[(through-each :start :end) :location] (includes :Point)})
   
   [[each-of]] is a synonym. I tend to use `each-of` for the end of the path,
   `through-each` for a fork earlier than that."
  [& alternatives]
  (->Fork (map force-vector alternatives)))
(def each-of
  "Use `each-of` to describe a \"forking\" path. This is convenient when two
   parts of a bigger data structure should be built the same way.
   
       (type! :Plat {[:corners (each-of :nw :ne :sw :se)] (includes :GeoPoint)})
   
   [[through-each]] is a synonym. I tend to use `each-of` for the end of the path,
   `through-each` for a fork earlier than that."
  through-each)

(defn paths-of
  "Include all the paths of a type (or a literal map) within a path.
   
        (type! :StrictX (includes :X)
                        (requires (paths-of :X)))
                        
   
   The above example constructs a stricter version of `:X` by insisting 
   all of its paths are required.
"
  [type-signifier-or-map]
  (let [handle-kvs #(->Fork (keys %))]
    (if (map? type-signifier-or-map)
      (handle-kvs (map->flatmap type-signifier-or-map))
      (-> (fn [type-map]
            (handle-kvs ( (includes/includes type-signifier-or-map) type-map)))
          includes/as-type-expander))))

;; TODO: don't know yet if this is the right place to reject types that can't be
;; used in one of our paths.
(extend-type Object
  PathComponent
  (add-on [this paths]
    (map #(conj % this) paths)))

(defn uncondense-path [condensed-path]
  (->paths condensed-path))


   


;;;; Flattening maps
{'fraction_non_alphanumeric': 0.13117134559535334, 'fraction_numerical': 0.0006453694740238787, 'mean_word_length': 2.6187974314068887, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1368612', 'n_tokens_mistral': 2047, 'n_tokens_neox': 1992, 'n_words': 656}