<commit_before>(ns core
  (:refer-clojure :exclude [replace])
  (:require
    [clojure.java.io :as io]
    [clojure.java.shell :refer [sh]]
    [clojure.tools.reader :as reader]
    [clojure.tools.reader.reader-types :as readers]
    [clojure.string :refer [split-lines join replace trim]]
    [cljs.tagged-literals :refer [*cljs-data-readers*]]
    [me.raynes.fs :refer [mkdir]]))

;; location of the documents generated by this program
(def cljsdoc-dir "../../docs-generated")

;; location of the clojure & clojurescript repos to parse
(def repo-dir "repos")

;; SHA1 hashes of the checked out commits of the language repos
(def repo-sha1
  (atom {"clojure" nil
         "clojurescript" nil
         "core.async" nil}))

;; Table of namespaces that we will parse
(def cljs-ns-paths
  ; NS                        REPO             FILE               FULL PATH
  {"cljs.core"              {"clojurescript" {"core.cljs"        "src/cljs/cljs/core.cljs"
                                              "core.clj"         "src/clj/cljs/core.clj"}
                             "clojure"       {"core.clj"         "src/clj/clojure/core.clj"
                                              "core_deftype.clj" "src/clj/clojure/core_deftype.clj"}}
   "cljs.reader"            {"clojurescript" {"reader.cljs"      "src/cljs/cljs/reader.cljs"}}
   "clojure.set"            {"clojurescript" {"set.cljs"         "src/cljs/clojure/set.cljs"}}
   "clojure.string"         {"clojurescript" {"string.cljs"      "src/cljs/clojure/string.cljs"}}
   "clojure.walk"           {"clojurescript" {"walk.cljs"        "src/cljs/clojure/walk.cljs"}}
   "clojure.zip"            {"clojurescript" {"zip.cljs"         "src/cljs/clojure/zip.cljs"}}
   "clojure.data"           {"clojurescript" {"data.cljs"        "src/cljs/clojure/data.cljs"}}
   "cljs.core.async"        {"core.async"    {"async.cljs"       "src/main/clojure/cljs/core/async.cljs"}}
   "cljs.core.async.macros" {"core.async"    {"macros.clj"       "src/main/clojure/cljs/core/async/macros.clj"}}})

;;------------------------------------------------------------
;; Form Reading
;;------------------------------------------------------------

(defn read-forms
  [r]
  (loop [forms (transient [])]
    (if-let [f (try (binding [reader/*data-readers* *cljs-data-readers*]
                      (reader/read r))
                    (catch Exception e
                      (when-not (= (.getMessage e) "EOF") (throw e))))]
      (recur (conj! forms f))
      (persistent! forms))))

(defn read-forms-from-file
  [path]
  (let [is (io/input-stream path)
        r1 (readers/input-stream-push-back-reader is)
        r  (readers/source-logging-push-back-reader r1 1 path)]
    (read-forms r)))

(defn read-forms-from-str
  [s]
  (let [r (readers/string-push-back-reader s)]
    (read-forms r)))

;;------------------------------------------------------------
;; Repos
;;------------------------------------------------------------

(defn get-repo-sha1
  [repo]
  (trim (:out (sh "git" "rev-parse" "HEAD" :dir (str repo-dir "/" repo)))))

(defn get-github-file-link
  [repo path [start-line end-line]]
  (let [sha1 (get @repo-sha1 repo)
        strip-path (subs path (inc (count repo)))]
    (str "https://github.com/clojure/" repo "/blob/" sha1 "/" strip-path
         "#L" start-line "-L" end-line)))

;;------------------------------------------------------------
;; Repo Helpers
;;------------------------------------------------------------

(defn get-repo-path
  "Get path to the given repo file"
  [ns- repo file]
  (let [path (get-in cljs-ns-paths [ns- repo file])]
    (str repo-dir "/" repo "/" path)))

(defn get-forms
  "Get forms from the given repo file"
  [ns- repo file]
  (read-forms-from-file (get-repo-path ns- repo file)))

;;------------------------------------------------------------
;; Docstring Helpers
;;------------------------------------------------------------

(defn get-docstring-indent
  [docstring]
  (let [lines (split-lines docstring)]
    (if (> (count lines) 1)
      (let [[first-line & indented-lines] lines
            get-indent-length #(count (re-find #"^ *" %))
            has-content? #(pos? (count (trim %)))]
        (->> indented-lines
             (filter has-content?)
             (map get-indent-length)
             (apply min 3)))
      0)))

(defn fix-docstring
  "Remove indentation from docstring."
  [docstring]
  (when (string? docstring)
    (let [indent-length (get-docstring-indent docstring)]
      (if (zero? indent-length)
        docstring
        (let [[first-line & indented-lines] (split-lines docstring)
              indent (re-pattern (str "^ {" indent-length "}"))
              remove-indent #(replace % indent "")]
          (->> indented-lines
               (map remove-indent)
               (cons first-line)
               (join "\n")))))))

(defn try-remove-docs
  "Try to remove docstring/attr-map from source if they are on their expected lines."
  [source {:keys [start-line end-line forms] :as expected-docs}]
  (if (nil? expected-docs)
    source
    (let [i-lines (map-indexed vector (split-lines source))
          to-str #(join "\n" (map second %))
          doc-line? #(<= start-line (first %) end-line)
          doc-str (to-str (filter doc-line? i-lines))
          actual-forms (read-forms-from-str doc-str)]
      (if (= actual-forms forms)
        (to-str (remove doc-line? i-lines))
        (do
          (binding [*out* *err*]
            (println "=====================================")
            (println "Warning: couldn't remove docstring:")
            (println "expected:" (pr-str forms))
            (println "actual:" (pr-str actual-forms))
            (println "source:" (pr-str source))
            (println "====================================="))
          source)))))

(defn try-locate-docs
  "Try to guess which lines the given docs are on (for defn/defmacro)."
  [{:keys [whole head doc sig-body] :as forms}]
  (when (seq doc)
    (let [get-line #(:line (meta %))
          first-line (get-line whole)
          before-line (or (get-line (second head))
                          (get-line (first head)))
          after-line (get-line (first sig-body))]
      (when (< before-line after-line)
        {:start-line (-> before-line inc (- first-line))
         :end-line (-> after-line dec (- first-line))
         :forms doc}))))

;;------------------------------------------------------------
;; Form Parsing
;;------------------------------------------------------------

(defn parse-defn-or-macro
  [form]
  (let [docstring (let [ds (nth form 2)]
                    (when (string? ds)
                      ds))
        attr-map (let [m (nth form (if docstring 3 2))]
                   (when (map? m) m))
        doc-forms (cond-> []
                    docstring (conj docstring)
                    attr-map (conj attr-map))
        sig-body-forms (drop (+ 2 (count doc-forms)) form)
        signatures (if (vector? (first sig-body-forms))
                     (take 1 sig-body-forms)
                     (map first sig-body-forms))
        expected-docs (try-locate-docs
                        {:whole form
                         :head (take 2 form)
                         :doc doc-forms
                         :sig-body sig-body-forms})]
    {:expected-docs expected-docs
     :docstring (fix-docstring docstring)
     :signatures signatures}))

(defn parse-def-fn
  [form]
  (let [name- (second form)
        m (meta name-)
        docstring (fix-docstring (:doc m))
        signatures (when-let [arglists (:arglists m)]
                     (when (= 'quote (first arglists))
                       (second arglists)))]
    {:docstring docstring
     :signatures signatures}))

(defmulti parse-form*
  (fn [form]
    (cond
      (and (= 'defn (first form))
           (not (:private (meta (second form)))))
      "defn"

      (and (= 'defmacro (first form))
           (not (:private (meta (second form)))))
      "defmacro"

      (and (= 'def (first form))
           (list? (nth form 2 nil))
           (= 'fn (first (nth form 2 nil)))
           (not (:private (meta (second form)))))
      "def fn"

      :else nil)))

(defmethod parse-form* "def fn"
  [form]
  (assoc (parse-def-fn form) :fn-or-macro "function"))

(defmethod parse-form* "defn"
  [form]
  (assoc (parse-defn-or-macro form) :fn-or-macro "function"))

(defmethod parse-form* "defmacro"
  [form]
  (assoc (parse-defn-or-macro form) :fn-or-macro "macro"))

(defmethod parse-form* nil
  [form]
  nil)

(defn parse-common
  [form ns- repo]
  (let [name- (second form)
        m (meta form)
        lines [(:line m) (:end-line m)]
        num-lines (inc (- (:end-line m) (:line m)))
        source (join "\n" (take-last num-lines (split-lines (:source m))))
        filename (subs (:file m) (inc (count repo-dir)))
        github-link (get-github-file-link repo filename lines)]
    {:ns ns-
     :name name-
     :full-name (str ns- "/" name-)
     :source source
     :filename filename
     :lines lines
     :github-link github-link}))

(defn parse-form
  [form ns- repo]
  (when-let [specific (parse-form* form)]
    (let [common (parse-common form ns- repo)
          merged (merge specific common)
          final (update-in merged [:source] try-remove-docs (:expected-docs specific))]
      final)))

(defn parse-api
  "Parse the functions and macros from the given repo file"
  [ns- repo file]
  (keep #(parse-form % ns- repo) (get-forms ns- repo file)))

(defn get-imported-macro-api
  [ns- repo file macro-api]
  (let [forms (get-forms ns- repo file)
        macro-names (-> (filter #(= 'import-macros (first %)) forms)
                        first (nth 2) set)]
    (filter #(macro-names (:name %)) macro-api)))

;;------------------------------------------------------------
;; Namespace API parsing
;;------------------------------------------------------------

(defmulti parse-ns-api (fn [ns-] ns-))

(defmethod parse-ns-api "cljs.core" [ns-]
  (let [clj-api       (parse-api ns- "clojure"       "core.clj")
        clj-type-api  (parse-api ns- "clojure"       "core_deftype.clj")
        clj-cljs-api  (parse-api ns- "clojurescript" "core.clj")
        cljs-cljs-api (parse-api ns- "clojurescript" "core.cljs")
        import-macro-api (get-imported-macro-api ns- "clojurescript" "core.clj" (concat clj-api clj-type-api))]
    (concat import-macro-api
            clj-cljs-api
            cljs-cljs-api)))

(defmethod parse-ns-api "cljs.reader" [ns-]
  (parse-api ns- "clojurescript" "reader.cljs"))

(defmethod parse-ns-api "clojure.set" [ns-]
  (parse-api ns- "clojurescript" "set.cljs"))

(defmethod parse-ns-api "clojure.string" [ns-]
  (parse-api ns- "clojurescript" "string.cljs"))

(defmethod parse-ns-api "clojure.walk" [ns-]
  (parse-api ns- "clojurescript" "walk.cljs"))

(defmethod parse-ns-api "clojure.zip" [ns-]
  (parse-api ns- "clojurescript" "zip.cljs"))

(defmethod parse-ns-api "clojure.data" [ns-]
  (parse-api ns- "clojurescript" "data.cljs"))

(defmethod parse-ns-api "cljs.core.async" [ns-]
  (parse-api ns- "core.async" "async.cljs"))

(defmethod parse-ns-api "cljs.core.async.macros" [ns-]
  (parse-api ns- "core.async" "macros.clj"))

;;------------------------------------------------------------
;; Doc file writing
;;------------------------------------------------------------

(defn symbol->filename
  [s]
  (-> (name s)
      (replace "." "DOT")
      (replace ">" "GT")
      (replace "<" "LT")
      (replace "!" "BANG")
      (replace "?" "QMARK")
      (replace "*" "STAR")
      (replace "+" "PLUS")
      (replace "/" "SLASH")))

(defn item-filename
  [item]
  (str cljsdoc-dir "/" (:ns item) "_" (symbol->filename (:name item))))

(defn cljsdoc-section
  [title content]
  (when content
    (str "===== " title "\n" content "\n")))

(defn make-cljsdoc
  [item]
  (join "\n"
    (keep identity
      [(cljsdoc-section "Name" (:full-name item))
       (cljsdoc-section "Type" (:fn-or-macro item))
       (cljsdoc-section "Docstring" (:docstring item))
       (cljsdoc-section "Signature" (join "\n" (:signatures item)))
       (cljsdoc-section "Filename" (:filename item))
       (cljsdoc-section "Source" (:source item))
       (cljsdoc-section "Github" (:github-link item))])))

(defn dump-doc-file!
  [item]
  (let [filename (item-filename item)
        cljsdoc-content (make-cljsdoc item)]
    (spit (str filename ".cljsdoc") cljsdoc-content)))

(defn dump-api-docs!
  [api]
  (doseq [item api]
    (dump-doc-file! item)))

;;------------------------------------------------------------
;; Program Entry
;;------------------------------------------------------------

(defn -main
  []
  (try
    ;; Retrieve the SHA1 hashes for the checked out repos (for github links)
    (doseq [repo (keys @repo-sha1)]
      (swap! repo-sha1 assoc repo (get-repo-sha1 repo)))
<commit_msg>add return type in doc generator if present (close #32)<commit_after>(ns core
  (:refer-clojure :exclude [replace])
  (:require
    [clojure.java.io :as io]
    [clojure.java.shell :refer [sh]]
    [clojure.tools.reader :as reader]
    [clojure.tools.reader.reader-types :as readers]
    [clojure.string :refer [split-lines join replace trim]]
    [cljs.tagged-literals :refer [*cljs-data-readers*]]
    [me.raynes.fs :refer [mkdir]]))

;; location of the documents generated by this program
(def cljsdoc-dir "../../docs-generated")

;; location of the clojure & clojurescript repos to parse
(def repo-dir "repos")

;; SHA1 hashes of the checked out commits of the language repos
(def repo-sha1
  (atom {"clojure" nil
         "clojurescript" nil
         "core.async" nil}))

;; Table of namespaces that we will parse
(def cljs-ns-paths
  ; NS                        REPO             FILE               FULL PATH
  {"cljs.core"              {"clojurescript" {"core.cljs"        "src/cljs/cljs/core.cljs"
                                              "core.clj"         "src/clj/cljs/core.clj"}
                             "clojure"       {"core.clj"         "src/clj/clojure/core.clj"
                                              "core_deftype.clj" "src/clj/clojure/core_deftype.clj"}}
   "cljs.reader"            {"clojurescript" {"reader.cljs"      "src/cljs/cljs/reader.cljs"}}
   "clojure.set"            {"clojurescript" {"set.cljs"         "src/cljs/clojure/set.cljs"}}
   "clojure.string"         {"clojurescript" {"string.cljs"      "src/cljs/clojure/string.cljs"}}
   "clojure.walk"           {"clojurescript" {"walk.cljs"        "src/cljs/clojure/walk.cljs"}}
   "clojure.zip"            {"clojurescript" {"zip.cljs"         "src/cljs/clojure/zip.cljs"}}
   "clojure.data"           {"clojurescript" {"data.cljs"        "src/cljs/clojure/data.cljs"}}
   "cljs.core.async"        {"core.async"    {"async.cljs"       "src/main/clojure/cljs/core/async.cljs"}}
   "cljs.core.async.macros" {"core.async"    {"macros.clj"       "src/main/clojure/cljs/core/async/macros.clj"}}})

;;------------------------------------------------------------
;; Form Reading
;;------------------------------------------------------------

(defn read-forms
  [r]
  (loop [forms (transient [])]
    (if-let [f (try (binding [reader/*data-readers* *cljs-data-readers*]
                      (reader/read r))
                    (catch Exception e
                      (when-not (= (.getMessage e) "EOF") (throw e))))]
      (recur (conj! forms f))
      (persistent! forms))))

(defn read-forms-from-file
  [path]
  (let [is (io/input-stream path)
        r1 (readers/input-stream-push-back-reader is)
        r  (readers/source-logging-push-back-reader r1 1 path)]
    (read-forms r)))

(defn read-forms-from-str
  [s]
  (let [r (readers/string-push-back-reader s)]
    (read-forms r)))

;;------------------------------------------------------------
;; Repos
;;------------------------------------------------------------

(defn get-repo-sha1
  [repo]
  (trim (:out (sh "git" "rev-parse" "HEAD" :dir (str repo-dir "/" repo)))))

(defn get-github-file-link
  [repo path [start-line end-line]]
  (let [sha1 (get @repo-sha1 repo)
        strip-path (subs path (inc (count repo)))]
    (str "https://github.com/clojure/" repo "/blob/" sha1 "/" strip-path
         "#L" start-line "-L" end-line)))

;;------------------------------------------------------------
;; Repo Helpers
;;------------------------------------------------------------

(defn get-repo-path
  "Get path to the given repo file"
  [ns- repo file]
  (let [path (get-in cljs-ns-paths [ns- repo file])]
    (str repo-dir "/" repo "/" path)))

(defn get-forms
  "Get forms from the given repo file"
  [ns- repo file]
  (read-forms-from-file (get-repo-path ns- repo file)))

;;------------------------------------------------------------
;; Docstring Helpers
;;------------------------------------------------------------

(defn get-docstring-indent
  [docstring]
  (let [lines (split-lines docstring)]
    (if (> (count lines) 1)
      (let [[first-line & indented-lines] lines
            get-indent-length #(count (re-find #"^ *" %))
            has-content? #(pos? (count (trim %)))]
        (->> indented-lines
             (filter has-content?)
             (map get-indent-length)
             (apply min 3)))
      0)))

(defn fix-docstring
  "Remove indentation from docstring."
  [docstring]
  (when (string? docstring)
    (let [indent-length (get-docstring-indent docstring)]
      (if (zero? indent-length)
        docstring
        (let [[first-line & indented-lines] (split-lines docstring)
              indent (re-pattern (str "^ {" indent-length "}"))
              remove-indent #(replace % indent "")]
          (->> indented-lines
               (map remove-indent)
               (cons first-line)
               (join "\n")))))))

(defn try-remove-docs
  "Try to remove docstring/attr-map from source if they are on their expected lines."
  [source {:keys [start-line end-line forms] :as expected-docs}]
  (if (nil? expected-docs)
    source
    (let [i-lines (map-indexed vector (split-lines source))
          to-str #(join "\n" (map second %))
          doc-line? #(<= start-line (first %) end-line)
          doc-str (to-str (filter doc-line? i-lines))
          actual-forms (read-forms-from-str doc-str)]
      (if (= actual-forms forms)
        (to-str (remove doc-line? i-lines))
        (do
          (binding [*out* *err*]
            (println "=====================================")
            (println "Warning: couldn't remove docstring:")
            (println "expected:" (pr-str forms))
            (println "actual:" (pr-str actual-forms))
            (println "source:" (pr-str source))
            (println "====================================="))
          source)))))

(defn try-locate-docs
  "Try to guess which lines the given docs are on (for defn/defmacro)."
  [{:keys [whole head doc sig-body] :as forms}]
  (when (seq doc)
    (let [get-line #(:line (meta %))
          first-line (get-line whole)
          before-line (or (get-line (second head))
                          (get-line (first head)))
          after-line (get-line (first sig-body))]
      (when (< before-line after-line)
        {:start-line (-> before-line inc (- first-line))
         :end-line (-> after-line dec (- first-line))
         :forms doc}))))

;;------------------------------------------------------------
;; Form Parsing
;;------------------------------------------------------------

(defn parse-defn-or-macro
  [form]
  (let [docstring (let [ds (nth form 2)]
                    (when (string? ds)
                      ds))
        attr-map (let [m (nth form (if docstring 3 2))]
                   (when (map? m) m))
        doc-forms (cond-> []
                    docstring (conj docstring)
                    attr-map (conj attr-map))
        sig-body-forms (drop (+ 2 (count doc-forms)) form)
        signatures (if (vector? (first sig-body-forms))
                     (take 1 sig-body-forms)
                     (map first sig-body-forms))
        expected-docs (try-locate-docs
                        {:whole form
                         :head (take 2 form)
                         :doc doc-forms
                         :sig-body sig-body-forms})]
    {:expected-docs expected-docs
     :docstring (fix-docstring docstring)
     :signatures signatures}))

(defn parse-def-fn
  [form]
  (let [name- (second form)
        m (meta name-)
        docstring (fix-docstring (:doc m))
        signatures (when-let [arglists (:arglists m)]
                     (when (= 'quote (first arglists))
                       (second arglists)))]
    {:docstring docstring
     :signatures signatures}))

(defmulti parse-form*
  (fn [form]
    (cond
      (and (= 'defn (first form))
           (not (:private (meta (second form)))))
      "defn"

      (and (= 'defmacro (first form))
           (not (:private (meta (second form)))))
      "defmacro"

      (and (= 'def (first form))
           (list? (nth form 2 nil))
           (= 'fn (first (nth form 2 nil)))
           (not (:private (meta (second form)))))
      "def fn"

      :else nil)))

(defmethod parse-form* "def fn"
  [form]
  (assoc (parse-def-fn form) :fn-or-macro "function"))

(defmethod parse-form* "defn"
  [form]
  (assoc (parse-defn-or-macro form) :fn-or-macro "function"))

(defmethod parse-form* "defmacro"
  [form]
  (assoc (parse-defn-or-macro form) :fn-or-macro "macro"))

(defmethod parse-form* nil
  [form]
  nil)

(defn parse-common
  [form ns- repo]
  (let [name- (second form)
        return-type (-> name- meta :tag)
        m (meta form)
        lines [(:line m) (:end-line m)]
        num-lines (inc (- (:end-line m) (:line m)))
        source (join "\n" (take-last num-lines (split-lines (:source m))))
        filename (subs (:file m) (inc (count repo-dir)))
        github-link (get-github-file-link repo filename lines)]
    {:ns ns-
     :name name-
     :return-type return-type
     :full-name (str ns- "/" name-)
     :source source
     :filename filename
     :lines lines
     :github-link github-link}))

(defn parse-form
  [form ns- repo]
  (when-let [specific (parse-form* form)]
    (let [common (parse-common form ns- repo)
          merged (merge specific common)
          final (update-in merged [:source] try-remove-docs (:expected-docs specific))]
      final)))

(defn parse-api
  "Parse the functions and macros from the given repo file"
  [ns- repo file]
  (keep #(parse-form % ns- repo) (get-forms ns- repo file)))

(defn get-imported-macro-api
  [ns- repo file macro-api]
  (let [forms (get-forms ns- repo file)
        macro-names (-> (filter #(= 'import-macros (first %)) forms)
                        first (nth 2) set)]
    (filter #(macro-names (:name %)) macro-api)))

;;------------------------------------------------------------
;; Namespace API parsing
;;------------------------------------------------------------

(defmulti parse-ns-api (fn [ns-] ns-))

(defmethod parse-ns-api "cljs.core" [ns-]
  (let [clj-api       (parse-api ns- "clojure"       "core.clj")
        clj-type-api  (parse-api ns- "clojure"       "core_deftype.clj")
        clj-cljs-api  (parse-api ns- "clojurescript" "core.clj")
        cljs-cljs-api (parse-api ns- "clojurescript" "core.cljs")
        import-macro-api (get-imported-macro-api ns- "clojurescript" "core.clj" (concat clj-api clj-type-api))]
    (concat import-macro-api
            clj-cljs-api
            cljs-cljs-api)))

(defmethod parse-ns-api "cljs.reader" [ns-]
  (parse-api ns- "clojurescript" "reader.cljs"))

(defmethod parse-ns-api "clojure.set" [ns-]
  (parse-api ns- "clojurescript" "set.cljs"))

(defmethod parse-ns-api "clojure.string" [ns-]
  (parse-api ns- "clojurescript" "string.cljs"))

(defmethod parse-ns-api "clojure.walk" [ns-]
  (parse-api ns- "clojurescript" "walk.cljs"))

(defmethod parse-ns-api "clojure.zip" [ns-]
  (parse-api ns- "clojurescript" "zip.cljs"))

(defmethod parse-ns-api "clojure.data" [ns-]
  (parse-api ns- "clojurescript" "data.cljs"))

(defmethod parse-ns-api "cljs.core.async" [ns-]
  (parse-api ns- "core.async" "async.cljs"))

(defmethod parse-ns-api "cljs.core.async.macros" [ns-]
  (parse-api ns- "core.async" "macros.clj"))

;;------------------------------------------------------------
;; Doc file writing
;;------------------------------------------------------------

(defn symbol->filename
  [s]
  (-> (name s)
      (replace "." "DOT")
      (replace ">" "GT")
      (replace "<" "LT")
      (replace "!" "BANG")
      (replace "?" "QMARK")
      (replace "*" "STAR")
      (replace "+" "PLUS")
      (replace "/" "SLASH")))

(defn item-filename
  [item]
  (str cljsdoc-dir "/" (:ns item) "_" (symbol->filename (:name item))))

(defn cljsdoc-section
  [title content]
  (when content
    (str "===== " title "\n" content "\n")))

(defn make-cljsdoc
  [item]
  (join "\n"
    (keep identity
      [(cljsdoc-section "Name" (:full-name item))
       (cljsdoc-section "Type" (:fn-or-macro item))
       (cljsdoc-section "Return" (:return-type item))
       (cljsdoc-section "Docstring" (:docstring item))
       (cljsdoc-section "Signature" (join "\n" (:signatures item)))
       (cljsdoc-section "Filename" (:filename item))
       (cljsdoc-section "Source" (:source item))
       (cljsdoc-section "Github" (:github-link item))])))

(defn dump-doc-file!
  [item]
  (let [filename (item-filename item)
        cljsdoc-content (make-cljsdoc item)]
    (spit (str filename ".cljsdoc") cljsdoc-content)))

(defn dump-api-docs!
  [api]
  (doseq [item api]
    (dump-doc-file! item)))

;;------------------------------------------------------------
;; Program Entry
;;------------------------------------------------------------

(defn -main
  []
  (try
    ;; Retrieve the SHA1 hashes for the checked out repos (for github links)
    (doseq [repo (keys @repo-sha1)]
      (swap! repo-sha1 assoc repo (get-repo-sha1 repo)))

{'fraction_non_alphanumeric': 0.23289946313390752, 'fraction_numerical': 0.0020084199142559192, 'mean_word_length': 1.773944718234412, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 30, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '490867', 'n_tokens_mistral': 8552, 'n_tokens_neox': 8314, 'n_words': 2087}