<commit_before>(ns zest.settings
  (:require-macros [cljs.core.async.macros :refer [go go-loop]])
  (:require [cljs.core.async :as async]
            [reagent.core :as reagent]
            [zest.docs.registry]
            [zest.docs.devdocs]))

(def visible (reagent/atom false))
(def devdocs-visible (reagent/atom false))
(def devdocs-reindexing (reagent/atom false))
(def devdocs-reindexing-progress (reagent/atom ""))
(def so-visible (reagent/atom false))
(def downloading (reagent/atom {}))
(def so-indexing (reagent/atom false))
(def so-downloading (reagent/atom false))
(def so-download-progress (reagent/atom 0))
(def so-download-total (reagent/atom 0))
(def so-download-peers (reagent/atom 0))
(def so-dl-speed (reagent/atom 0))
(def so-ul-speed (reagent/atom 0))
(def so-grep-progress (reagent/atom nil))
(def so-index-progress (reagent/atom nil))
(def so-index-tags (reagent/atom #{}))
(def so-archives-total (reagent/atom nil))
(def so-archives-available
  (let [fs (.require js/window "fs")
        path (.require js/window "path")
        done-filename (.join path (zest.docs.registry/get-so-root)
                             "archive" "zest_done")]
    (reagent/atom
      (and
        (.existsSync fs done-filename)
        (.isFile (.statSync fs done-filename))))))

(defn async-add-file [idx name contents]
  (let [ret (async/chan)]
    (.setImmediate
      js/window #(go (.addFile idx name contents)
                     (async/>! ret true)))
    ret))

(defn devdocs-rebuild-index []
  (reset! devdocs-reindexing true)
  (reset! devdocs-reindexing-progress "")
  (let [path (.require js/window "path")
        rimraf (.require js/window "rimraf")
        fs (.require js/window "fs-extra")
        LuceneIndex
        (.-LuceneIndex (.require js/window "nodelucene"))
        idx (let [new-lucene-path (.join path
                                         (zest.docs.registry/get-devdocs-root)
                                         "new_lucene")]
              (.sync rimraf new-lucene-path)
              (LuceneIndex. new-lucene-path))
        docs-count (count @zest.docs.registry/installed-devdocs-atom)
        all-count (if (> docs-count 0)
                    (inc docs-count) 0)
        indexed-count (atom 0)

        check-done
        (fn []
          (reset! indexed-count (inc @indexed-count))
          (if (= @indexed-count all-count)
            (do (reset! devdocs-reindexing false)
                (.endWriting idx)
                (.sync rimraf (.join path (zest.docs.registry/get-devdocs-root) "lucene"))
                (.move
                  fs
                  (.join path (zest.docs.registry/get-devdocs-root) "new_lucene")
                  (.join path (zest.docs.registry/get-devdocs-root) "lucene")
                  (fn []))
                (.close
                  @zest.core/symbol-db
                  (fn []
                    (.sync rimraf (.join path (zest.docs.registry/get-devdocs-root) "symbols"))
                    (.move
                      fs
                      (.join path (zest.docs.registry/get-devdocs-root) "new_symbols")
                      (.join path (zest.docs.registry/get-devdocs-root) "symbols")
                      (fn []
                        (zest.core/open-symbol-db
                          #(zest.docs.registry/update-installed-devdocs))))))) ; update query results
            (reset! devdocs-reindexing-progress
                    (str @indexed-count "/" all-count))))]
    (if (> all-count 0)
      (do
        (go
          (async/<! (zest.core/rebuild-symbol-db))
          (check-done))
        (.startWriting idx)
        (doall
          (for [docset @zest.docs.registry/installed-devdocs-atom]
            (do
              (zest.docs.devdocs/get-from-cache docset)
              (let [db-cache (aget zest.docs.devdocs/docset-db-cache docset)
                    keys (.keys js/Object db-cache)
                    i (atom 0)]
                (go-loop []
                         (if (< @i (.-length keys))
                           (do (async/<! (async-add-file
                                           idx
                                           (str docset "/" (nth keys @i))
                                           (zest.docs.stackoverflow/unfluff
                                             (aget db-cache (nth keys @i)))))
                               (reset! i (inc @i))
                               (recur))
                           (check-done))))))))
      (reset! devdocs-reindexing false))))


(def
  SO-URL
  "https://archive.org/download/stackexchange/stackexchange_archive.torrent")

(def suggested-tags
  ["javascript"
   "java"
   "c#"
   "php"
   "android"
   "jquery"
   "python"
   "django"
   "html"
   "c++"
   "ios"
   "objective-c"
   "clojure"])

(defn start-so-download []
  (let [WebTorrent (.require js/window "webtorrent")
        fs (.require js/window "fs")
        path (.require js/window "path")
        client (WebTorrent.)
        archive-dir (.join path (zest.docs.registry/get-so-root) "archive")]
    (reset! so-downloading true)
    (.add
      client
      SO-URL
      (js-obj "path" archive-dir)
      (fn [torrent]
        (.pause torrent)
        ;; torrent.deselect is broken - see https://github.com/feross/webtorrent/issues/164#issuecomment-107915164
        (aset torrent "_selections" (make-array 0))
        (doall
          (for [f (.-files torrent)]
            (if
              (contains?
                #{"stackoverflow.com-Posts.7z"
                  "stackoverflow.com-Comments.7z"
                  "stackoverflow.com-Users.7z"}
                (.-name f))
              (do (.select f)
                  (reset! so-download-total
                          (+ @so-download-total
                             (.-length f)))))))
        (.resume torrent)
        (.on torrent "wire"
             (fn []
               (reset! so-download-peers
                       (count (.-wires (.-swarm torrent))))))
        (.on torrent "download"
             (fn [chunkSize]
               (reset! so-download-progress
                       (.round js/Math (.-downloaded torrent)))
               (reset! so-dl-speed
                       (.round js/Math (/ (.-downloadSpeed torrent) 1000)))
               (reset! so-ul-speed
                       (.round js/Math (/ (.-uploadSpeed torrent) 1000)))))
        (.on torrent "done"
             (fn []
               (.writeFileSync
                 fs
                 (.join path archive-dir "zest_done")
                 "")
               (reset! so-archives-available true)))))))

(defn run-so-extractor [additional-args]
  (let [child-process (.require js/window "child_process")
        path (.require js/window "path")]
    (.spawn
      child-process
      (zest.core/get-binary-path "extractor")
      (.concat
        (array
          (.join path (zest.docs.registry/get-so-root)
                 "archive" "stackexchange" (str "stackoverflow.com-Posts.7z"))
          (.join path (zest.docs.registry/get-so-root)
                 "archive" "stackexchange" (str "stackoverflow.com-Comments.7z"))
          (.join path (zest.docs.registry/get-so-root)
                 "archive" "stackexchange" (str "stackoverflow.com-Users.7z")))
        additional-args)
      (js-obj
        "env" (js-obj
                "PATH"
                (.dirname path (.dirname path (.-__dirname js/window))))))))

(defn get-so-archive-sizes-sum []
  (let [ret (async/chan)
        ret-str (atom "")
        process (run-so-extractor (array "sizes"))]
    (.on (.-stdout process) "data"
         #(reset! ret-str (+ @ret-str (.toString % "utf8"))))
    (.on (.-stdout process) "end"
         #(let [vals (.split @ret-str " ")]
           (go (async/>! ret (+ (.parseFloat js/window (nth vals 0))
                                (.parseFloat js/window (nth vals 1))
                                (.parseFloat js/window (nth vals 2)))))))
    ret))

(defn start-so-indexing []
  (reset! so-index-progress 0)
  (let [levelup (.require js/window "levelup")
        path (.require js/window "path")
        fs (.require js/window "fs-extra")
        rimraf (.require js/window "rimraf")
        db (levelup.
             (.join path (zest.docs.registry/get-so-root) "new_index" "leveldb"))
        rStream (.createReadStream
                  db
                  (js-obj "gt" "p_"
                          "lt" "p_a"))

        LuceneIndex
        (.-LuceneIndex (.require js/window "nodelucene"))
        idx (LuceneIndex. (.join path (zest.docs.registry/get-so-root)
                                 "new_index" "lucene"))

        render-answer-blob
        (fn [data]
          (str (.-Title data) " "
               (zest.docs.stackoverflow/unfluff (.-Body data))
               (apply str (.map (.-comments data) #(str " " (.-Text %))))))

        render-blob
        (fn [data]
          (str (.-Title data) " "
               (zest.docs.stackoverflow/unfluff (.-Body data))
               (apply str (.map (.-comments data)
                                #(str " " (.-Text %))))
               (apply str (.map (.-answers data)
                                #(str " " (render-answer-blob %))))))

        done (atom 0)
        started (atom 0)
        ended (atom false)
        check-all-done
        (fn []
          (if (and @ended (= @started @done))
            (do
              (.endWriting idx)
              (zest.searcher/stop-all-searchers)
              (.close
                @zest.core/so-db
                (fn []
                  (reset! zest.core/so-db db)
                  (.sync rimraf (.join path (zest.docs.registry/get-so-root) "lucene"))
                  (.sync rimraf (.join path (zest.docs.registry/get-so-root) "leveldb"))
                  (.move
                    fs
                    (.join path (zest.docs.registry/get-so-root) "new_index" "symbols")
                    (.join path (zest.docs.registry/get-so-root) "lucene")
                    (fn []))
                  (.move
                    fs
                    (.join path (zest.docs.registry/get-so-root) "new_index" "lucene")
                    (.join path (zest.docs.registry/get-so-root) "lucene")
                    (fn []))
                  (.move
                    fs
                    (.join path (zest.docs.registry/get-so-root) "new_index" "leveldb")
                    (.join path (zest.docs.registry/get-so-root) "leveldb")
                    (fn []))
                  (reset! so-indexing false))))))]
    (.startWriting idx)
    (.on rStream "data"
         (fn [v]
           (let [data (.parse js/JSON (.-value v))]
             (reset! started (+ @started 1))
             (go (.addFile
                   idx
                   (str (.-Id data) ";" (.-Title data))
                   (render-blob
                     (async/<!
                       (zest.docs.stackoverflow/process-so-post data false))))
                 (reset! done (+ @done 1))
                 (if (= 0 (mod @done 15))
                   (reset! so-index-progress @done))
                 (check-all-done)))))
    (.on rStream "end"
         (fn []
           (reset! ended true)
           (check-all-done)))))

(defn start-so-grepping []
  (let [mkdirp (.require js/window "mkdirp")
        rimraf (.require js/window "rimraf")
        path (.require js/window "path")]
    (.sync rimraf (.join path (zest.docs.registry/get-so-root) "new_index"))
    (.sync mkdirp (.join path (zest.docs.registry/get-so-root) "new_index"))
    (go (let [so-archives-total-val (async/<! (get-so-archive-sizes-sum))
              child-process (.require js/window "child_process")
              Lazy (.require js/window "lazy")
              sogrep (if (= (.-platform js/process) "linux")
                       (let [shell-escape (js/require "shell-escape")]
                         (.spawn child-process
                                 "sh"
                                 (array "-c"
                                        (str (shell-escape
                                               (array
                                                 (zest.core/get-binary-path "extractor")
                                                 (.join path (zest.docs.registry/get-so-root)
                                                        "archive" "stackexchange" (str "stackoverflow.com-Posts.7z"))
                                                 (.join path (zest.docs.registry/get-so-root)
                                                        "archive" "stackexchange" (str "stackoverflow.com-Comments.7z"))
                                                 (.join path (zest.docs.registry/get-so-root)
                                                        "archive" "stackexchange" (str "stackoverflow.com-Users.7z"))))
                                             " | "
                                             (shell-escape
                                               (.concat (array (zest.core/get-binary-path "sogrep"))
                                                        (apply array @so-index-tags)))))
                                 (js-obj
                                   "cwd"
                                   (.join path
                                          (zest.docs.registry/get-so-root)
                                          "new_index"))))
                       (.spawn child-process
                               (zest.core/get-binary-path "sogrep")
                               (apply array @so-index-tags)
                               "env" (js-obj
                                       "PATH"
                                       (.dirname path (.dirname path (.-__dirname js/window))))
                               "cwd"
                               (.join path
                                      (zest.docs.registry/get-so-root)
                                      "new_index")))
              extractor (if (= (.-platform js/process) "linux")
                          nil
                          (run-so-extractor (array)))]
          (if (not (nil? extractor))
            (do
              (.pipe (.-stdout extractor)
                     (.-stdin sogrep))
              (.on extractor "close" #(.end (.-stdin sogrep)))))

          (reset! so-archives-total so-archives-total-val)
          (reset! so-indexing true)
          (reset! so-grep-progress 0)
          (.on sogrep "close"
               (fn [code]
                 (.log js/console "sogrep exited with code" code)
                 (if (= 0 code) (start-so-indexing))))
          (.forEach (.-lines (Lazy. (.-stdout sogrep)))
                    (fn [line]
                      (.log js/console (.toString line "utf8"))
                      (reset! so-grep-progress<commit_msg>Fixed spawn call for Windows and OS X<commit_after>(ns zest.settings
  (:require-macros [cljs.core.async.macros :refer [go go-loop]])
  (:require [cljs.core.async :as async]
            [reagent.core :as reagent]
            [zest.docs.registry]
            [zest.docs.devdocs]))

(def visible (reagent/atom false))
(def devdocs-visible (reagent/atom false))
(def devdocs-reindexing (reagent/atom false))
(def devdocs-reindexing-progress (reagent/atom ""))
(def so-visible (reagent/atom false))
(def downloading (reagent/atom {}))
(def so-indexing (reagent/atom false))
(def so-downloading (reagent/atom false))
(def so-download-progress (reagent/atom 0))
(def so-download-total (reagent/atom 0))
(def so-download-peers (reagent/atom 0))
(def so-dl-speed (reagent/atom 0))
(def so-ul-speed (reagent/atom 0))
(def so-grep-progress (reagent/atom nil))
(def so-index-progress (reagent/atom nil))
(def so-index-tags (reagent/atom #{}))
(def so-archives-total (reagent/atom nil))
(def so-archives-available
  (let [fs (.require js/window "fs")
        path (.require js/window "path")
        done-filename (.join path (zest.docs.registry/get-so-root)
                             "archive" "zest_done")]
    (reagent/atom
      (and
        (.existsSync fs done-filename)
        (.isFile (.statSync fs done-filename))))))

(defn async-add-file [idx name contents]
  (let [ret (async/chan)]
    (.setImmediate
      js/window #(go (.addFile idx name contents)
                     (async/>! ret true)))
    ret))

(defn devdocs-rebuild-index []
  (reset! devdocs-reindexing true)
  (reset! devdocs-reindexing-progress "")
  (let [path (.require js/window "path")
        rimraf (.require js/window "rimraf")
        fs (.require js/window "fs-extra")
        LuceneIndex
        (.-LuceneIndex (.require js/window "nodelucene"))
        idx (let [new-lucene-path (.join path
                                         (zest.docs.registry/get-devdocs-root)
                                         "new_lucene")]
              (.sync rimraf new-lucene-path)
              (LuceneIndex. new-lucene-path))
        docs-count (count @zest.docs.registry/installed-devdocs-atom)
        all-count (if (> docs-count 0)
                    (inc docs-count) 0)
        indexed-count (atom 0)

        check-done
        (fn []
          (reset! indexed-count (inc @indexed-count))
          (if (= @indexed-count all-count)
            (do (reset! devdocs-reindexing false)
                (.endWriting idx)
                (.sync rimraf (.join path (zest.docs.registry/get-devdocs-root) "lucene"))
                (.move
                  fs
                  (.join path (zest.docs.registry/get-devdocs-root) "new_lucene")
                  (.join path (zest.docs.registry/get-devdocs-root) "lucene")
                  (fn []))
                (.close
                  @zest.core/symbol-db
                  (fn []
                    (.sync rimraf (.join path (zest.docs.registry/get-devdocs-root) "symbols"))
                    (.move
                      fs
                      (.join path (zest.docs.registry/get-devdocs-root) "new_symbols")
                      (.join path (zest.docs.registry/get-devdocs-root) "symbols")
                      (fn []
                        (zest.core/open-symbol-db
                          #(zest.docs.registry/update-installed-devdocs))))))) ; update query results
            (reset! devdocs-reindexing-progress
                    (str @indexed-count "/" all-count))))]
    (if (> all-count 0)
      (do
        (go
          (async/<! (zest.core/rebuild-symbol-db))
          (check-done))
        (.startWriting idx)
        (doall
          (for [docset @zest.docs.registry/installed-devdocs-atom]
            (do
              (zest.docs.devdocs/get-from-cache docset)
              (let [db-cache (aget zest.docs.devdocs/docset-db-cache docset)
                    keys (.keys js/Object db-cache)
                    i (atom 0)]
                (go-loop []
                         (if (< @i (.-length keys))
                           (do (async/<! (async-add-file
                                           idx
                                           (str docset "/" (nth keys @i))
                                           (zest.docs.stackoverflow/unfluff
                                             (aget db-cache (nth keys @i)))))
                               (reset! i (inc @i))
                               (recur))
                           (check-done))))))))
      (reset! devdocs-reindexing false))))


(def
  SO-URL
  "https://archive.org/download/stackexchange/stackexchange_archive.torrent")

(def suggested-tags
  ["javascript"
   "java"
   "c#"
   "php"
   "android"
   "jquery"
   "python"
   "django"
   "html"
   "c++"
   "ios"
   "objective-c"
   "clojure"])

(defn start-so-download []
  (let [WebTorrent (.require js/window "webtorrent")
        fs (.require js/window "fs")
        path (.require js/window "path")
        client (WebTorrent.)
        archive-dir (.join path (zest.docs.registry/get-so-root) "archive")]
    (reset! so-downloading true)
    (.add
      client
      SO-URL
      (js-obj "path" archive-dir)
      (fn [torrent]
        (.pause torrent)
        ;; torrent.deselect is broken - see https://github.com/feross/webtorrent/issues/164#issuecomment-107915164
        (aset torrent "_selections" (make-array 0))
        (doall
          (for [f (.-files torrent)]
            (if
              (contains?
                #{"stackoverflow.com-Posts.7z"
                  "stackoverflow.com-Comments.7z"
                  "stackoverflow.com-Users.7z"}
                (.-name f))
              (do (.select f)
                  (reset! so-download-total
                          (+ @so-download-total
                             (.-length f)))))))
        (.resume torrent)
        (.on torrent "wire"
             (fn []
               (reset! so-download-peers
                       (count (.-wires (.-swarm torrent))))))
        (.on torrent "download"
             (fn [chunkSize]
               (reset! so-download-progress
                       (.round js/Math (.-downloaded torrent)))
               (reset! so-dl-speed
                       (.round js/Math (/ (.-downloadSpeed torrent) 1000)))
               (reset! so-ul-speed
                       (.round js/Math (/ (.-uploadSpeed torrent) 1000)))))
        (.on torrent "done"
             (fn []
               (.writeFileSync
                 fs
                 (.join path archive-dir "zest_done")
                 "")
               (reset! so-archives-available true)))))))

(defn run-so-extractor [additional-args]
  (let [child-process (.require js/window "child_process")
        path (.require js/window "path")]
    (.spawn
      child-process
      (zest.core/get-binary-path "extractor")
      (.concat
        (array
          (.join path (zest.docs.registry/get-so-root)
                 "archive" "stackexchange" (str "stackoverflow.com-Posts.7z"))
          (.join path (zest.docs.registry/get-so-root)
                 "archive" "stackexchange" (str "stackoverflow.com-Comments.7z"))
          (.join path (zest.docs.registry/get-so-root)
                 "archive" "stackexchange" (str "stackoverflow.com-Users.7z")))
        additional-args)
      (js-obj
        "env" (js-obj
                "PATH"
                (.dirname path (.dirname path (.-__dirname js/window))))))))

(defn get-so-archive-sizes-sum []
  (let [ret (async/chan)
        ret-str (atom "")
        process (run-so-extractor (array "sizes"))]
    (.on (.-stdout process) "data"
         #(reset! ret-str (+ @ret-str (.toString % "utf8"))))
    (.on (.-stdout process) "end"
         #(let [vals (.split @ret-str " ")]
           (go (async/>! ret (+ (.parseFloat js/window (nth vals 0))
                                (.parseFloat js/window (nth vals 1))
                                (.parseFloat js/window (nth vals 2)))))))
    ret))

(defn start-so-indexing []
  (reset! so-index-progress 0)
  (let [levelup (.require js/window "levelup")
        path (.require js/window "path")
        fs (.require js/window "fs-extra")
        rimraf (.require js/window "rimraf")
        db (levelup.
             (.join path (zest.docs.registry/get-so-root) "new_index" "leveldb"))
        rStream (.createReadStream
                  db
                  (js-obj "gt" "p_"
                          "lt" "p_a"))

        LuceneIndex
        (.-LuceneIndex (.require js/window "nodelucene"))
        idx (LuceneIndex. (.join path (zest.docs.registry/get-so-root)
                                 "new_index" "lucene"))

        render-answer-blob
        (fn [data]
          (str (.-Title data) " "
               (zest.docs.stackoverflow/unfluff (.-Body data))
               (apply str (.map (.-comments data) #(str " " (.-Text %))))))

        render-blob
        (fn [data]
          (str (.-Title data) " "
               (zest.docs.stackoverflow/unfluff (.-Body data))
               (apply str (.map (.-comments data)
                                #(str " " (.-Text %))))
               (apply str (.map (.-answers data)
                                #(str " " (render-answer-blob %))))))

        done (atom 0)
        started (atom 0)
        ended (atom false)
        check-all-done
        (fn []
          (if (and @ended (= @started @done))
            (do
              (.endWriting idx)
              (zest.searcher/stop-all-searchers)
              (.close
                @zest.core/so-db
                (fn []
                  (reset! zest.core/so-db db)
                  (.sync rimraf (.join path (zest.docs.registry/get-so-root) "lucene"))
                  (.sync rimraf (.join path (zest.docs.registry/get-so-root) "leveldb"))
                  (.move
                    fs
                    (.join path (zest.docs.registry/get-so-root) "new_index" "symbols")
                    (.join path (zest.docs.registry/get-so-root) "lucene")
                    (fn []))
                  (.move
                    fs
                    (.join path (zest.docs.registry/get-so-root) "new_index" "lucene")
                    (.join path (zest.docs.registry/get-so-root) "lucene")
                    (fn []))
                  (.move
                    fs
                    (.join path (zest.docs.registry/get-so-root) "new_index" "leveldb")
                    (.join path (zest.docs.registry/get-so-root) "leveldb")
                    (fn []))
                  (reset! so-indexing false))))))]
    (.startWriting idx)
    (.on rStream "data"
         (fn [v]
           (let [data (.parse js/JSON (.-value v))]
             (reset! started (+ @started 1))
             (go (.addFile
                   idx
                   (str (.-Id data) ";" (.-Title data))
                   (render-blob
                     (async/<!
                       (zest.docs.stackoverflow/process-so-post data false))))
                 (reset! done (+ @done 1))
                 (if (= 0 (mod @done 15))
                   (reset! so-index-progress @done))
                 (check-all-done)))))
    (.on rStream "end"
         (fn []
           (reset! ended true)
           (check-all-done)))))

(defn start-so-grepping []
  (let [mkdirp (.require js/window "mkdirp")
        rimraf (.require js/window "rimraf")
        path (.require js/window "path")]
    (.sync rimraf (.join path (zest.docs.registry/get-so-root) "new_index"))
    (.sync mkdirp (.join path (zest.docs.registry/get-so-root) "new_index"))
    (go (let [so-archives-total-val (async/<! (get-so-archive-sizes-sum))
              child-process (.require js/window "child_process")
              Lazy (.require js/window "lazy")
              sogrep (if (= (.-platform js/process) "linux")
                       (let [shell-escape (js/require "shell-escape")]
                         (.spawn child-process
                                 "sh"
                                 (array "-c"
                                        (str (shell-escape
                                               (array
                                                 (zest.core/get-binary-path "extractor")
                                                 (.join path (zest.docs.registry/get-so-root)
                                                        "archive" "stackexchange" (str "stackoverflow.com-Posts.7z"))
                                                 (.join path (zest.docs.registry/get-so-root)
                                                        "archive" "stackexchange" (str "stackoverflow.com-Comments.7z"))
                                                 (.join path (zest.docs.registry/get-so-root)
                                                        "archive" "stackexchange" (str "stackoverflow.com-Users.7z"))))
                                             " | "
                                             (shell-escape
                                               (.concat (array (zest.core/get-binary-path "sogrep"))
                                                        (apply array @so-index-tags)))))
                                 (js-obj
                                   "cwd"
                                   (.join path
                                          (zest.docs.registry/get-so-root)
                                          "new_index"))))
                       (.spawn child-process
                               (zest.core/get-binary-path "sogrep")
                               (apply array @so-index-tags)
                               (js-obj "env" (js-obj
                                               "PATH"
                                               (.dirname path (.dirname path (.-__dirname js/window))))
                                       "cwd"
                                       (.join path
                                              (zest.docs.registry/get-so-root)
                                              "new_index"))))
              extractor (if (= (.-platform js/process) "linux")
                          nil
                          (run-so-extractor (array)))]
          (if (not (nil? extractor))
            (do
              (.pipe (.-stdout extractor)
                     (.-stdin sogrep))
              (.on extractor "close" #(.end (.-stdin sogrep)))))

          (reset! so-archives-total so-archives-total-val)
          (reset! so-indexing true)
          (reset! so-grep-progress 0)
          (.on sogrep "close"
               (fn [code]
                 (.log js/console "sogrep exited with code" code)
                 (if (= 0 code) (start-so-indexing))))
          (.forEach (.-lines (Lazy. (.-stdout sogrep)))
                    (fn [line]
                      (.log js/console (.toString line "utf8"))
                      (reset! so-grep-progress
{'fraction_non_alphanumeric': 0.13762392875147034, 'fraction_numerical': 0.0036968576709796672, 'mean_word_length': 1.2176181249068416, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 140, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6245891', 'n_tokens_mistral': 8786, 'n_tokens_neox': 8181, 'n_words': 1847}