<commit_before>#!/usr/bin/env boot

;;
;; Boot related scafolding.
(set-env!
  :version "3.12-SNAPSHOT"
  :license {"Apache 2.0" "http://www.apache.org/licenses/LICENSE-2.0.txt"}
  :edition "community"

  :dependencies '[[org.clojure/clojure "1.8.0"]
                  [sixsq/build-utils "0.1.4" :scope "test"]])

(require '[sixsq.build-fns :refer [merge-defaults
                                   sixsq-nexus-url]])
(set-env!
  :repositories
  #(reduce conj % [["sixsq" {:url (sixsq-nexus-url)}]])

  :dependencies
  #(vec (concat %
                (merge-defaults
                  ['sixsq/default-deps (get-env :version)]
                  '[[org.clojure/clojure]
                    [org.clojure/data.xml]
                    [com.sixsq.slipstream/SlipStreamPersistence]
                    [com.sixsq.slipstream/SlipStreamCljResources-jar]
                    [superstring]
                    [me.raynes/fs]
                    [clj-http]]))))
(set-env!
  :resource-paths #{"src"})
(require '[boot.cli :refer [defclifn]])
;; Boot end.

(require '[com.sixsq.slipstream.db.serializers.service-config-impl :as sci])
(require '[com.sixsq.slipstream.db.serializers.service-config-util :as scu])
(require '[me.raynes.fs :as fs])
(require '[clojure.string :as s])
(require '[clj-http.client :as http])

;;
;; Dynamic vars.
(def ^:dynamic *c-names* #{})
(def ^:dynamic *cfg-path-url* nil)
(def ^:dynamic *creds* nil)

(defn fwrite
  [o fpath]
  (let [f (fs/expand-home fpath)]
    (with-open [^java.io.Writer w (apply clojure.java.io/writer f {})]
      (clojure.pprint/pprint o w))))

(defn ->config-resource
  [url]
  (str url "/configuration"))

(defn conf-xml
  [path-url]
  (if (s/starts-with? path-url "https")
    (-> path-url
        ->config-resource
        (http/get {:follow-redirects false
                   :accept           :xml
                   :basic-auth       *creds*})
        :body)
    (slurp path-url)))

(defn run
  []
  (let [sc (-> *cfg-path-url* conf-xml scu/conf-xml->sc)]
    (doseq [[ckey [vals desc]] (sci/sc->connectors sc *c-names*)]
      (let [cname (name ckey)]
        (println "Saving connector:" cname)
        (fwrite vals (format "connector-%s.edn" cname))
        (fwrite desc (format "connector-%s-desc.edn" cname))))))

(def usage "Usage -x <path or URL> [-s <user:pass>] [-c <connector name>]")

(defn error-usage
  [& msg]
  (println (apply str msg))
  (println usage)
  (System/exit 1))

(defclifn -main<commit_msg>WIP: connector-dump CLI with corresponding changes and refactor of the code.<commit_after>#!/usr/bin/env boot

;;
;; Boot related scafolding.
(set-env!
  :version "3.12-SNAPSHOT"
  :license {"Apache 2.0" "http://www.apache.org/licenses/LICENSE-2.0.txt"}
  :edition "community"

  :dependencies '[[org.clojure/clojure "1.8.0"]
                  [sixsq/build-utils "0.1.4" :scope "test"]])

(require '[sixsq.build-fns :refer [merge-defaults
                                   sixsq-nexus-url]])
(set-env!
  :repositories
  #(reduce conj % [["sixsq" {:url (sixsq-nexus-url)}]])

  :dependencies
  #(vec (concat %
                (merge-defaults
                  ['sixsq/default-deps (get-env :version)]
                  '[[org.clojure/clojure]
                    [org.clojure/data.xml]
                    [com.sixsq.slipstream/SlipStreamPersistence]
                    [com.sixsq.slipstream/SlipStreamCljResources-jar]
                    [superstring]
                    [me.raynes/fs]
                    [clj-http]]))))
(set-env!
  :resource-paths #{"src"})
(require '[boot.cli :refer [defclifn]])
;; Boot end.

(require '[com.sixsq.slipstream.db.serializers.service-config-impl :as sci])
(require '[com.sixsq.slipstream.db.serializers.service-config-util :as scu])
(require '[me.raynes.fs :as fs])
(require '[clojure.string :as s])
(require '[clj-http.client :as http])

;;
;; Dynamic vars.
(def ^:dynamic *c-names* #{})
(def ^:dynamic *cfg-path-url* nil)
(def ^:dynamic *creds* nil)

(defn fwrite
  [o fpath]
  (let [f (fs/expand-home fpath)]
    (with-open [^java.io.Writer w (apply clojure.java.io/writer f {})]
      (clojure.pprint/pprint o w))))

(defn ->config-resource
  [url]
  (str url "/configuration"))

(defn conf-xml
  [path-url]
  (if (s/starts-with? path-url "https")
    (-> path-url
        ->config-resource
        (http/get {:follow-redirects false
                   :accept           :xml
                   :basic-auth       *creds*})
        :body)
    (slurp path-url)))

(defn run
  []
  (let [sc (-> *cfg-path-url* conf-xml scu/conf-xml->sc)]
    (doseq [[ckey [vals desc]] (sci/sc->connectors sc *c-names*)]
      (let [cname (name ckey)]
        (println "Saving connector:" cname)
        (fwrite vals (format "connector-%s.edn" cname))
        (fwrite desc (format "connector-%s-desc.edn" cname))))))

(def usage "Usage: -x <path or URL> [-s <user:pass>] [-c <connector name>]")

(defn error-usage
  [& msg]
  (println (apply str msg))
  (println usage)
  (System/exit 1))

(defclifn -main
{'fraction_non_alphanumeric': 0.17668269230769232, 'fraction_numerical': 0.005608974358974359, 'mean_word_length': 2.362289562289562, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7647251', 'n_tokens_mistral': 1865, 'n_tokens_neox': 1808, 'n_words': 385}