<commit_before>(ns pho.core
  (:require [pantomime.mime :refer [mime-type-of]])
  (:require [aws.sdk.s3 :as s3])
  (:require [clojure.tools.cli :refer [cli]])
  (:import com.amazonaws.services.cloudfront.AmazonCloudFrontClient
           com.amazonaws.auth.BasicAWSCredentials
           com.amazonaws.ClientConfiguration
           com.amazonaws.services.cloudfront.model.ListDistributionsRequest
           com.amazonaws.services.cloudfront.model.ListDistributionsResult
           com.amazonaws.services.cloudfront.model.DistributionList
           com.amazonaws.services.cloudfront.model.DistributionSummary
           com.amazonaws.services.cloudfront.model.CreateInvalidationRequest
           com.amazonaws.services.cloudfront.model.CreateInvalidationResult
           com.amazonaws.services.cloudfront.model.Invalidation
           com.amazonaws.services.cloudfront.model.InvalidationBatch
           com.amazonaws.services.cloudfront.model.Paths
           java.io.File)
  (:gen-class))

;;from: http://stackoverflow.com/q/16748447
;; we don't need a uuid for this
(defn gen-string[chunk-size, key-length]
 (apply str
  (flatten
   (interpose "-"
    (partition chunk-size
     (take key-length
      (repeatedly #(rand-nth "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"))))))))

;;; Local file sync
(defn get-names [file-s]
  (map #(.getPath %) file-s))

(defn just-files [file-s]
  (filter #(.isFile %) file-s))

(defn local-list [some-dir]
  (get-names (just-files (file-seq (clojure.java.io/file some-dir)))))

;; Removes the ./ if it appears in the key name
(defn clean-if-dot [some-file-name]
  (if (.startsWith some-file-name "./")
    (subs some-file-name 2)
    some-file-name))


(defn build-cf-client [creds]
  (AmazonCloudFrontClient. (BasicAWSCredentials. (:access-key creds) (:secret-key creds))))

(defn list-distros [creds]
  (println "Listing distros...")
  (for [distro (.getItems (.getDistributionList
                  (.listDistributions (build-cf-client creds) (ListDistributionsRequest.))))]
        (println "Distro: " (.getId distro))))

(defn get-invalidation-files []
  (vec (map #(str "/" %) (map #(clean-if-dot %) (local-list ".")))))

(defn build-index-path []
  (let [index-path (Paths.)
        file-list (get-invalidation-files)]
    (.setItems index-path (java.util.ArrayList. file-list))
    (.setQuantity index-path (Integer. (count file-list)))
    index-path))

(defn invalidate-index [creds which-distro]
  (let [cf-client (build-cf-client creds)
        index-path (build-index-path)]
   (print "Invalidating index.html with ID: "
     (.getId
       (.getInvalidation
         (.createInvalidation cf-client
           (CreateInvalidationRequest. which-distro
             (InvalidationBatch. index-path (gen-string 12 13)))))))))

;; Returns a list of the filenames in the bucket
(defn pull-fnames-by-bucket [creds bucket-name]
   (map (fn [i] (:key i))
        (:objects (s3/list-objects creds bucket-name))))

;; dumps filenames to a specified file
;;(dump-fnames-to-file "/tmp/foo.txt3" (pull-fnames-by-bucket cred "www.demoit.io"))
(defn dump-fnames-to-file [some-filename some-seq]
  (with-open [w (clojure.java.io/writer some-filename)]
    (doseq [line some-seq]
      (.write w line)<commit_msg>fixed doc strings<commit_after>(ns pho.core
  (:require [pantomime.mime :refer [mime-type-of]])
  (:require [aws.sdk.s3 :as s3])
  (:require [clojure.tools.cli :refer [cli]])
  (:import com.amazonaws.services.cloudfront.AmazonCloudFrontClient
           com.amazonaws.auth.BasicAWSCredentials
           com.amazonaws.ClientConfiguration
           com.amazonaws.services.cloudfront.model.ListDistributionsRequest
           com.amazonaws.services.cloudfront.model.ListDistributionsResult
           com.amazonaws.services.cloudfront.model.DistributionList
           com.amazonaws.services.cloudfront.model.DistributionSummary
           com.amazonaws.services.cloudfront.model.CreateInvalidationRequest
           com.amazonaws.services.cloudfront.model.CreateInvalidationResult
           com.amazonaws.services.cloudfront.model.Invalidation
           com.amazonaws.services.cloudfront.model.InvalidationBatch
           com.amazonaws.services.cloudfront.model.Paths
           java.io.File)
  (:gen-class))

;;from: http://stackoverflow.com/q/16748447
;; we don't need a uuid for this
(defn gen-string
  "generates a random string of chunk-size and key-length"
  [chunk-size, key-length]
 (apply str
  (flatten
   (interpose "-"
    (partition chunk-size
     (take key-length
      (repeatedly #(rand-nth "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"))))))))

;;; Local file sync
(defn get-names [file-s]
  (map #(.getPath %) file-s))

(defn just-files [file-s]
  (filter #(.isFile %) file-s))

(defn local-list
  "Lists just the files in your current directory"
  [some-dir]
  (get-names (just-files (file-seq (clojure.java.io/file some-dir)))))

(defn clean-if-dot
  "Removes the annoying ./ from the filename"
  [some-file-name]
  (if (.startsWith some-file-name "./")
    (subs some-file-name 2)
    some-file-name))


(defn build-cf-client [creds]
  (AmazonCloudFrontClient. (BasicAWSCredentials. (:access-key creds) (:secret-key creds))))

(defn list-distros [creds]
  (println "Listing distros...")
  (for [distro (.getItems (.getDistributionList
                  (.listDistributions (build-cf-client creds) (ListDistributionsRequest.))))]
        (println "Distro: " (.getId distro))))

(defn get-invalidation-files
  "Gets a vector of files in the current dir and formats them for the invalidation listing"
  []
  (vec (map #(str "/" %) (map #(clean-if-dot %) (local-list ".")))))

(defn build-index-path
  "Sets up a Path for holding the invalidation listing. This is misnamed"
  []
  (let [index-path (Paths.)
        file-list (get-invalidation-files)]
    (.setItems index-path (java.util.ArrayList. file-list))
    (.setQuantity index-path (Integer. (count file-list)))
    index-path))

(defn invalidate-index
  "Invalidates all the files we just pushed that are in S3."
  [creds which-distro]
  (let [cf-client (build-cf-client creds)
        index-path (build-index-path)]
   (print "Invalidating files with ID: "
     (.getId
       (.getInvalidation
         (.createInvalidation cf-client
           (CreateInvalidationRequest. which-distro
             (InvalidationBatch. index-path (gen-string 12 13)))))))))

(defn pull-fnames-by-bucket
  "Returns a list of the filenames in the bucket"
  [creds bucket-name]
   (map (fn [i] (:key i))
        (:objects (s3/list-objects creds bucket-name))))

;; dumps filenames to a specified file
;;(dump-fnames-to-file "/tmp/foo.txt3" (pull-fnames-by-bucket cred "www.demoit.io"))
(defn dump-fnames-to-file [some-filename some-seq]
  (with-open [w (clojure.java.io/writer some-filename)]
    (doseq [line some-seq]
      (.write w line)
{'fraction_non_alphanumeric': 0.14074831920491085, 'fraction_numerical': 0.008038585209003215, 'mean_word_length': 3.24240545567266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2592553', 'n_tokens_mistral': 2318, 'n_tokens_neox': 2239, 'n_words': 492}