<commit_before>(ns ceres-collector.mongo
  (:require [ceres-collector.polymorph :refer [Database transact]]
            [clj-time.core :as t]
            [clj-time.coerce :as c]
            [clojure.java.shell :refer [sh]]
            [monger.core :as mg]
            [monger.collection :as mc]
            [monger.operators :refer :all]
            [monger.conversion :refer [from-db-object]]
            [monger.joda-time]
            [taoensso.timbre :as timbre])
  (:import [ceres_collector.polymorph DbEntry DbQuery]
           [com.mongodb MongoOptions ServerAddress]))


(timbre/refer-timbre)

(defn create-index
  "Define mongodb indices on first start"
  [db]
  (info "MONGODB - Creating index!")
  (do
    (mc/ensure-index db "urls" (array-map :url 1))
    (mc/ensure-index db "urls" (array-map :ts 1))
    (mc/ensure-index db "mentions" (array-map :source 1))
    (mc/ensure-index db "mentions" (array-map :target 1))
    (mc/ensure-index db "mentions" (array-map :ts 1))
    (mc/ensure-index db "sources" (array-map :source 1))
    (mc/ensure-index db "sources" (array-map :target 1))
    (mc/ensure-index db "sources" (array-map :ts 1))
    (mc/ensure-index db "replies" (array-map :source 1))
    (mc/ensure-index db "replies" (array-map :target 1))
    (mc/ensure-index db "replies" (array-map :ts 1))
    (mc/ensure-index db "retweets" (array-map :source 1))
    (mc/ensure-index db "retweets" (array-map :target 1))
    (mc/ensure-index db "retweets" (array-map :ts 1))
    (mc/ensure-index db "pubs" (array-map :source 1))
    (mc/ensure-index db "pubs" (array-map :target 1))
    (mc/ensure-index db "pubs" (array-map :ts 1))
    (mc/ensure-index db "shares" (array-map :source 1))
    (mc/ensure-index db "shares" (array-map :target 1))
    (mc/ensure-index db "shares" (array-map :ts 1))
    (mc/ensure-index db "urlrefs" (array-map :source 1))
    (mc/ensure-index db "urlrefs" (array-map :target 1))
    (mc/ensure-index db "urlrefs" (array-map :ts 1))
    (mc/ensure-index db "tagrefs" (array-map :source 1))
    (mc/ensure-index db "tagrefs" (array-map :target 1))
    (mc/ensure-index db "tagrefs" (array-map :ts 1))
    (mc/ensure-index db "unknown" (array-map :source 1))
    (mc/ensure-index db "unknown" (array-map :target 1))
    (mc/ensure-index db "unknown" (array-map :ts 1))
    (mc/ensure-index db "tags" (array-map :text 1))
    (mc/ensure-index db "tags" (array-map :ts 1))
    (mc/ensure-index db "messages" (array-map :ts 1))
    (mc/ensure-index db "messages" (array-map :text 1))
    (mc/ensure-index db "messages" (array-map :tweet 1))
    (mc/ensure-index db "messages" (array-map :tid 1))
    (mc/ensure-index db "messages" (array-map :ts 1))
    (mc/ensure-index db "htmls" (array-map :ts 1))
    (mc/ensure-index db "users" (array-map :id 1))
    (mc/ensure-index db "users" (array-map :ts 1))
    (mc/ensure-index db "tweets" (array-map :user.screen_name 1))
    (mc/ensure-index db "tweets" (array-map :id 1))
    (mc/ensure-index db "tweets" (array-map :retweeted_status.id 1))
    (mc/ensure-index db "tweets" (array-map :in_reply_to_status_id 1))
    (mc/ensure-index db "tweets" (array-map :created_at 1))))


(defn type->coll [type]
  (case type
    :pub "pubs"
    :reply "replies"
    :retweet "retweets"
    :share "shares"
    :mention "mentions"
    :urlref "urlrefs"
    :tagref "tagrefs"
    :source "sources"
    :user "users"
    :tag "tags"
    :html "htmls"
    :tweet "tweets"
    :message "messages"
    :url "urls"
    :unknown "unknown"
    "unrelated"))


(defn store [db db-entry]
  (info (:type db-entry) " -> " (:value db-entry))
  (mc/insert-and-return db (type->coll (:type db-entry)) (assoc (:value db-entry) :ts (t/now))))
<commit_msg>remove debug info<commit_after>(ns ceres-collector.mongo
  (:require [ceres-collector.polymorph :refer [Database transact]]
            [clj-time.core :as t]
            [clj-time.coerce :as c]
            [clojure.java.shell :refer [sh]]
            [monger.core :as mg]
            [monger.collection :as mc]
            [monger.operators :refer :all]
            [monger.conversion :refer [from-db-object]]
            [monger.joda-time]
            [taoensso.timbre :as timbre])
  (:import [ceres_collector.polymorph DbEntry DbQuery]
           [com.mongodb MongoOptions ServerAddress]))


(timbre/refer-timbre)

(defn create-index
  "Define mongodb indices on first start"
  [db]
  (info "MONGODB - Creating index!")
  (do
    (mc/ensure-index db "urls" (array-map :url 1))
    (mc/ensure-index db "urls" (array-map :ts 1))
    (mc/ensure-index db "mentions" (array-map :source 1))
    (mc/ensure-index db "mentions" (array-map :target 1))
    (mc/ensure-index db "mentions" (array-map :ts 1))
    (mc/ensure-index db "sources" (array-map :source 1))
    (mc/ensure-index db "sources" (array-map :target 1))
    (mc/ensure-index db "sources" (array-map :ts 1))
    (mc/ensure-index db "replies" (array-map :source 1))
    (mc/ensure-index db "replies" (array-map :target 1))
    (mc/ensure-index db "replies" (array-map :ts 1))
    (mc/ensure-index db "retweets" (array-map :source 1))
    (mc/ensure-index db "retweets" (array-map :target 1))
    (mc/ensure-index db "retweets" (array-map :ts 1))
    (mc/ensure-index db "pubs" (array-map :source 1))
    (mc/ensure-index db "pubs" (array-map :target 1))
    (mc/ensure-index db "pubs" (array-map :ts 1))
    (mc/ensure-index db "shares" (array-map :source 1))
    (mc/ensure-index db "shares" (array-map :target 1))
    (mc/ensure-index db "shares" (array-map :ts 1))
    (mc/ensure-index db "urlrefs" (array-map :source 1))
    (mc/ensure-index db "urlrefs" (array-map :target 1))
    (mc/ensure-index db "urlrefs" (array-map :ts 1))
    (mc/ensure-index db "tagrefs" (array-map :source 1))
    (mc/ensure-index db "tagrefs" (array-map :target 1))
    (mc/ensure-index db "tagrefs" (array-map :ts 1))
    (mc/ensure-index db "unknown" (array-map :source 1))
    (mc/ensure-index db "unknown" (array-map :target 1))
    (mc/ensure-index db "unknown" (array-map :ts 1))
    (mc/ensure-index db "tags" (array-map :text 1))
    (mc/ensure-index db "tags" (array-map :ts 1))
    (mc/ensure-index db "messages" (array-map :ts 1))
    (mc/ensure-index db "messages" (array-map :text 1))
    (mc/ensure-index db "messages" (array-map :tweet 1))
    (mc/ensure-index db "messages" (array-map :tid 1))
    (mc/ensure-index db "messages" (array-map :ts 1))
    (mc/ensure-index db "htmls" (array-map :ts 1))
    (mc/ensure-index db "users" (array-map :id 1))
    (mc/ensure-index db "users" (array-map :ts 1))
    (mc/ensure-index db "tweets" (array-map :user.screen_name 1))
    (mc/ensure-index db "tweets" (array-map :id 1))
    (mc/ensure-index db "tweets" (array-map :retweeted_status.id 1))
    (mc/ensure-index db "tweets" (array-map :in_reply_to_status_id 1))
    (mc/ensure-index db "tweets" (array-map :created_at 1))))


(defn type->coll [type]
  (case type
    :pub "pubs"
    :reply "replies"
    :retweet "retweets"
    :share "shares"
    :mention "mentions"
    :urlref "urlrefs"
    :tagref "tagrefs"
    :source "sources"
    :user "users"
    :tag "tags"
    :html "htmls"
    :tweet "tweets"
    :message "messages"
    :url "urls"
    :unknown "unknown"
    "unrelated"))


(defn store [db db-entry]
  (mc/insert-and-return db (type->coll (:type db-entry)) (assoc (:value db-entry) :ts (t/now))))

{'fraction_non_alphanumeric': 0.16890334824454387, 'fraction_numerical': 0.011928968415344991, 'mean_word_length': 2.9795037756202807, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4014109', 'n_tokens_mistral': 2902, 'n_tokens_neox': 2787, 'n_words': 743}