<commit_before>(ns oc.storage.util.search
  "
  Publish search service triggers to AWS SQS.
  Usage:

  lein run -m oc.storage.util.search

  "
  (:require [clojure.string :as s]
            [amazonica.aws.sqs :as sqs]
            [cheshire.core :as json]
            [clojure.tools.cli :refer (parse-opts)]
            [taoensso.timbre :as timbre]
            [oc.storage.config :as config]
            [oc.lib.db.common :as db-common]
            [oc.storage.resources.org :as org-res]
            [oc.storage.resources.board :as board-res]
            [oc.storage.resources.entry :as entry-res]
            [oc.lib.db.pool :as db]
            [oc.storage.config :as c])
  (:gen-class))

(defn exit [status msg]
  (println msg)
  (System/exit status))

;; ----- Search triggering -----

(defn ->trigger [resource-type content org board]
  {:Message
   (json/generate-string {:resource-type resource-type
                          :org org
                          :board board
                          :content {:new content}})
   })

(defn send-trigger! [trigger]
  (timbre/debug "Search request:" trigger)
  (timbre/info "Sending request to queue:" config/aws-sqs-search-index-queue)
  (sqs/send-message
    {:access-key config/aws-access-key-id
     :secret-key config/aws-secret-access-key}
    config/aws-sqs-search-index-queue
    (json/generate-string trigger {:pretty true}))
  (timbre/info "Request sent to:" config/aws-sqs-search-index-queue))

(defn index-all-entries [conn]
  (let [now (db-common/current-timestamp)]
    (doseq [org (org-res/list-orgs conn [:team-id])]
      (let [boards (board-res/list-boards-by-org conn (:uuid org))<commit_msg>need access<commit_after>(ns oc.storage.util.search
  "
  Publish search service triggers to AWS SQS.
  Usage:

  lein run -m oc.storage.util.search

  "
  (:require [clojure.string :as s]
            [amazonica.aws.sqs :as sqs]
            [cheshire.core :as json]
            [clojure.tools.cli :refer (parse-opts)]
            [taoensso.timbre :as timbre]
            [oc.storage.config :as config]
            [oc.lib.db.common :as db-common]
            [oc.storage.resources.org :as org-res]
            [oc.storage.resources.board :as board-res]
            [oc.storage.resources.entry :as entry-res]
            [oc.lib.db.pool :as db]
            [oc.storage.config :as c])
  (:gen-class))

(defn exit [status msg]
  (println msg)
  (System/exit status))

;; ----- Search triggering -----

(defn ->trigger [resource-type content org board]
  {:Message
   (json/generate-string {:resource-type resource-type
                          :org org
                          :board board
                          :content {:new content}})
   })

(defn send-trigger! [trigger]
  (timbre/debug "Search request:" trigger)
  (timbre/info "Sending request to queue:" config/aws-sqs-search-index-queue)
  (sqs/send-message
    {:access-key config/aws-access-key-id
     :secret-key config/aws-secret-access-key}
    config/aws-sqs-search-index-queue
    (json/generate-string trigger {:pretty true}))
  (timbre/info "Request sent to:" config/aws-sqs-search-index-queue))

(defn index-all-entries [conn]
  (let [now (db-common/current-timestamp)]
    (doseq [org (org-res/list-orgs conn [:team-id])]
      (let [boards (board-res/list-boards-by-org conn (:uuid org) [:access])
{'fraction_non_alphanumeric': 0.14418465227817745, 'fraction_numerical': 0.0, 'mean_word_length': 2.242954324586978, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4714965', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1091, 'n_words': 249}