<commit_before>(ns heffalump.db
  (import [com.google.common.cache CacheBuilder])
  (require [korma.core :as kc]))

(defn setup-db
  [db]
  (defentity statuses
    (entity-fields
      :id 	
      :uri     ;fediverse-unique resource ID
      :url     ;URL to the status page (can be remote)
      :account     ;Account
      :in_reply_to_id     ;null or ID of status it replies to
      :reblog     ;null or Status
      :content     ;Body of the status. This will contain HTML (remote HTML already sanitized)
      :created_at     ;
      :reblogs_count     ;
      :favourites_count     ;
      :reblogged     ;Boolean for authenticated user
      :favourited     ;Boolean for authenticated user
      :media_attachments     ;array of MediaAttachments
      :mentions     ;array of Mentions
      :application     ;Application from which the status was posted

(def-api-type media_attribute
  :url  ;URL of the original image (can be remote)
  :preview_url  ;URL of the preview image
  :type  ;Image or video
)

(def-api-type mention
  :url  ;URL of user's profile (can be remote)
  :acct  ;Username for local or username@domain for remote users
  :id  ;Account ID
)

(def-api-type account
  :id  ;
  :username  ;
  :acct  ;Equals username for local users, includes @domain for remote ones
  :display_name  ;
  :note  ;Biography of user
  :url  ;URL of the user's profile page (can be remote)
  :avatar  ;URL to the avatar image
  :header  ;URL to the header image
  :followers_count  ;
  :following_count  ;
  :statuses_count  ;
)

(defn setup-db
  [db]
  (doseq [[k v] api-type-fields]
    (defentity k
      (table (keyword (str (name k) s)))
      (entity-fields v)
      (database db))))



(defn create-cache
  []
  (->
    (CacheBuilder/newBuilder)
    (.softValues)))

(defn init
  [config]
  (let [conn (kc/create-db (:db config))]
    {:conn conn
     :cache (create-cache)}))
<commit_msg>copy/paste docs into code WIP<commit_after>(ns heffalump.db
  (import [com.google.common.cache CacheBuilder])
  (require [korma.core :as kc]))

(defn setup-db
  [db]
  (defentity statuses
    (entity-fields
      :id 	
      :uri     ;fediverse-unique resource ID
      :url     ;URL to the status page (can be remote)
      :account     ;Account
      :in_reply_to_id     ;null or ID of status it replies to
      :reblog     ;null or Status
      :content     ;Body of the status. This will contain HTML (remote HTML already sanitized)
      :created_at     ;
      :reblogs_count     ;
      :favourites_count     ;
      :reblogged     ;Boolean for authenticated user
      :favourited     ;Boolean for authenticated user
      :media_attachments     ;array of MediaAttachments
      :mentions     ;array of Mentions
      :application     ;Application from which the status was posted

(def-api-type media_attribute
  :url  ;URL of the original image (can be remote)
  :preview_url  ;URL of the preview image
  :type  ;Image or video
)

(def-api-type mention
  :url  ;URL of user's profile (can be remote)
  :acct  ;Username for local or username@domain for remote users
  :id  ;Account ID
)

(def-api-type account
  :id  ;
  :username  ;
  :acct  ;Equals username for local users, includes @domain for remote ones
  :display_name  ;
  :note  ;Biography of user
  :url  ;URL of the user's profile page (can be remote)
  :avatar  ;URL to the avatar image
  :header  ;URL to the header image
  :followers_count  ;
  :following_count  ;
  :statuses_count  ;
)

(defn setup-db
  [db]
  (doseq [[k v] api-type-fields]
    (defentity k
      (table (keyword (str (name k) s)))
      (entity-fields v)
      (database db))))

(defn create-cache
  []
  (->
    (CacheBuilder/newBuilder)
    (.softValues)))

(defn init
  [config]
  (let [conn (kc/create-db (:db config))]
    {:conn conn
     :cache (create-cache)}))

{'fraction_non_alphanumeric': 0.09617400419287211, 'fraction_numerical': 0.0, 'mean_word_length': 2.6561302681992336, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6303753', 'n_tokens_mistral': 1288, 'n_tokens_neox': 1249, 'n_words': 422}