<commit_before>(ns ctia.entity.investigation
  (:require
   [ctia.http.routes
    [common :refer [BaseEntityFilterParams
                    PagingParams
                    SourcableEntityFilterParams]]
    [crud :refer [entity-crud-routes]]]
   [ctia.stores.es
    [mapping :as em]
    [store :refer [def-es-store]]]
   [ctia.entity.investigation.schemas
    :refer [Investigation
            PartialInvestigation
            PartialStoredInvestigation
            NewInvestigation
            StoredInvestigation
            PartialInvestigationList
            realize-investigation]]
   [schema-tools.core :as st]
   [schema.core :as s]
   [ctia.schemas.sorting :as sorting]))

(def snapshot-action-fields-mapping
  {:object_ids {:type "text"
                :analyzer "text_analyzer"
                :search_quote_analyzer "text_analyzer"
                :search_analyzer "search_analyzer"
                :include_in_all false}
   :targets {:type "nested"
             :include_in_all false}
   :investigated_observables {:type "text"
                              :analyzer "text_analyzer"
                              :search_quote_analyzer "text_analyzer"
                              :search_analyzer "search_analyzer"
                              :include_in_all false}})

(def investigation-mapping
  {"investigation"
   {:dynamic false
    :properties
    (merge
     em/base-entity-mapping
     em/describable-entity-mapping
     em/sourcable-entity-mapping
     em/stored-entity-mapping
     snapshot-action-fields-mapping)}})

(def-es-store InvestigationStore :investigation
  StoredInvestigation
  PartialStoredInvestigation)

(def investigation-fields
  (concat sorting/default-entity-sort-fields
          sorting/describable-entity-sort-fields<commit_msg>Use em/text and em/token<commit_after>(ns ctia.entity.investigation
  (:require
   [ctia.http.routes
    [common :refer [BaseEntityFilterParams
                    PagingParams
                    SourcableEntityFilterParams]]
    [crud :refer [entity-crud-routes]]]
   [ctia.stores.es
    [mapping :as em]
    [store :refer [def-es-store]]]
   [ctia.entity.investigation.schemas
    :refer [Investigation
            PartialInvestigation
            PartialStoredInvestigation
            NewInvestigation
            StoredInvestigation
            PartialInvestigationList
            realize-investigation]]
   [schema-tools.core :as st]
   [schema.core :as s]
   [ctia.schemas.sorting :as sorting]))

(def snapshot-action-fields-mapping
  {:object_ids em/token
   :targets {:type "nested"
             :include_in_all false}
   :investigated_observables em/text})

(def investigation-mapping
  {"investigation"
   {:dynamic false
    :properties
    (merge
     em/base-entity-mapping
     em/describable-entity-mapping
     em/sourcable-entity-mapping
     em/stored-entity-mapping
     snapshot-action-fields-mapping)}})

(def-es-store InvestigationStore :investigation
  StoredInvestigation
  PartialStoredInvestigation)

(def investigation-fields
  (concat sorting/default-entity-sort-fields
          sorting/describable-entity-sort-fields
{'fraction_non_alphanumeric': 0.08951406649616368, 'fraction_numerical': 0.0, 'mean_word_length': 2.446035242290749, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4208063', 'n_tokens_mistral': 945, 'n_tokens_neox': 902, 'n_words': 160}