<commit_before>(ns status-im.chat.db
  (:require [clojure.set :as clojure.set]
            [clojure.string :as string]
            [status-im.contact.db :as contact.db]
            [status-im.chat.commands.core :as commands]
            [status-im.chat.commands.input :as commands.input]
            [status-im.group-chats.db :as group-chats.db]
            [status-im.utils.gfycat.core :as gfycat]
            [status-im.transport.partitioned-topic :as topic]
            [status-im.mailserver.core :as mailserver]))

(defn group-chat-name
  [{:keys [public? name]}]
  (str (when public? "#") name))

(defn enrich-active-chat
  [contacts {:keys [chat-id public? group-chat name] :as chat} current-public-key]
  (if group-chat
    (let [pending-invite-inviter-name
          (group-chats.db/get-pending-invite-inviter-name contacts
                                                          chat
                                                          current-public-key)
          inviter-name
          (group-chats.db/get-inviter-name contacts
                                           chat
                                           current-public-key)]
      (cond-> chat
        pending-invite-inviter-name
        (assoc :pending-invite-inviter-name pending-invite-inviter-name)
        inviter-name
        (assoc :inviter-name inviter-name)
        :always
        (assoc :chat-name (group-chat-name chat))))
    (let [{contact-name :name :as contact}
          (get contacts chat-id
               (contact.db/public-key->new-contact chat-id))
          random-name (gfycat/generate-gfy chat-id)]
      (-> chat
          (assoc :contact contact
                 :chat-name (or contact-name random-name)
                 :name contact-name
                 :random-name random-name)
          (update :tags clojure.set/union (:tags contact))))))

(defn active-chats
  [contacts chats {:keys [public-key]}]
  (reduce (fn [acc [chat-id {:keys [is-active] :as chat}]]
            (if is-active
              (assoc acc chat-id (enrich-active-chat contacts chat public-key))
              acc))
          {}
          chats))

(defn topic-by-current-chat
  [{:keys [current-chat-id chats] :as db}]
  (let [{:keys [public?]} (get chats current-chat-id)
        public-key (get-in db [:account/account :public-key])]
    (if public?
      (get-in db [:transport/chats current-chat-id :topic])
      (topic/public-key->discovery-topic-hash public-key))))

(defn sort-message-groups
  "Sorts message groups according to timestamp of first message in group"
  [message-groups messages]
  (sort-by
   (comp :timestamp (partial get messages) :message-id first second)
   message-groups))

(defn quoted-message-data
  "Selects certain data from quoted message which must be available in the view"
  [message-id messages referenced-messages]
  (when-let [{:keys [from content]} (get messages message-id
                                         (get referenced-messages message-id))]
    {:from from
     :text (:text content)}))

(defn add-datemark
  [[datemark
    message-references]]
  (let [{:keys [whisper-timestamp timestamp]} (first message-references)]
    (conj message-references
          {:value             datemark
           :type              :datemark
           :whisper-timestamp whisper-timestamp
           :timestamp         timestamp})))

(defn datemark? [{:keys [type]}]
  (= type :datemark))

(defn gap? [{:keys [type]}]
  (= type :gap))

(defn transform-message
  [messages message-statuses referenced-messages]
  (fn [{:keys [message-id timestamp-str] :as reference}]
    (if (or (datemark? reference)
            (gap? reference))
      reference
      (let [{:keys [content] :as message} (get messages message-id)
            {:keys [response-to response-to-v2]} content
            quote (some-> (or response-to-v2 response-to)
                          (quoted-message-data messages referenced-messages))]
        (cond-> (-> message
                    (update :content dissoc :response-to :response-to-v2)
                    (assoc :timestamp-str timestamp-str
                           :user-statuses (get message-statuses message-id)))
          ;; quoted message reference
          quote
          (assoc-in [:content :response-to] quote))))))

(defn check-gap
  [gaps previous-message next-message]
  (let [previous-timestamp     (:whisper-timestamp previous-message)
        next-whisper-timestamp (:whisper-timestamp next-message)
        next-timestamp         (quot (:timestamp next-message) 1000)
        ignore-next-message?   (> (js/Math.abs
                                   (- next-whisper-timestamp next-timestamp))
                                  120)]
    (reduce
     (fn [acc {:keys [from to id]}]
       (if (and next-message
                (not ignore-next-message?)
                (or
                 (and (nil? previous-timestamp)
                      (< from next-whisper-timestamp))
                 (and
                  (< previous-timestamp from)
                  (< to next-whisper-timestamp))
                 (and
                  (< from previous-timestamp)
                  (< to next-whisper-timestamp))))
         (-> acc
             (update :gaps-number inc)
             (update-in [:gap :ids] conj id))
         (reduced acc)))
     {:gaps-number 0
      :gap         nil}
     gaps)))

(defn add-gap [messages gaps]
  (conj messages
        {:type  :gap
         :value (clojure.string/join (:ids gaps))
         :gaps gaps}))

(defn messages-with-datemarks-and-statuses
  "Converts message groups into sequence of messages interspersed with datemarks,
  with correct user statuses associated into message"
  [message-groups messages message-statuses referenced-messages messages-gaps
   {:keys [highest-request-to lowest-request-from]} all-loaded? public?]
  (transduce
   (comp
    (mapcat add-datemark)
    (map (transform-message messages message-statuses referenced-messages)))
   (fn
     ([]
      (let [acc {:messages         (list)
                 :previous-message nil
                 :gaps             messages-gaps}]
        (if (and
             public?
             all-loaded?
             (< (- highest-request-to lowest-request-from)
                mailserver/max-gaps-range))
          (update acc :messages conj {:type       :gap
                                      :value      (str :first-gap)<commit_msg>[#8123] Show fetch more messages only after sync<commit_after>(ns status-im.chat.db
  (:require [clojure.set :as clojure.set]
            [clojure.string :as string]
            [status-im.contact.db :as contact.db]
            [status-im.chat.commands.core :as commands]
            [status-im.chat.commands.input :as commands.input]
            [status-im.group-chats.db :as group-chats.db]
            [status-im.utils.gfycat.core :as gfycat]
            [status-im.transport.partitioned-topic :as topic]
            [status-im.mailserver.core :as mailserver]))

(defn group-chat-name
  [{:keys [public? name]}]
  (str (when public? "#") name))

(defn enrich-active-chat
  [contacts {:keys [chat-id public? group-chat name] :as chat} current-public-key]
  (if group-chat
    (let [pending-invite-inviter-name
          (group-chats.db/get-pending-invite-inviter-name contacts
                                                          chat
                                                          current-public-key)
          inviter-name
          (group-chats.db/get-inviter-name contacts
                                           chat
                                           current-public-key)]
      (cond-> chat
        pending-invite-inviter-name
        (assoc :pending-invite-inviter-name pending-invite-inviter-name)
        inviter-name
        (assoc :inviter-name inviter-name)
        :always
        (assoc :chat-name (group-chat-name chat))))
    (let [{contact-name :name :as contact}
          (get contacts chat-id
               (contact.db/public-key->new-contact chat-id))
          random-name (gfycat/generate-gfy chat-id)]
      (-> chat
          (assoc :contact contact
                 :chat-name (or contact-name random-name)
                 :name contact-name
                 :random-name random-name)
          (update :tags clojure.set/union (:tags contact))))))

(defn active-chats
  [contacts chats {:keys [public-key]}]
  (reduce (fn [acc [chat-id {:keys [is-active] :as chat}]]
            (if is-active
              (assoc acc chat-id (enrich-active-chat contacts chat public-key))
              acc))
          {}
          chats))

(defn topic-by-current-chat
  [{:keys [current-chat-id chats] :as db}]
  (let [{:keys [public?]} (get chats current-chat-id)
        public-key (get-in db [:account/account :public-key])]
    (if public?
      (get-in db [:transport/chats current-chat-id :topic])
      (topic/public-key->discovery-topic-hash public-key))))

(defn sort-message-groups
  "Sorts message groups according to timestamp of first message in group"
  [message-groups messages]
  (sort-by
   (comp :timestamp (partial get messages) :message-id first second)
   message-groups))

(defn quoted-message-data
  "Selects certain data from quoted message which must be available in the view"
  [message-id messages referenced-messages]
  (when-let [{:keys [from content]} (get messages message-id
                                         (get referenced-messages message-id))]
    {:from from
     :text (:text content)}))

(defn add-datemark
  [[datemark
    message-references]]
  (let [{:keys [whisper-timestamp timestamp]} (first message-references)]
    (conj message-references
          {:value             datemark
           :type              :datemark
           :whisper-timestamp whisper-timestamp
           :timestamp         timestamp})))

(defn datemark? [{:keys [type]}]
  (= type :datemark))

(defn gap? [{:keys [type]}]
  (= type :gap))

(defn transform-message
  [messages message-statuses referenced-messages]
  (fn [{:keys [message-id timestamp-str] :as reference}]
    (if (or (datemark? reference)
            (gap? reference))
      reference
      (let [{:keys [content] :as message} (get messages message-id)
            {:keys [response-to response-to-v2]} content
            quote (some-> (or response-to-v2 response-to)
                          (quoted-message-data messages referenced-messages))]
        (cond-> (-> message
                    (update :content dissoc :response-to :response-to-v2)
                    (assoc :timestamp-str timestamp-str
                           :user-statuses (get message-statuses message-id)))
          ;; quoted message reference
          quote
          (assoc-in [:content :response-to] quote))))))

(defn check-gap
  [gaps previous-message next-message]
  (let [previous-timestamp     (:whisper-timestamp previous-message)
        next-whisper-timestamp (:whisper-timestamp next-message)
        next-timestamp         (quot (:timestamp next-message) 1000)
        ignore-next-message?   (> (js/Math.abs
                                   (- next-whisper-timestamp next-timestamp))
                                  120)]
    (reduce
     (fn [acc {:keys [from to id]}]
       (if (and next-message
                (not ignore-next-message?)
                (or
                 (and (nil? previous-timestamp)
                      (< from next-whisper-timestamp))
                 (and
                  (< previous-timestamp from)
                  (< to next-whisper-timestamp))
                 (and
                  (< from previous-timestamp)
                  (< to next-whisper-timestamp))))
         (-> acc
             (update :gaps-number inc)
             (update-in [:gap :ids] conj id))
         (reduced acc)))
     {:gaps-number 0
      :gap         nil}
     gaps)))

(defn add-gap [messages gaps]
  (conj messages
        {:type  :gap
         :value (clojure.string/join (:ids gaps))
         :gaps gaps}))

(defn messages-with-datemarks-and-statuses
  "Converts message groups into sequence of messages interspersed with datemarks,
  with correct user statuses associated into message"
  [message-groups messages message-statuses referenced-messages messages-gaps
   {:keys [highest-request-to lowest-request-from]} all-loaded? public?]
  (transduce
   (comp
    (mapcat add-datemark)
    (map (transform-message messages message-statuses referenced-messages)))
   (fn
     ([]
      (let [acc {:messages         (list)
                 :previous-message nil
                 :gaps             messages-gaps}]
        (if (and
             public?
             all-loaded?
             (not (nil? highest-request-to))
             (not (nil? lowest-request-from))
             (< (- highest-request-to lowest-request-from)
                mailserver/max-gaps-range))
          (update acc :messages conj {:type       :gap
                                      :value      (str :first-gap)
{'fraction_non_alphanumeric': 0.12002161160852115, 'fraction_numerical': 0.0020067922198209326, 'mean_word_length': 1.7948662640207076, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4793237', 'n_tokens_mistral': 3758, 'n_tokens_neox': 3649, 'n_words': 918}