<commit_before>(ns metabase.driver.sync
  "The logic for doing DB and Table syncing itself."
  (:require [clojure.math.numeric-tower :as math]
            (clojure [set :as set]
                     [string :as s])
            [clojure.tools.logging :as log]
            [cheshire.core :as json]
            [korma.core :as k]
            [medley.core :as m]
            [metabase.db :refer :all]
            (metabase.driver [interface :refer :all]
                             [query-processor :as qp])
            [metabase.driver.sync.queries :as queries]
            (metabase.models [common :as common]
                             [field :refer [Field] :as field]
                             [field-values :as field-values]
                             [foreign-key :refer [ForeignKey]]
                             [table :refer [Table]])
            [metabase.util :as u]))

(declare auto-assign-field-special-type-by-name!
         mark-category-field-or-update-field-values!
         mark-json-field!
         mark-no-preview-display-field!
         mark-url-field!
         maybe-driver-specific-sync-field!
         set-field-display-name-if-needed!
         sync-database-active-tables!
         sync-field!
         sync-table-active-fields-and-pks!
         sync-table-fks!
         sync-table-fields-metadata!
         update-table-display-name!
         sync-field-nested-fields!
         update-table-row-count!)

;; ## sync-database! and sync-table!

(defn sync-database!
  "Sync DATABASE and all its Tables and Fields."
  [driver database]
  (binding [qp/*disable-qp-logging* true
            *sel-disable-logging* true]
    (sync-in-context driver database
      (fn []
        (log/info (u/format-color 'magenta "Syncing %s database '%s'..." (name (:engine database)) (:name database)))

        (let [active-table-names (active-table-names driver database)
              table-name->id     (sel :many :field->id [Table :name] :db_id (:id database) :active true)]
          (assert (set? active-table-names) "active-table-names should return a set.")
          (assert (every? string? active-table-names) "active-table-names should return the names of Tables as *strings*.")

          ;; First, let's mark any Tables that are no longer active as such.
          ;; These are ones that exist in table-name->id but not in active-table-names.
          (doseq [[table-name table-id] table-name->id]
            (when-not (contains? active-table-names table-name)
              (upd Table table-id :active false)
              (log/info (u/format-color 'cyan "Marked table %s.%s as inactive." (:name database) table-name))

              ;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.)
              (k/update Field
                        (k/where {:table_id table-id})
                        (k/set-fields {:active false}))))

          ;; Next, we'll create new Tables (ones that came back in active-table-names but *not* in table-name->id)
          (let [existing-table-names (set (keys table-name->id))
                new-table-names      (set/difference active-table-names existing-table-names)]
            (when (seq new-table-names)
              (log/info (u/format-color 'blue "Found new tables: %s" (u/pprint-to-str new-table-names)))
              (doseq [new-table-name new-table-names]
                (ins Table :db_id (:id database), :active true, :name new-table-name)))))

        ;; Now sync the active tables
        (->> (sel :many Table :db_id (:id database) :active true)
             (map #(assoc % :db (delay database))) ; replace default delays with ones that reuse database (and don't require a DB call)
             (sync-database-active-tables! driver))

        (log/info (u/format-color 'magenta "Finished syncing %s database %s." (name (:engine database)) (:name database)))))))

(defn sync-table!
  "Sync a *single* TABLE by running all the sync steps for it.
   This is used *instead* of `sync-database!` when syncing just one Table is desirable."
  [driver table]
  (let [database @(:db table)]
    (binding [qp/*disable-qp-logging* true]
      (sync-in-context driver database
        (fn []
          (sync-database-active-tables! driver [table]))))))


;; ### sync-database-active-tables! -- runs the sync-table steps over sequence of Tables

(def sync-progress-meter-string
  "Create a string that shows sync progress for a database.

     (sync-progress-meter-string 10 40)
       -> \"[**********                              ] 25%\""
  (let [^:const meter-width 40]
    (fn [tables-finished total-tables]
      (let [percent-done (float (/ tables-finished total-tables))
            filleds      (int (* percent-done meter-width))
            blanks       (- meter-width filleds)]
        (str "["
             (apply str (repeat filleds "*"))
             (apply str (repeat blanks " "))
             (format "] 😋  %3d%%" (int (* percent-done 100.0))))))))

(defn- sync-database-active-tables!
  "Sync active tables by running each of the sync table steps.
   Note that we want to completely finish each step for *all* tables before starting the next, since they depend on the results of the previous step.
   (e.g., `sync-table-fks!` can't run until all tables have finished `sync-table-active-fields-and-pks!`, since creating `ForeignKeys` to `Fields` of *other*
   Tables can't take place before they exist."
  [driver active-tables]
  (let [active-tables (sort-by :name active-tables)]
    ;; First, create all the Fields / PKs for all of the Tables
    (u/pdoseq [table active-tables]
      (u/try-apply sync-table-active-fields-and-pks! driver table))

    ;; After that, we can do all the other syncing for the Tables
    (let [tables-count          (count active-tables)
          finished-tables-count (atom 0)]
      (u/pdoseq [table active-tables]
        ;; make sure table has :display_name
        (u/try-apply update-table-display-name! table)

        ;; update the row counts for every Table
        (u/try-apply update-table-row-count! table)<commit_msg>even better sync progress meter<commit_after>(ns metabase.driver.sync
  "The logic for doing DB and Table syncing itself."
  (:require [clojure.math.numeric-tower :as math]
            (clojure [set :as set]
                     [string :as s])
            [clojure.tools.logging :as log]
            [cheshire.core :as json]
            [korma.core :as k]
            [medley.core :as m]
            [metabase.db :refer :all]
            (metabase.driver [interface :refer :all]
                             [query-processor :as qp])
            [metabase.driver.sync.queries :as queries]
            (metabase.models [common :as common]
                             [field :refer [Field] :as field]
                             [field-values :as field-values]
                             [foreign-key :refer [ForeignKey]]
                             [table :refer [Table]])
            [metabase.util :as u]))

(declare auto-assign-field-special-type-by-name!
         mark-category-field-or-update-field-values!
         mark-json-field!
         mark-no-preview-display-field!
         mark-url-field!
         maybe-driver-specific-sync-field!
         set-field-display-name-if-needed!
         sync-database-active-tables!
         sync-field!
         sync-table-active-fields-and-pks!
         sync-table-fks!
         sync-table-fields-metadata!
         update-table-display-name!
         sync-field-nested-fields!
         update-table-row-count!)

;; ## sync-database! and sync-table!

(defn sync-database!
  "Sync DATABASE and all its Tables and Fields."
  [driver database]
  (binding [qp/*disable-qp-logging* true
            *sel-disable-logging* true]
    (sync-in-context driver database
      (fn []
        (log/info (u/format-color 'magenta "Syncing %s database '%s'..." (name (:engine database)) (:name database)))

        (let [active-table-names (active-table-names driver database)
              table-name->id     (sel :many :field->id [Table :name] :db_id (:id database) :active true)]
          (assert (set? active-table-names) "active-table-names should return a set.")
          (assert (every? string? active-table-names) "active-table-names should return the names of Tables as *strings*.")

          ;; First, let's mark any Tables that are no longer active as such.
          ;; These are ones that exist in table-name->id but not in active-table-names.
          (doseq [[table-name table-id] table-name->id]
            (when-not (contains? active-table-names table-name)
              (upd Table table-id :active false)
              (log/info (u/format-color 'cyan "Marked table %s.%s as inactive." (:name database) table-name))

              ;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.)
              (k/update Field
                        (k/where {:table_id table-id})
                        (k/set-fields {:active false}))))

          ;; Next, we'll create new Tables (ones that came back in active-table-names but *not* in table-name->id)
          (let [existing-table-names (set (keys table-name->id))
                new-table-names      (set/difference active-table-names existing-table-names)]
            (when (seq new-table-names)
              (log/info (u/format-color 'blue "Found new tables: %s" (u/pprint-to-str new-table-names)))
              (doseq [new-table-name new-table-names]
                (ins Table :db_id (:id database), :active true, :name new-table-name)))))

        ;; Now sync the active tables
        (->> (sel :many Table :db_id (:id database) :active true)
             (map #(assoc % :db (delay database))) ; replace default delays with ones that reuse database (and don't require a DB call)
             (sync-database-active-tables! driver))

        (log/info (u/format-color 'magenta "Finished syncing %s database %s." (name (:engine database)) (:name database)))))))

(defn sync-table!
  "Sync a *single* TABLE by running all the sync steps for it.
   This is used *instead* of `sync-database!` when syncing just one Table is desirable."
  [driver table]
  (let [database @(:db table)]
    (binding [qp/*disable-qp-logging* true]
      (sync-in-context driver database
        (fn []
          (sync-database-active-tables! driver [table]))))))


;; ### sync-database-active-tables! -- runs the sync-table steps over sequence of Tables

(def sync-progress-meter-string
  "Create a string that shows sync progress for a database.

     (sync-progress-meter-string 10 40)
       -> \"[************······································] 25%\""
  (let [^:const meter-width 50]
    (fn [tables-finished total-tables]
      (let [percent-done (float (/ tables-finished total-tables))
            filleds      (int (* percent-done meter-width))
            blanks       (- meter-width filleds)]
        (str "["
             (apply str (repeat filleds "*"))
             (apply str (repeat blanks "·"))
             (format "] 😋  %3d%%" (int (* percent-done 100.0))))))))

(defn- sync-database-active-tables!
  "Sync active tables by running each of the sync table steps.
   Note that we want to completely finish each step for *all* tables before starting the next, since they depend on the results of the previous step.
   (e.g., `sync-table-fks!` can't run until all tables have finished `sync-table-active-fields-and-pks!`, since creating `ForeignKeys` to `Fields` of *other*
   Tables can't take place before they exist."
  [driver active-tables]
  (let [active-tables (sort-by :name active-tables)]
    ;; First, create all the Fields / PKs for all of the Tables
    (u/pdoseq [table active-tables]
      (u/try-apply sync-table-active-fields-and-pks! driver table))

    ;; After that, we can do all the other syncing for the Tables
    (let [tables-count          (count active-tables)
          finished-tables-count (atom 0)]
      (u/pdoseq [table active-tables]
        ;; make sure table has :display_name
        (u/try-apply update-table-display-name! table)

        ;; update the row counts for every Table
        (u/try-apply update-table-row-count! table)
{'fraction_non_alphanumeric': 0.1300686723348594, 'fraction_numerical': 0.002289077828646174, 'mean_word_length': 2.1956635318704283, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4485110', 'n_tokens_mistral': 3688, 'n_tokens_neox': 3512, 'n_words': 1121}