<commit_before>(ns akvo.lumen.lib.import.flow
  (:require [akvo.commons.psql-util :as pg]
            [akvo.lumen.lib.import.common :as import]
            [akvo.lumen.protocols :as p]
            [akvo.lumen.lib.import.flow-common :as flow-common]
            [akvo.lumen.lib.import.flow-v2 :as v2]
            [akvo.lumen.lib.import.flow-v3 :as v3]))


(defmethod import/dataset-importer "AKVO_FLOW"
  [{:strs [instance surveyId formId refreshToken version] :as spec}
   {:keys [flow-api-url keycloak-realm keycloak-url] :as config}]
  (let [version (if version version 1)
        token-endpoint (format "%s/realms/%s/protocol/openid-connect/token"
                               keycloak-url
                               keycloak-realm)
        headers-fn #(flow-common/flow-api-headers token-endpoint refreshToken)
        survey (delay (flow-common/survey-definition flow-api-url
                                                     headers-fn
                                                     instance
                                                     surveyId))]
    (reify
      java.io.Closeable
      (close [this])
      p/DatasetImporter
      (columns [this]
        (cond
          (<= version 2) (v2/dataset-columns (flow-common/form @survey formId))
          (<= version 3) (v3/dataset-columns (flow-common/form @survey formId))))
      (records [this]
        (try
          (cond
            (<= version 2) (v2/form-data headers-fn @survey formId)
            (<= version 3) (v3/form-data headers-fn @survey formId))
          (catch Throwable e
            (if-let [ex-d (ex-data e)]
              (throw (ex-info (:cause e) (assoc ex-d :instance instance)))
              (throw e))))))))
<commit_msg>[#1845] Add instance details to import flow data ex<commit_after>(ns akvo.lumen.lib.import.flow
  (:require [akvo.commons.psql-util :as pg]
            [akvo.lumen.lib.import.common :as import]
            [akvo.lumen.protocols :as p]
            [akvo.lumen.lib.import.flow-common :as flow-common]
            [akvo.lumen.lib.import.flow-v2 :as v2]
            [akvo.lumen.lib.import.flow-v3 :as v3]))


(defmethod import/dataset-importer "AKVO_FLOW"
  [{:strs [instance surveyId formId refreshToken version] :as spec}
   {:keys [flow-api-url keycloak-realm keycloak-url] :as config}]
  (let [version (if version version 1)
        token-endpoint (format "%s/realms/%s/protocol/openid-connect/token"
                               keycloak-url
                               keycloak-realm)
        headers-fn #(flow-common/flow-api-headers token-endpoint refreshToken)
        survey (delay (flow-common/survey-definition flow-api-url
                                                     headers-fn
                                                     instance
                                                     surveyId))]
    (reify
      java.io.Closeable
      (close [this])
      p/DatasetImporter
      (columns [this]
        (try
          (cond
            (<= version 2) (v2/dataset-columns (flow-common/form @survey formId))
            (<= version 3) (v3/dataset-columns (flow-common/form @survey formId)))
          (catch Throwable e
            (if-let [ex-d (ex-data e)]
              (throw (ex-info (:cause e) (assoc ex-d :instance instance)))
              (throw e)))))
      (records [this]
        (try
          (cond
            (<= version 2) (v2/form-data headers-fn @survey formId)
            (<= version 3) (v3/form-data headers-fn @survey formId))
          (catch Throwable e
            (if-let [ex-d (ex-data e)]
              (throw (ex-info (:cause e) (assoc ex-d :instance instance)))
              (throw e))))))))

{'fraction_non_alphanumeric': 0.12367778681855167, 'fraction_numerical': 0.008136696501220505, 'mean_word_length': 1.5862552594670407, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3942365', 'n_tokens_mistral': 1123, 'n_tokens_neox': 1095, 'n_words': 252}