<commit_before>(ns akvo.lumen.import.csv-test
  {:functional true}
  (:require [akvo.lumen.fixtures :refer [*tenant-conn*
                                         tenant-conn-fixture
                                         *error-tracker*
                                         error-tracker-fixture]]
            [akvo.lumen.test-utils :refer [import-file]]
            [clojure.string :as string]
            [clojure.test :refer :all]
            [hugsql.core :as hugsql]))

(hugsql/def-db-fns "akvo/lumen/job-execution.sql")
(hugsql/def-db-fns "akvo/lumen/transformation.sql")


(use-fixtures :once tenant-conn-fixture error-tracker-fixture)


(deftest ^:functional test-dos-file
  (testing "Import of DOS-formatted CSV file"
    (try
      (let [dataset-id (import-file *tenant-conn* *error-tracker* "dos.csv" {:dataset-name "DOS data"})
            dataset (dataset-version-by-dataset-id *tenant-conn* {:dataset-id dataset-id
                                                                  :version 1})]
        (is (= 2 (count (:columns dataset)))))
      (catch Exception e
        (.printStackTrace (.getCause e))))))

(deftest ^:functional test-mixed-columns
  (testing "Import of mixed-type data"<commit_msg>Update csv_test.clj<commit_after>(ns akvo.lumen.import.csv-test
  {:functional true}
  (:require [akvo.lumen.fixtures :refer [*tenant-conn*
                                         tenant-conn-fixture
                                         *error-tracker*
                                         error-tracker-fixture]]
            [akvo.lumen.test-utils :refer [import-file]]
            [clojure.string :as string]
            [clojure.test :refer :all]
            [hugsql.core :as hugsql]))

(hugsql/def-db-fns "akvo/lumen/job-execution.sql")
(hugsql/def-db-fns "akvo/lumen/transformation.sql")


(use-fixtures :once tenant-conn-fixture error-tracker-fixture)


(deftest ^:functional test-dos-file
  (testing "Import of DOS-formatted CSV file"
    (let [dataset-id (import-file *tenant-conn* *error-tracker* "dos.csv" {:dataset-name "DOS data"})
          dataset (dataset-version-by-dataset-id *tenant-conn* {:dataset-id dataset-id
                                                                :version 1})]
      (is (= 2 (count (:columns dataset)))))))

(deftest ^:functional test-mixed-columns
  (testing "Import of mixed-type data"
{'fraction_non_alphanumeric': 0.13212325875897002, 'fraction_numerical': 0.0016884761502743773, 'mean_word_length': 1.8692493946731235, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1204168', 'n_tokens_mistral': 740, 'n_tokens_neox': 708, 'n_words': 145}