<commit_before>(ns athena-cmd.core
  (:gen-class)
  (:require [cheshire.core :refer :all]))

(import '(java.sql DriverManager) 
        'java.util.Properties
        'com.amazonaws.athena.jdbc.AthenaDriver)

(use '[clojure.pprint :only [print-table]])

(def athenaURI 
  "jdbc:awsathena://athena.us-east-1.amazonaws.com:443")

(def info (Properties.))

(def s3_path_env "ATHENA_S3_PATH")
(def s3_path (System/getenv s3_path_env))

(.put info "s3_staging_dir" 
      (if s3_path s3_path 
        (throw (Exception. (str s3_path_env " is not set.")))))

(.put info "aws_credentials_provider_class"
           "com.amazonaws.auth.DefaultAWSCredentialsProviderChain")

(.put info "log_path"
           "/tmp/athenajdbc.log")

(def conn (DriverManager/getConnection athenaURI info))
(def stmt (.createStatement conn))

(defn read_row 
  [rs]
  (let [m (.getMetaData rs)
        cnt (.getColumnCount m)]

    (reduce 
      #(assoc %1 (.getColumnName m %2) 
                (.getString rs %2)) 
                {}
                (range 1 (+ 1 cnt)))))

(defn exec
  [query]
  (let [rs (.executeQuery stmt query)
        m (.getMetaData rs)]

    (loop [
           res []
           more (.next rs)]

      (if-not more 
        res
        (do 
          (recur 
            (conj res (read_row rs))
            (.next rs)))))))

(defn query 
  [file]
  (exec (slurp file)))

(defn query-table 
  [file]
  (print-table (query file)))
<commit_msg>moved statement creation to function<commit_after>(ns athena-cmd.core
  (:gen-class)
  (:require [cheshire.core :refer :all]))

(import '(java.sql DriverManager) 
        'java.util.Properties
        'com.amazonaws.athena.jdbc.AthenaDriver)

(use '[clojure.pprint :only [print-table]])

(defn get_properties
  []
  (let [info (Properties.)
        s3_path_env "ATHENA_S3_PATH"
        s3_path (System/getenv s3_path_env)
        ]
    (doto info 

      (.put "s3_staging_dir" 
        (if s3_path s3_path 
          (throw (Exception. (str s3_path_env " is not set.")))))

      (.put "aws_credentials_provider_class"
            "com.amazonaws.auth.DefaultAWSCredentialsProviderChain")

      (.put "log_path"
            "/tmp/athenajdbc.log")) 

          info))

(defn get_stmt 
  [] 
  (let [athenaURI 
        "jdbc:awsathena://athena.us-east-1.amazonaws.com:443"
        info (get_properties)
        conn (DriverManager/getConnection athenaURI info)
        stmt (.createStatement conn)] stmt))

(defn read_row 
  [rs]
  (let [m (.getMetaData rs)
        cnt (.getColumnCount m)]

    (reduce 
      #(assoc %1 (.getColumnName m %2) 
                (.getString rs %2)) 
                {}
                (range 1 (+ 1 cnt)))))

(defn exec
  [query]
  (let [rs (.executeQuery (get_stmt) query)]

    (loop [res []
           more (.next rs)]

      (if-not more 
        res
        (do 
          (recur 
            (conj res (read_row rs))
            (.next rs)))))))

(defn query 
  [file]
  (exec (slurp file)))

(defn query-table 
  [file]
  (print-table (query file)))

{'fraction_non_alphanumeric': 0.13960396039603962, 'fraction_numerical': 0.011221122112211221, 'mean_word_length': 2.1838235294117645, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2096943', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1070, 'n_words': 246}