<commit_before>(ns json-csv-converter.core
  (:use [clojure.java.io :only [file reader]])
  (:require [cheshire.core :as json]
            [clojure.string :as string]
            [clojure.tools.cli :refer [parse-opts]])
  (:import (java.io FileInputStream)))

(def cli-options
  [
   ["-i" "--input FILE" "Input file"]
   ["-c" "--col-spec COLUMN_SPEC" "Column spec"]
   ["-o" "--output FILE" "Output file"]
   ["-h" "--help"]])

(defn usage
  [options-summary & reminders]
  (->> ["Usage: program-name [options]"
        ""
        "Options:"
        options-summary
        ""]
       (concat reminders)
       (string/join \newline)))

(defn error-msg
  [errors]
  (str "The following errors occurred while parsing your command:\n\n"
       (string/join \newline errors)))

(defn exit
  [status msg]
  (println msg)
  (System/exit status))

(defn -main
  [& args]
  (let [{:keys [options arguments summary errors]} (parse-opts args cli-options)]
    (cond
     (:help options) (exit 0 (usage summary))
     (nil? (:input options)) (exit 1 (usage summary "Missing Input file"))
     (nil? (:col-spec options)) (exit 1 (usage summary "Missing Column spec"))
     errors (exit 1 (error-msg errors))) 
    (with-open [s (FileInputStream. (file (:input options)))]
      (let [r (reader s)]
        (doall (map prn (json/parsed-seq r)))))))<commit_msg>read col-spec file<commit_after>(ns json-csv-converter.core
  (:use [clojure.java.io :only [file reader]])
  (:require [cheshire.core :as json]
            [clojure.string :as string]
            [clojure.tools.cli :refer [parse-opts]])
  (:import (java.io FileInputStream)))

(def cli-options
  [
   ["-i" "--input FILE" "Input file"]
   ["-c" "--col-spec COLUMN_SPEC" "Column spec"]
   ["-o" "--output FILE" "Output file"]
   ["-h" "--help"]])

(defn usage
  [options-summary & reminders]
  (->> ["Usage: program-name [options]"
        ""
        "Options:"
        options-summary
        ""]
       (concat reminders)
       (string/join \newline)))

(defn error-msg
  [errors]
  (str "The following errors occurred while parsing your command:\n\n"
       (string/join \newline errors)))

(defn exit
  [status msg]
  (println msg)
  (System/exit status))

(defn get-columns
  [column-file]
  (string/split (slurp column-file) #"\n"))

(defn -main
  [& args]
  (let [{:keys [options arguments summary errors]} (parse-opts args cli-options)]
    (cond
     (:help options) (exit 0 (usage summary))
     (nil? (:input options)) (exit 1 (usage summary "Missing Input file"))
     (nil? (:col-spec options)) (exit 1 (usage summary "Missing Column spec"))
     errors (exit 1 (error-msg errors))) 
    (with-open [s (FileInputStream. (file (:input options)))]
      (let [r (reader s)
            cols (get-columns (:col-spec options))]
        (do
          (doall (map prn cols))
          (doall (map prn (json/parsed-seq r))))))))
{'fraction_non_alphanumeric': 0.17273673257023933, 'fraction_numerical': 0.002774887270204648, 'mean_word_length': 2.794736842105263, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5426855', 'n_tokens_mistral': 984, 'n_tokens_neox': 961, 'n_words': 285}