<commit_before>;; Copyright (c) <NAME>. All rights reserved.
;; The use and distribution terms for this software are covered by the
;; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
;; which can be found in the file LICENSE at the root of this distribution.
;; By using this software in any fashion, you are agreeing to be bound by
;; the terms of this license.
;; You must not remove this notice, or any other, from this software.

(ns conexp.contrib.exec
  (:use conexp.main
        [conexp.io.util :only (tmpfile with-out-writer)])
  (:use clojure.contrib.shell-out))

(ns-doc "Executing external programs with a common interface.")

;;;

(defn run-in-shell
  "Runs given commandline in an external shell process. Returns the
  output on stdout as result."
  [& cmdln]
  (let [result (apply sh
                      :return-map true
                      cmdln)]
    (when-not (zero? (:exit result))
      (illegal-state "External program " (first cmdln) " returned with non-zero status: " (:err result)))
    (:out result)))

(defn- context-to-file
  "Writes context to a file using format and returns the name of the
  file."
  [context format]
  (let [^java.io.File file (tmpfile)]
    (write-context format context file)
    (.getAbsolutePath file)))

(defmacro define-external-program
  "Defines program-name to be a function calling an external
  program. Keywords are treated as arguments to the function and for
  every keyword there will be a corresponding function argument with
  the same name (without the colon). Special treatment is given
  to :input-file, which must be followed by a valid format to
  write-context. Its argument will be called \"context\" (thus you
  shall not name any other argument this way) and will get a context
  as input. This context will then be written to a file using the give
  format and this file will be given as an argument to the external
  process.

  Returns a map of :exit, :out and :err pointing to corresponding exit
  value, value at stdout and value at stderr, respectively."
  [program-name & command-line]
  (let [cmdln                      (map (fn [x]
                                          (if (keyword? x)
                                            x,
                                            (str x)))
                                        command-line),
        [before-input after-input] (split-with #(not= :input-file %) cmdln),
        input-file?                (first after-input)
        input-format               (second after-input)
        cmdln                      (if input-file?
                                     (concat before-input [:context] (drop 2 after-input))
                                     before-input),
        arguments                  (filter keyword? cmdln),
        arg-names                  (vec (map (comp symbol name) arguments))]
    `(defn ~program-name ~arg-names
       (run-in-shell ~@(replace (merge {:context `(context-to-file ~'context ~input-format)}
                                       (dissoc (zipmap arguments arg-names) :context))
                                cmdln)))))

(defmacro with-context-from-output
  "Runs body, expecting to get a string containing a formal
  context. Writes this string to a file and reads it in
  with read-context, returning the result."
  [& body]
  `(let [result# (do ~@body),
         file#   (tmpfile)]
     (with-out-writer file#
       (print result#))
     (read-context file#)))

(defmacro with-context-from-tmpfile
  "Creates a new temporary file, executes body and returns the result
  from calling read-context on file."
  [file & body]
  `(let [~file (tmpfile),
         result# (do ~@body)]
     (read-context ~file)))

;;;<commit_msg>Using clojure.java.shell instead of clojure.contrib.shell-out<commit_after>;; Copyright (c) <NAME>. All rights reserved.
;; The use and distribution terms for this software are covered by the
;; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
;; which can be found in the file LICENSE at the root of this distribution.
;; By using this software in any fashion, you are agreeing to be bound by
;; the terms of this license.
;; You must not remove this notice, or any other, from this software.

(ns conexp.contrib.exec
  (:use conexp.main
        [conexp.io.util :only (tmpfile with-out-writer)])
  (:use [clojure.java.shell :only (sh)]))

(ns-doc "Executing external programs with a common interface.")

;;;

(defn run-in-shell
  "Runs given commandline in an external shell process. Returns the
  output on stdout as result."
  [& cmdln]
  (let [result (apply sh cmdln)]
    (when-not (zero? (:exit result))
      (illegal-state "External program " (first cmdln) " returned with non-zero status: " (:err result)))
    (:out result)))

(defn- context-to-file
  "Writes context to a file using format and returns the name of the
  file."
  [context format]
  (let [^java.io.File file (tmpfile)]
    (write-context format context file)
    (.getAbsolutePath file)))

(defmacro define-external-program
  "Defines program-name to be a function calling an external
  program. Keywords are treated as arguments to the function and for
  every keyword there will be a corresponding function argument with
  the same name (without the colon). Special treatment is given
  to :input-file, which must be followed by a valid format to
  write-context. Its argument will be called \"context\" (thus you
  shall not name any other argument this way) and will get a context
  as input. This context will then be written to a file using the give
  format and this file will be given as an argument to the external
  process.

  Returns a map of :exit, :out and :err pointing to corresponding exit
  value, value at stdout and value at stderr, respectively."
  [program-name & command-line]
  (let [cmdln                      (map (fn [x]
                                          (if (keyword? x)
                                            x,
                                            (str x)))
                                        command-line),
        [before-input after-input] (split-with #(not= :input-file %) cmdln),
        input-file?                (first after-input)
        input-format               (second after-input)
        cmdln                      (if input-file?
                                     (concat before-input [:context] (drop 2 after-input))
                                     before-input),
        arguments                  (filter keyword? cmdln),
        arg-names                  (vec (map (comp symbol name) arguments))]
    `(defn ~program-name ~arg-names
       (run-in-shell ~@(replace (merge {:context `(context-to-file ~'context ~input-format)}
                                       (dissoc (zipmap arguments arg-names) :context))
                                cmdln)))))

;;;

(defmacro with-context-from-output
  "Runs body, expecting to get a string containing a formal
  context. Writes this string to a file and reads it in
  with read-context, returning the result."
  [& body]
  `(let [result# (do ~@body),
         file#   (tmpfile)]
     (with-out-writer file#
       (print result#))
     (read-context file#)))

(defmacro with-context-from-tmpfile
  "Creates a new temporary file, executes body and returns the result
  from calling read-context on file."
  [file & body]
  `(let [~file (tmpfile),
         result# (do ~@body)]
     (read-context ~file)))

;;;
{'fraction_non_alphanumeric': 0.09722222222222222, 'fraction_numerical': 0.0013354700854700855, 'mean_word_length': 2.184098639455782, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7291483', 'n_tokens_mistral': 2042, 'n_tokens_neox': 1980, 'n_words': 830}