<commit_before>(ns async.client
  (:use [de.kotka.lazymap :only (lazy-hash-map)])
  (:import (com.ning.http.client AsyncHttpClient AsyncHandler
				 HttpResponseStatus HttpResponseHeaders
				 HttpResponseBodyPart Request RequestBuilder
				 RequestType)))

(def ahc (AsyncHttpClient.))

(defstruct status
  :protocol :major :minor :code :text)

;; default set of callbacks
(defn body-collect [#^byteL bytes]
  ;; TODO append to rest of the body collected already
)

(defn body-completed []
  ;; TODO return completed body
)

(defn ignore-headers [id headers])

(defn print-headers [id headers]
  (doall (map #(println (str id "> " % ": " (get headers %))) (keys headers))))

(defn accept-ok [id status]
  (if (not (= (:code status) 200)) :abort))

(defn print-status [id st]
  (println (str id "> "
		(:protocol st) "/" (:major st) "." (:minor st) " "
		(:code st) " " (:text st))))

(defn status-callback [id status]
    (println (str "Status code: " (status :code)))
    :abort)

(defn- default-status-callback [id status])

(defn- convert-method [method]
  "Converts clj method (:get, :put, ...) to Async Client specific.
   Defaults to :get"
  (cond
   (= method :get) RequestType/GET
   (= method :post) RequestType/POST
   (= method :put) RequestType/PUT
   (= method :delete) RequestType/DELETE
   (= method :head) RequestType/HEAD
   :default RequestType/GET))

(defn prepare-request [method #^String url]
  "Prepares request."
  {:tag Request}
  (let [mtd (convert-method method)
	rb (RequestBuilder. mtd)]
    (.. rb (setUrl url) (build))))

(defn prepare-get [#^String url]
  "Prepares GET reqeust for given url."
  {:tag Request}
  (prepare-request :get url))

(defn execute-request
  "Executes provided reqeust with given callback functions."
  ([#^Request req]
     (execute-request req body-collect body-completed))
  ([#^Request req body-fn completed-fn]
     (execute-request req body-fn completed-fn print-headers))
  ([#^Request req body-fn completed-fn headers-fn]
     (execute-request req body-fn completed-fn headers-fn print-status))
  ([#^Request req body-fn completed-fn headers-fn status-fn]
     (let [id (gensym "RequestId__")]
       (.executeRequest
	ahc req
	(proxy [AsyncHandler] []
	  (onStatusReceived [#^HttpResponseStatus resp]
	    (let [stat (lazy-hash-map
			:protocol (.getProtocolName resp)
			:major (.getProtocolMajorVersion resp)
			:minor (.getProtocolMinorVersion resp)
			:code (.getStatusCode resp)
			:text (.getStatusText resp))
		  action (status-fn id stat)]
	      (if (= action :abort)
		com.ning.http.client.AsyncHandler$STATE/ABORT
		com.ning.http.client.AsyncHandler$STATE/CONTINUE)))
	  (onHeadersReceived [#^HttpResponseHeaders response]
            (let [action (headers-fn id (.getHeadersMap response))]
	      (if (= action :abort)
		com.ning.http.client.AsyncHandler$STATE/ABORT
		com.ning.http.client.AsyncHandler$STATE/CONTINUE)))
	  (onBodyPartReceived [#^HttpResponseBodyPart response]
 	    (do
	      (println response)
	      com.ning.http.client.AsyncHandler$STATE/ABORT))
	  (onCompleted []
	    (do
	      (println "Completed")
	      200))
	  (onThrowable [#^Throwable t]
	    (println (.getMessage t))))))))<commit_msg>convert-method<commit_after>(ns async.client
  (:use [de.kotka.lazymap :only (lazy-hash-map)])
  (:import (com.ning.http.client AsyncHttpClient AsyncHandler
				 HttpResponseStatus HttpResponseHeaders
				 HttpResponseBodyPart Request RequestBuilder
				 RequestType)))

(def ahc (AsyncHttpClient.))

(defstruct status
  :protocol :major :minor :code :text)

;; default set of callbacks
(defn body-collect [#^byteL bytes]
  ;; TODO append to rest of the body collected already
)

(defn body-completed []
  ;; TODO return completed body
)

(defn ignore-headers [id headers])

(defn print-headers [id headers]
  (println (bean headers))
  (doall (map #(println (str id "< " % ": " (get headers %))) (keys headers)))
  (if (not (contains? headers "content-type")) :abort ))

(defn accept-ok [id status]
  (if (not (= (:code status) 200)) :abort))

(defn print-status [id st]
  (println (str id "< "
		(:protocol st) "/" (:major st) "." (:minor st) " "
		(:code st) " " (:text st))))

(defn status-callback [id status]
    (println (str "Status code: " (status :code)))
    :abort)

(defn- default-status-callback [id status])

(defn- convert-method [method]
  "Converts clj method (:get, :put, ...) to Async Client specific.
   Defaults to :get"
  (cond
   (= method :get) RequestType/GET
   (= method :post) RequestType/POST
   (= method :put) RequestType/PUT
   (= method :delete) RequestType/DELETE
   (= method :head) RequestType/HEAD
   :default RequestType/GET))

(defn prepare-request [method #^String url]
  "Prepares request."
  {:tag Request}
  (let [mtd (convert-method method)
	rb (RequestBuilder. mtd)]
    (.. rb (setUrl url) (build))))

(defn prepare-get [#^String url]
  "Prepares GET reqeust for given url."
  {:tag Request}
  (prepare-request :get url))

(defn convert-action
  "Converts action (:abort, nil) to Async client STATE."
  [action]
  {:tag com.ning.http.client.AsyncHandler$STATE}
  (if (= action :abort)
    com.ning.http.client.AsyncHandler$STATE/ABORT
    com.ning.http.client.AsyncHandler$STATE/CONTINUE))

(defn execute-request
  "Executes provided reqeust with given callback functions."
  ([#^Request req]
     (execute-request req body-collect body-completed))
  ([#^Request req body-fn completed-fn]
     (execute-request req body-fn completed-fn print-headers))
  ([#^Request req body-fn completed-fn headers-fn]
     (execute-request req body-fn completed-fn headers-fn print-status))
  ([#^Request req body-fn completed-fn headers-fn status-fn]
     (println
      (str req " " body-fn " " completed-fn " " headers-fn " " status-fn))
     (let [id (gensym "req-id__")]
       (.executeRequest
	ahc req
	(proxy [AsyncHandler] []
	  (onStatusReceived [#^HttpResponseStatus resp]
	    (let [stat (lazy-hash-map
			:protocol (.getProtocolName resp)
			:major (.getProtocolMajorVersion resp)
			:minor (.getProtocolMinorVersion resp)
			:code (.getStatusCode resp)
			:text (.getStatusText resp))]
	      convert-method (status-fn id stat)))
	  (onHeadersReceived [#^HttpResponseHeaders response]
	    (println "onHeadersReceived")
	    convert-method (headers-fn id (.getHeadersMap response)))
	  (onBodyPartReceived [#^HttpResponseBodyPart response]
	    (do
	      (println response)
	      com.ning.http.client.AsyncHandler$STATE/ABORT))
	  (onCompleted []
	    (do
	      (println "Completed")
	      200))
	  (onThrowable [#^Throwable t]
	    (println t)
	    (.printStackTrace t)))))))
{'fraction_non_alphanumeric': 0.14421624886741166, 'fraction_numerical': 0.001812141347025068, 'mean_word_length': 3.7004968062455643, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5581334', 'n_tokens_mistral': 2290, 'n_tokens_neox': 2118, 'n_words': 601}