<commit_before>(ns appengine-magic.services.url-fetch
  (:import [com.google.appengine.api.urlfetch
            URLFetchServiceFactory
            FetchOptions
            FetchOptions$Builder
            HTTPHeader
            HTTPRequest
            HTTPMethod]))

(defonce *urlfetch-service* (atom nil))

(defn get-urlfetch-service []
  (do (when (nil? @*urlfetch-service*)
	(reset! *urlfetch-service* (URLFetchServiceFactory/getURLFetchService)))
      @*urlfetch-service*))

(defn- urlify [url] (if (string? url) (java.net.URL. url) url))

(defrecord HTTPResponse [content
			 final-url
			 headers
			 response-code])

(defn- parse-headers [headers]
  (zipmap (map #(keyword (.getName %)) headers)
	  (map #(.getValue %) headers)))

(defn- make-headers [header-map]
  (map
   (fn [[name-key val]]
     (com.google.appengine.api.urlfetch.HTTPHeader.
      (if (keyword? name-key)
          (.substring (str name-key) 1)
          (str name-key))
      val))
   header-map))

(defn- parse-response
  [^com.google.appengine.api.urlfetch.HTTPResponse r]
  (HTTPResponse. (.getContent r)
		 (.getFinalUrl r)
		 (parse-headers (.getHeaders r))
		 (.getResponseCode r)))

(defn make-request
  [url &
   {:keys [method headers payload allow-truncate follow-redirects deadline]
    :or {method :get
          headers {}
          payload nil
          allow-truncate FetchOptions/DEFAULT_ALLOW_TRUNCATE
          follow-redirects FetchOptions/DEFAULT_FOLLOW_REDIRECTS
          deadline FetchOptions/DEFAULT_DEADLINE}}]
  (let [fetch-options (FetchOptions$Builder/withDefaults)]
    (if allow-truncate
      (.allowTruncate fetch-options)
      (.disallowTruncate fetch-options))
    (if follow-redirects
      (.followRedirects fetch-options)
      (.doNotFollowRedirects fetch-options))
    (.setDeadline fetch-options (double deadline))
    (let [method-obj (method {:delete HTTPMethod/DELETE
                              :get    HTTPMethod/GET<commit_msg>Fixed a bug with omitting deadlines in calls to URL fetch.<commit_after>(ns appengine-magic.services.url-fetch
  (:import [com.google.appengine.api.urlfetch
            URLFetchServiceFactory
            FetchOptions
            FetchOptions$Builder
            HTTPHeader
            HTTPRequest
            HTTPMethod]))

(defonce *urlfetch-service* (atom nil))

(defn get-urlfetch-service []
  (do (when (nil? @*urlfetch-service*)
	(reset! *urlfetch-service* (URLFetchServiceFactory/getURLFetchService)))
      @*urlfetch-service*))

(defn- urlify [url] (if (string? url) (java.net.URL. url) url))

(defrecord HTTPResponse [content
			 final-url
			 headers
			 response-code])

(defn- parse-headers [headers]
  (zipmap (map #(keyword (.getName %)) headers)
	  (map #(.getValue %) headers)))

(defn- make-headers [header-map]
  (map
   (fn [[name-key val]]
     (com.google.appengine.api.urlfetch.HTTPHeader.
      (if (keyword? name-key)
          (.substring (str name-key) 1)
          (str name-key))
      val))
   header-map))

(defn- parse-response
  [^com.google.appengine.api.urlfetch.HTTPResponse r]
  (HTTPResponse. (.getContent r)
		 (.getFinalUrl r)
		 (parse-headers (.getHeaders r))
		 (.getResponseCode r)))

(defn make-request
  [url &
   {:keys [method headers payload allow-truncate follow-redirects deadline]
    :or {method :get
          headers {}
          payload nil
          allow-truncate FetchOptions/DEFAULT_ALLOW_TRUNCATE
          follow-redirects FetchOptions/DEFAULT_FOLLOW_REDIRECTS
          deadline FetchOptions/DEFAULT_DEADLINE}}]
  (let [fetch-options (FetchOptions$Builder/withDefaults)]
    (if allow-truncate
      (.allowTruncate fetch-options)
      (.disallowTruncate fetch-options))
    (if follow-redirects
      (.followRedirects fetch-options)
      (.doNotFollowRedirects fetch-options))
    (when-not (nil? deadline)
      (.setDeadline fetch-options (double deadline)))
    (let [method-obj (method {:delete HTTPMethod/DELETE
                              :get    HTTPMethod/GET
{'fraction_non_alphanumeric': 0.12149298597194388, 'fraction_numerical': 0.000501002004008016, 'mean_word_length': 2.810114503816794, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3619691', 'n_tokens_mistral': 1290, 'n_tokens_neox': 1208, 'n_words': 274}