<commit_before>(ns clj-http.core2
  "Core HTTP request/response implementation. Rewrite for Apache 4.3"
  (:require [clj-http.conn-mgr :as conn]
            [clj-http.headers :as headers]
            [clj-http.multipart :as mp]
            [clj-http.util :refer [opt]]
            [clojure.pprint])
  (:import (java.io ByteArrayOutputStream File FilterInputStream InputStream)
           (java.net URI)
           (org.apache.http HttpHost)
           (org.apache.http.client.methods HttpGet)
           (org.apache.http.client.protocol HttpClientContext)
           (org.apache.http.config RegistryBuilder)
           (org.apache.http.conn.routing HttpRoute)
           (org.apache.http.conn.socket PlainConnectionSocketFactory)
           (org.apache.http.impl.client HttpClients)
           (org.apache.http.impl.conn PoolingHttpClientConnectionManager)))

(defn http-route []
  ;; TODO add proxy support
  (HttpRoute. (HttpHost. "www.google.com" 80 "https")))

(defn registry-builder []
  (-> (RegistryBuilder/create)
      (.register "http" PlainConnectionSocketFactory/INSTANCE)
      (.build)))

(defn pooling-conn-mgr []
  (PoolingHttpClientConnectionManager. (registry-builder)))

(defn http-client [conn-mgr]
  (-> (HttpClients/custom)
      (.setConnectionManager conn-mgr)
      (.build)))

(defn http-get []
  (HttpGet. "http://www.google.com"))

(defn http-context []
  (HttpClientContext/create))

(defn request [{:keys [] :as req}]
  (let [conn-mgr (pooling-conn-mgr)
        client (http-client conn-mgr)
        context (http-context)
        get-req (http-get)
        response (.execute client get-req context)
        entity (.getEntity response)]
    {:body (.getContent entity)
     :length (.getContentLength entity)
     :chunked? (.isChunked entity)
     :repeatable? (.isRepeatable entity)
     :streaming? (.isStreaming entity)<commit_msg>Initial basic https support<commit_after>(ns clj-http.core2
  "Core HTTP request/response implementation. Rewrite for Apache 4.3"
  (:require [clj-http.conn-mgr :as conn]
            [clj-http.headers :as headers]
            [clj-http.multipart :as mp]
            [clj-http.util :refer [opt]]
            [clojure.pprint])
  (:import (java.io ByteArrayOutputStream File FilterInputStream InputStream)
           (java.net URI)
           (javax.net.ssl SSLContext)
           (org.apache.http HttpHost)
           (org.apache.http.client.methods HttpGet)
           (org.apache.http.client.protocol HttpClientContext)
           (org.apache.http.config RegistryBuilder)
           (org.apache.http.conn.routing HttpRoute)
           (org.apache.http.conn.ssl BrowserCompatHostnameVerifier
                                     SSLConnectionSocketFactory SSLContexts)
           (org.apache.http.conn.socket PlainConnectionSocketFactory)
           (org.apache.http.impl.client HttpClients)
           (org.apache.http.impl.conn PoolingHttpClientConnectionManager)))

(defn http-route []
  ;; TODO add proxy support
  (HttpRoute. (HttpHost. "www.google.com" 80 "https")))

(defn ssl-context []
  (SSLContexts/createSystemDefault))

(defn hostname-verifier []
  (BrowserCompatHostnameVerifier.))

(defn registry-builder []
  (-> (RegistryBuilder/create)
      (.register "http" PlainConnectionSocketFactory/INSTANCE)
      (.register "https" (SSLConnectionSocketFactory.
                          (ssl-context) (hostname-verifier)))
      (.build)))

(defn pooling-conn-mgr []
  (PoolingHttpClientConnectionManager. (registry-builder)))

(defn http-client [conn-mgr]
  (-> (HttpClients/custom)
      (.setConnectionManager conn-mgr)
      (.build)))

(defn http-get []
  (HttpGet. "https://www.google.com"))

(defn http-context []
  (HttpClientContext/create))

(defn request [{:keys [cookie-store] :as req}]
  (let [conn-mgr (pooling-conn-mgr)
        client (http-client conn-mgr)
        context (http-context)
        get-req (http-get)
        response (.execute client get-req context)
        entity (.getEntity response)]
    ;; TODO response, conn-mgr, and client needs to be closed
    {:body (.getContent entity)
     :length (.getContentLength entity)
     :chunked? (.isChunked entity)
     :repeatable? (.isRepeatable entity)
     :streaming? (.isStreaming entity)
{'fraction_non_alphanumeric': 0.1281809613572102, 'fraction_numerical': 0.00235626767200754, 'mean_word_length': 2.783422459893048, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5769812', 'n_tokens_mistral': 1318, 'n_tokens_neox': 1291, 'n_words': 273}