<commit_before>(ns cljs-rest.core
  (:refer-clojure :exclude [get])
  (:require-macros [cljs.core.async.macros :refer [go]])
  (:require [clojure.string :as string]
            [cljs.core.async :as async]
            [cljs.core.async.impl.protocols :refer [ReadPort]]
            [goog.Uri :as uri]
            [cljs-http.client :as http]))

;; Async helpers

(defn ensure-channel [x]
  (if (satisfies? ReadPort x)
      x
      (go x)))

;; HTTP

(def format-request-defaults
  {:edn      {:accept "application/edn"}
   :json     {:accept "application/json"}
   :transit  {:accept "application/transit+json"}})

(def format->params-key
  {:edn      :edn-params
   :json     :json-params
   :transit  :transit-params})

(def config
  (atom
    {:format nil
     :request-defaults nil}))

(defn configure-format [config format]
  (-> config
      (assoc :format format)
      (update :request-defaults merge (format-request-defaults format))))

(defn configure-format! [format]
  (swap! config #(configure-format % format)))

(defn request-defaults [x]
  (let [{:keys [format request-defaults]} @config]
    (if (coll? x)
        (merge request-defaults (format-request-defaults format))
        request-defaults)))

(defprotocol InfersRequestOptions
  "Provides extension semantics for generating request options for arbitrary
  types passed as body params. Example:

  (extend-protocol InfersRequestOptions
    MyTransitType
    (-opts [_] {:accept \"application/transit+json\"})
    (-params [this] {:transit-params this}))"

  (-opts [_] "Returns arbitrary request options for params type")
  (-params [_] "Returns a map of cljs-http params for request options"))

(extend-protocol InfersRequestOptions
  js/FormData
  (-params [form-data] {:body form-data})

  nil
  (-params [_] nil)

  default
  (-opts [x] (request-defaults x))
  (-params [x]
    ;; The conversion to JSON or similar serializations depends on `clj->js`,
    ;; which for our purposes is restricted to types satisfying ICollection.
    (let [k (if (coll? x)
                (format->params-key (:format @config) :params)
                :params)]
      {k x})))

(defrecord MultipartParams []
  InfersRequestOptions
  (-params [this] {:multipart-params (seq this)}))

(def multipart-params map->MultipartParams)

;; Seen in cljs-http issues, some APIs require multipart payloads to be sent in
;; a specific order.
(deftype OrderedMultipartParams [xs]
  InfersRequestOptions
  (-params [_] {:multipart-params xs}))

(defn ordered-multipart-params [xs]
  {:pre [(seqable? xs)]}
  (->OrderedMultipartParams xs))

(defn request-options [url opts]
  (let [params (:params opts)
        base-opts (-opts params)
        opts* (merge base-opts opts)
        params* (-params params)]
    (-> opts*
        (dissoc :params)
        (merge params*)
        (assoc :url url)
        (dissoc nil))))

(defn put-error! [opts error]
  (when-let [error-chan (or (:error-chan opts) (:error-chan @config))]
    (async/put! error-chan error)))

(defn with-keywords
  "Given a map `m`, returns a map preserving the original keys and appending
  keywords for each non-keyword field. Example:

  (with-keywords {\"a\" \"b\"})
  ;;=> {\"a\" \"b\"
  ;;    :a \"b\"}"
  [m]
  (reduce
    (fn [acc [k v]]
      (assoc acc (keyword k) v))
    m
    m))

(def link-pattern
  "Matches a `link-value` according to https://tools.ietf.org/html/rfc5988#section-5,
  capturing `URI-Reference` and a `rel` value for `link-param`, ignoring all other
  `link-param` fields if present."
  #"(?i)<(.*?)>(?:;\s*(?!rel)[^=]+=\".*?\")*;\s*rel=\"(.*?)\"(?:;\s*[^=]+=\".*?\")*")

(defn parse-url-query-params [url]
  (let [uri (uri/parse url)
        query-data (.getQueryData uri)]
    (when-not (.isEmpty query-data)
      (http/parse-query-params (str query-data)))))

(defn expand-link [link]
  (when link
    (let [matches (re-seq link-pattern link)]
      (if (empty? matches)
          link
          (reduce
            (fn [acc [_ url key-str]]
              (let [k (keyword key-str)
                    params (parse-url-query-params url)
                    parsed {:url url}
                    with-params (if (empty? params)
                                    parsed
                                    (assoc parsed :params params))]
                (assoc acc k with-params)))
            {}
            matches)))))

(defn expand-headers [headers]
  (-> headers
      with-keywords
      (update :link expand-link)))
<commit_msg>I got 99 problems, but now I got two easier to understand regexen<commit_after>(ns cljs-rest.core
  (:refer-clojure :exclude [get])
  (:require-macros [cljs.core.async.macros :refer [go]])
  (:require [clojure.string :as string]
            [cljs.core.async :as async]
            [cljs.core.async.impl.protocols :refer [ReadPort]]
            [goog.Uri :as uri]
            [cljs-http.client :as http]))

;; Async helpers

(defn ensure-channel [x]
  (if (satisfies? ReadPort x)
      x
      (go x)))

;; HTTP

(def format-request-defaults
  {:edn      {:accept "application/edn"}
   :json     {:accept "application/json"}
   :transit  {:accept "application/transit+json"}})

(def format->params-key
  {:edn      :edn-params
   :json     :json-params
   :transit  :transit-params})

(def config
  (atom
    {:format nil
     :request-defaults nil}))

(defn configure-format [config format]
  (-> config
      (assoc :format format)
      (update :request-defaults merge (format-request-defaults format))))

(defn configure-format! [format]
  (swap! config #(configure-format % format)))

(defn request-defaults [x]
  (let [{:keys [format request-defaults]} @config]
    (if (coll? x)
        (merge request-defaults (format-request-defaults format))
        request-defaults)))

(defprotocol InfersRequestOptions
  "Provides extension semantics for generating request options for arbitrary
  types passed as body params. Example:

  (extend-protocol InfersRequestOptions
    MyTransitType
    (-opts [_] {:accept \"application/transit+json\"})
    (-params [this] {:transit-params this}))"

  (-opts [_] "Returns arbitrary request options for params type")
  (-params [_] "Returns a map of cljs-http params for request options"))

(extend-protocol InfersRequestOptions
  js/FormData
  (-params [form-data] {:body form-data})

  nil
  (-params [_] nil)

  default
  (-opts [x] (request-defaults x))
  (-params [x]
    ;; The conversion to JSON or similar serializations depends on `clj->js`,
    ;; which for our purposes is restricted to types satisfying ICollection.
    (let [k (if (coll? x)
                (format->params-key (:format @config) :params)
                :params)]
      {k x})))

(defrecord MultipartParams []
  InfersRequestOptions
  (-params [this] {:multipart-params (seq this)}))

(def multipart-params map->MultipartParams)

;; Seen in cljs-http issues, some APIs require multipart payloads to be sent in
;; a specific order.
(deftype OrderedMultipartParams [xs]
  InfersRequestOptions
  (-params [_] {:multipart-params xs}))

(defn ordered-multipart-params [xs]
  {:pre [(seqable? xs)]}
  (->OrderedMultipartParams xs))

(defn request-options [url opts]
  (let [params (:params opts)
        base-opts (-opts params)
        opts* (merge base-opts opts)
        params* (-params params)]
    (-> opts*
        (dissoc :params)
        (merge params*)
        (assoc :url url)
        (dissoc nil))))

(defn put-error! [opts error]
  (when-let [error-chan (or (:error-chan opts) (:error-chan @config))]
    (async/put! error-chan error)))

(defn with-keywords
  "Given a map `m`, returns a map preserving the original keys and appending
  keywords for each non-keyword field. Example:

  (with-keywords {\"a\" \"b\"})
  ;;=> {\"a\" \"b\"
  ;;    :a \"b\"}"
  [m]
  (reduce
    (fn [acc [k v]]
      (assoc acc (keyword k) v))
    m
    m))

(def link-pattern
  "Matches a `link-value` according to https://tools.ietf.org/html/rfc5988#section-5,
  capturing `URI-Reference`, capturing `link-param` for further processing."
  #"<(.*?)>(.*?)(?=<|$)")

(def rel-pattern
  "Matches a `rel` `link-param`, capturing its value"
  #"(?i);\s*rel=\"(.*?)\"")

(defn parse-url-query-params [url]
  (let [uri (uri/parse url)
        query-data (.getQueryData uri)]
    (when-not (.isEmpty query-data)
      (http/parse-query-params (str query-data)))))

(defn expand-link [link]
  (when link
    (let [matches (re-seq link-pattern link)]
      (if (empty? matches)
          link
          (reduce
            (fn [acc [_ url link-params]]
              (let [[_ rel] (re-find rel-pattern link-params)
                    k (keyword rel)
                    params (parse-url-query-params url)
                    parsed {:url url}
                    with-params (if (empty? params)
                                    parsed
                                    (assoc parsed :params params))]
                (assoc acc k with-params)))
            {}
            matches)))))

(defn expand-headers [headers]
  (-> headers
      with-keywords
      (update :link expand-link)))

{'fraction_non_alphanumeric': 0.1505897916436997, 'fraction_numerical': 0.0013228971447469958, 'mean_word_length': 2.6014291385470423, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5413721', 'n_tokens_mistral': 2988, 'n_tokens_neox': 2877, 'n_words': 837}