<commit_before>(ns qcast.infoq.site
  (:require [clojure.string   :refer [split]]
            [clj-http.client  :as http]
            [clj-http.cookies :as cookies]
            [qcast.html       :as html]
            [qcast.util       :refer :all]
            [taoensso.timbre  :refer :all])
  (:refer-clojure :exclude [resolve]))


;;; Globals

(def ^:private username "<EMAIL>")

(def ^:private password "<PASSWORD>")

(def ^:private ios-user-agent
  (str "Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) "
       "AppleWebKit/537.51.1 (KHTML, like Gecko) "
       "Version/7.0 "
       "Mobile/11A465 "
       "Safari/9537.53"))

(def ^:private http-options
  {:headers {"User-Agent" ios-user-agent}
   :as      :stream})


;;; Internals

(defn- HEAD [url & [opts]]
  (debug :head url opts)
  (http/head url (merge http-options {:follow-redirects false} opts)))

(defn- GET [url & [opts]]
  (debug :get url opts)
  (http/get url (merge http-options opts)))
<commit_msg>fixes unverified SSL certificate problem, makes the mp3 links work again<commit_after>(ns qcast.infoq.site
  (:require [clojure.string   :refer [split]]
            [clj-http.client  :as http]
            [clj-http.cookies :as cookies]
            [qcast.html       :as html]
            [qcast.util       :refer :all]
            [taoensso.timbre  :refer :all])
  (:refer-clojure :exclude [resolve]))


;;; Globals

(def ^:private username "<EMAIL>")

(def ^:private password "<PASSWORD>")

(def ^:private ios-user-agent
  (str "Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) "
       "AppleWebKit/537.51.1 (KHTML, like Gecko) "
       "Version/7.0 "
       "Mobile/11A465 "
       "Safari/9537.53"))

(def ^:private http-options
  {:headers {"User-Agent" ios-user-agent}
   :insecure? true
   :as      :stream})


;;; Internals

(defn- HEAD [url & [opts]]
  (debug :head url opts)
  (http/head url (merge http-options {:follow-redirects false} opts)))

(defn- GET [url & [opts]]
  (debug :get url opts)
  (http/get url (merge http-options opts)))

{'fraction_non_alphanumeric': 0.15823412698412698, 'fraction_numerical': 0.023313492063492064, 'mean_word_length': 2.6276978417266186, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5596939', 'n_tokens_mistral': 775, 'n_tokens_neox': 748, 'n_words': 192}