<commit_before>(ns bonews-rest.scraper.threads
  (:require [bonews-rest.scraper.utils :as utils]
            [bonews-rest.scraper.replies :as br]
            [bonews-rest.scraper.bulbs :as bulbs]
            [net.cgrand.enlive-html :as html]
            [guangyin.core :as t]
            [guangyin.format :as f]
            [clojure.string :as str]
            [clj-webdriver.taxi :as web]))

(def url-prefix "http://bo-ne.ws/forum/read.php?")

(def link-href [[:a (html/attr? :href)]])
(def link-label [:h4 [:a (html/attr? :href)]])

(def custom-formatter (f/date-time-formatter "MMMM dd, yyyy hh:mma"))

(defn get-thread-url
  "Construct URL for a single thread of a subforum"
  [subforum-id thread-id]
  (str url-prefix subforum-id "," thread-id))

(defn get-replies-table
  [thread-url]
  (-> thread-url
      (utils/fetch-url)
      (html/select [:div#phorum :table])
      last))

(defn get-reply-title
  [cols]
  (-> cols
      first
      (html/select link-label)
      last
      (:content)
      first))

(defn get-reply-url
  [cols]
  (-> cols
      first
      (html/select link-href)
      first
      (:attrs)
      (:href)))

(defn get-reply-id
  [reply-url]
  (-> reply-url
      (str/split #"\-")
      last
      (Integer/parseInt)))

(defn get-reply-post-time
  [cols]
  @(->  cols
        last
        (:content)
        first
        (t/local-date-time custom-formatter)))

(defn get-thread-id
  [reply-url]
  (-> reply-url
      (str/split #"\,")
      second
      (Integer/parseInt)))

(defn consolidate-users
  [user bulbs]
  (if (some? (:users bulbs))
    (into (list user) (:users bulbs))
    (list user)))

(defn get-data-helper
  [rows]
  (for [row rows
    :let [cols         (utils/get-cols        row)
          reply-url    (get-reply-url         cols)
          reply-id     (get-reply-id          reply-url)
          reply-title  (get-reply-title       cols)
          post-time    (get-reply-post-time   cols)
          bulbs        (bulbs/get-data        reply-url)
          user-url     (utils/get-user-url    cols)
          user-id      (utils/get-user-id     user-url)
          username     (utils/get-username    cols)
          reply        {
            :id         reply-id
            :title      reply-title
            :url        reply-url
            :post-time  post-time
            :user       user-id
          }
          user         {
            :id         user-id
            :name       username
            :url        user-url
          }
          users        (consolidate-users user bulbs)]]
    [reply users]))

(defn collate-users
  [tdata]
  (distinct (reduce into '() (map second tdata))))

(defn get-data-by-url
  [url]
  (web/set-driver! {:browser :firefox})
  (let [table      (get-replies-table  url)
        rows       (utils/get-rows     table)
        thread-id  (get-thread-id      url)
        tdata      (get-data-helper    rows)
        replies    (map first          tdata)
        users      (collate-users      tdata)
        data       {
                      :thread {
                        :id       thread-id
                        :replies  (map :id replies)
                      }
                      :replies replies
                      :users   users
                    }]
    (web/close)
    data))

; (defn get-data-by-ids
;   [subforum-id thread-id]
;   (let [url    (get-thread-url     subforum-id thread-id)
;         table  (get-replies-table  url)
;         rows   (utils/get-rows     table)]
;     {
;       :thread   (get-thread-data   rows thread-id)
;       :replies  (get-replies-data  rows)
;       :users    (get-users-data    rows)
;     }))

(defn rm-users-from-bulb
  [reply]
  (update-in reply [:bulbs] dissoc :users))

(defn clean-replies
  [replies]
  (utils/remove-nils (map rm-users-from-bulb replies)))

(defn clean-data
  [data]
  (let [replies          (:replies       data)
        cleaned-replies  (clean-replies  replies)]
  (assoc data :replies cleaned-replies)))

(defn get-data
  ([url]
    ; (clean-data (get-data-by-url url)))
    (get-data-by-url url))
  ([subforum-id thread-id]
    nil))
    ; (get-data-by-ids subforum-id thread-id)))<commit_msg>readded bulb data to replies<commit_after>(ns bonews-rest.scraper.threads
  (:require [bonews-rest.scraper.utils :as utils]
            [bonews-rest.scraper.replies :as br]
            [bonews-rest.scraper.bulbs :as bulbs]
            [net.cgrand.enlive-html :as html]
            [guangyin.core :as t]
            [guangyin.format :as f]
            [clojure.string :as str]
            [clj-webdriver.taxi :as web]))

(def url-prefix "http://bo-ne.ws/forum/read.php?")

(def link-href [[:a (html/attr? :href)]])
(def link-label [:h4 [:a (html/attr? :href)]])

(def custom-formatter (f/date-time-formatter "MMMM dd, yyyy hh:mma"))

(defn get-thread-url
  "Construct URL for a single thread of a subforum"
  [subforum-id thread-id]
  (str url-prefix subforum-id "," thread-id))

(defn get-replies-table
  [thread-url]
  (-> thread-url
      (utils/fetch-url)
      (html/select [:div#phorum :table])
      last))

(defn get-reply-title
  [cols]
  (-> cols
      first
      (html/select link-label)
      last
      (:content)
      first))

(defn get-reply-url
  [cols]
  (-> cols
      first
      (html/select link-href)
      first
      (:attrs)
      (:href)))

(defn get-reply-id
  [reply-url]
  (-> reply-url
      (str/split #"\-")
      last
      (Integer/parseInt)))

(defn get-reply-post-time
  [cols]
  @(->  cols
        last
        (:content)
        first
        (t/local-date-time custom-formatter)))

(defn get-thread-id
  [reply-url]
  (-> reply-url
      (str/split #"\,")
      second
      (Integer/parseInt)))

(defn consolidate-users
  [user bulbs]
  (if (some? (:users bulbs))
    (into (list user) (:users bulbs))
    (list user)))

(defn rm-nil-bulb
  [reply]
  (if (nil? (reply :bulbs))
    (dissoc reply :bulbs)
    reply))

(defn get-data-helper
  [rows]
  (for [row rows
    :let [cols         (utils/get-cols        row)
          reply-url    (get-reply-url         cols)
          reply-id     (get-reply-id          reply-url)
          reply-title  (get-reply-title       cols)
          post-time    (get-reply-post-time   cols)
          bulbs        (bulbs/get-data        reply-url)
          user-url     (utils/get-user-url    cols)
          user-id      (utils/get-user-id     user-url)
          username     (utils/get-username    cols)
          reply        {
            :id         reply-id
            :title      reply-title
            :url        reply-url
            :post-time  post-time
            :user       user-id
            :bulbs      (dissoc bulbs :users)
          }
          user         {
            :id         user-id
            :name       username
            :url        user-url
          }
          users        (consolidate-users user bulbs)]]
    [(rm-nil-bulb reply) users]))

(defn collate-users
  [tdata]
  (distinct (reduce into '() (map second tdata))))

(defn get-data-by-url
  [url]
  (web/set-driver! {:browser :firefox})
  (let [table      (get-replies-table  url)
        rows       (utils/get-rows     table)
        thread-id  (get-thread-id      url)
        tdata      (get-data-helper    rows)
        replies    (map first          tdata)
        users      (collate-users      tdata)
        data       {
                      :thread {
                        :id       thread-id
                        :replies  (map :id replies)
                      }
                      :replies replies
                      :users   users
                    }]
    (web/close)
    data))

; (defn get-data-by-ids
;   [subforum-id thread-id]
;   (let [url    (get-thread-url     subforum-id thread-id)
;         table  (get-replies-table  url)
;         rows   (utils/get-rows     table)]
;     {
;       :thread   (get-thread-data   rows thread-id)
;       :replies  (get-replies-data  rows)
;       :users    (get-users-data    rows)
;     }))

(defn get-data
  ([url]
    (get-data-by-url url))
  ([subforum-id thread-id]
    nil))
    ; (get-data-by-ids subforum-id thread-id)))
{'fraction_non_alphanumeric': 0.13605192260592702, 'fraction_numerical': 0.0002449179524859172, 'mean_word_length': 1.5160197165742453, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6797353', 'n_tokens_mistral': 2870, 'n_tokens_neox': 2807, 'n_words': 601}