<commit_before>(ns ghoul.parse-rss
  (:require [cljs-time.format :as format]
            [cljs-time.coerce :as coerce]
            [cuerdas.core :as str]))

; Wed, 10 Dec 2014 13:00:00 +0100
(defn parse-date [string-date]
  (coerce/to-long
   (try (format/parse (format/formatters :rfc822) (str/replace-first string-date #"GMT" "+0000"))
        (catch js/Object e (.log js/console string-date e)))))

(defn add-tag
  ([map node tag] (add-tag map node tag tag identity))
  ([map node target-keyword tag] (add-tag map node target-keyword tag identity))
  ([map node target-keyword tag transformer]
   (try
     (assoc map (keyword target-keyword) (-> node (.getElements tag) (aget 0) .getText transformer))
     (catch js/Object e
       (do (.log js/console (str e) node)
           (assoc map (keyword tag) nil))))))

(defn parse-node [node]
  (-> {}
      (add-tag node "title")
      (add-tag node "link")
      (add-tag node "description")
      (add-tag node "creator" "dc:creator")
      (add-tag node "pubDate" "pubDate" parse-date)
      (add-tag node "uid" "guid")))<commit_msg>Added trace when tag not found<commit_after>(ns ghoul.parse-rss
  (:require [cljs-time.format :as format]
            [cljs-time.coerce :as coerce]
            [cuerdas.core :as str]))

; Wed, 10 Dec 2014 13:00:00 +0100
(defn parse-date [string-date]
  (coerce/to-long
   (try (format/parse (format/formatters :rfc822) (str/replace-first string-date #"GMT" "+0000"))
        (catch js/Object e (.log js/console string-date e)))))

(defn add-tag
  ([map node tag] (add-tag map node tag tag identity))
  ([map node target-keyword tag] (add-tag map node target-keyword tag identity))
  ([map node target-keyword tag transformer]
   (try
     (let [tag-elements (-> node (.getElements tag))
           to-insert (if (and (not (nil? tag-elements))
                              (> (.-length tag-elements) 0))
                       (-> tag-elements (aget 0) .getText transformer)
                       (do (.log js/console (str "Not found " tag)) nil))]
       (assoc map (keyword target-keyword) to-insert))
     (catch js/Object e
       (do (.log js/console (str "Error processing " target-keyword " " tag) node e)
           (assoc map (keyword tag) nil))))))

(defn parse-node [node]
  (-> {}
      (add-tag node "title")
      (add-tag node "link")
      (add-tag node "description")
      (add-tag node "creator" "dc:creator")
      (add-tag node "pubDate" "pubDate" parse-date)
      (add-tag node "uid" "guid")))
{'fraction_non_alphanumeric': 0.15535143769968052, 'fraction_numerical': 0.019568690095846646, 'mean_word_length': 2.6043165467625897, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4135496', 'n_tokens_mistral': 895, 'n_tokens_neox': 826, 'n_words': 257}