<commit_before>(ns freestuffly.scraper
  (:require [clj-http.client :as client])
  (:require [hickory.core :as h])
  (:require [clojure.string :as string])
  (:require [clojure.set :as cset])
  (:require [hickory.select :as s]))

(def ^:private interesting-keywords
  #"(?i)paint|playstation|camping|emulsion|lamp|ladder|roller|xbox|mac|fish|marine|kitchen")

(def ^:private freecycle-group "southwark-freecycle")
(def ^:private freecycle-results-per-page 50)

(defn- interesting-finds
  [results]
  (cset/select
    (fn [result]
      (re-find (re-matcher interesting-keywords (first (:content (into {} result))))))
    (set results)))

(def ^:private my-group-urls
  (str "https://groups.freecycle.org/group/"
       freecycle-group
       "/posts/offer?page=1&resultsperpage="
       freecycle-results-per-page
       "&showall=off&include_offers=off&include_wanteds=off&include_receiveds=off&include_takens=off"))

(def ^:private site-tree
  (-> (client/get my-group-urls) :body h/parse h/as-hickory))

(defn- parsed-html
  [html]
  (-> (s/select
        (s/descendant
          (s/id "group_posts_table")
          (s/and (s/tag :tr))
          (s/and (s/tag :td) (s/nth-child 2))
          (s/and (s/tag :a) (s/nth-child 1)))
      html)))

(defn- rubbish-value?
  [content-values]
  (not= (content-values :content) ["See details"]))

(defn- content-for
  [html-vector]
  (filter rubbish-value? (map #(select-keys % [:attrs :content]) html-vector)))

(defn- presentable
  [results]
  (string/join "\n\n" results))

(defn scraped-content
  []
  (presentable
    (interesting-finds
      (content-for
        (parsed-html site-tree)))))<commit_msg>fixup<commit_after>(ns freestuffly.scraper
  (:require [clj-http.client :as client])
  (:require [hickory.core :as h])
  (:require [clojure.string :as string])
  (:require [clojure.set :as cset])
  (:require [hickory.select :as s]))

(def ^:private interesting-keywords
  #"(?i)paint|playstation|camping|emulsion|lamp|ladder|roller|xbox|mac|fish|marine|kitchen")

(def ^:private freecycle-group "southwark-freecycle")
(def ^:private freecycle-results-per-page 50)

(defn- interesting-finds
  [results]
  (cset/select
    (fn [result]
      (re-find
        (re-matcher interesting-keywords
                    (first (:content (into {} result))))))
    (set results)))

(def ^:private my-group-urls
  (str "https://groups.freecycle.org/group/"
       freecycle-group
       "/posts/offer?page=1&resultsperpage="
       freecycle-results-per-page
       "&showall=off&include_offers=off&include_wanteds=off&include_receiveds=off&include_takens=off"))

(def ^:private site-tree
  (-> (client/get my-group-urls) :body h/parse h/as-hickory))

(defn- parsed-html
  [html]
  (-> (s/select
        (s/descendant
          (s/id "group_posts_table")
          (s/and (s/tag :tr))
          (s/and (s/tag :td) (s/nth-child 2))
          (s/and (s/tag :a) (s/nth-child 1)))
      html)))

(defn- interesting-value?
  [content-values]
  (not= (content-values :content) ["See details"]))

(defn- content-for
  [html-vector]
  (filter interesting-value? (map #(select-keys % [:attrs :content]) html-vector)))

(defn- presentable
  [results]
  (string/join "\n\n" results))

(defn scraped-content
  []
  (presentable
    (interesting-finds
      (content-for
        (parsed-html site-tree)))))
{'fraction_non_alphanumeric': 0.17406962785114047, 'fraction_numerical': 0.003001200480192077, 'mean_word_length': 3.4262948207171315, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '222706', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1237, 'n_words': 235}