<commit_before>(ns scrape-summitpost-data.extract-pager-links
  (:require [reaver]
            [scrape-summitpost-data.ensure-sequence
             :refer [ensure-sequence]]))

(defn- extract-pager-links
  "Given a string containing a search results page from summitpost,
  return a sequence of pager links with page=\\d+ query params."
  [page]
  (->> (reaver/extract (reaver/parse page) [] ".pagertext" (reaver/attr :href))
       (ensure-sequence)
       (remove nil?)
       (map (partial re-find #".*page=\d+"))
       (remove nil?)))

(defn- extract-last-page
  "Given a sequence of links with a page=\\d+ query param, return the highest
  value of the param found in the links."
  [links]
  (->> links
       (map (partial re-find #"page=(\d+)"))
       (map #(nth % 1))
       (map #(Integer/parseInt %))
       (apply max)))

(defn extract-all-pager-links
  "Given a string containing a search results page from summitpost, return a
  sequence of links to all pages of search results. If there is no pagination,
  return nil."
  [page]
  (when-let [pager-links (-> page extract-pager-links not-empty)]
    (let [last-page (extract-last-page pager-links)
          template (first pager-links)]
      (->> (range 1 (+ 1 last-page))
           (map #(clojure.string/replace
                   template
                   #"page=\d+"
                   (str "page=" %)))))))
<commit_msg>Refactored extract-all-pager-links to read more clearly.<commit_after>(ns scrape-summitpost-data.extract-pager-links
  (:require [reaver]
            [scrape-summitpost-data.ensure-sequence
             :refer [ensure-sequence]]))

(defn- extract-pager-links
  "Given a string containing a search results page from summitpost,
  return a sequence of pager links with page=\\d+ query params."
  [page]
  (->> (reaver/extract (reaver/parse page) [] ".pagertext" (reaver/attr :href))
       (ensure-sequence)
       (remove nil?)
       (map (partial re-find #".*page=\d+"))
       (remove nil?)))

(defn- extract-last-page
  "Given a sequence of links with a page=\\d+ query param, return the highest
  value of the param found in the links."
  [links]
  (->> links
       (map (partial re-find #"page=(\d+)"))
       (map #(nth % 1))
       (map #(Integer/parseInt %))
       (apply max)))

(defn extract-all-pager-links
  "Given a string containing a search results page from summitpost, return a
  sequence of links to all pages of search results. If there is no pagination,
  return nil."
  [page]
  (when-let [pager-links (-> page extract-pager-links not-empty)]
    (let [last-page (extract-last-page pager-links)
          template (first pager-links)
          page-numbers (range 1 (+ 1 last-page))
          make-pager-link #(clojure.string/replace
                             template
                             #"page=\d+"
                             (str "page=" %))
          all-pager-links (map make-pager-link page-numbers)]
      all-pager-links)))
{'fraction_non_alphanumeric': 0.13745356298547787, 'fraction_numerical': 0.002026342451874367, 'mean_word_length': 2.4045977011494255, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1922276', 'n_tokens_mistral': 942, 'n_tokens_neox': 918, 'n_words': 279}