<commit_before>(ns scrape-trs.core
  (:require [scrape-trs.cascade-climbers.core :as cc]
            [scrape-trs.summitpost.core :as sp]))

(defn get-implementation
  [url]
  (let [base-url (re-find #"https?://.*?/" url)]
    (condp = base-url
      cc/base-url (cc/->CCScrapeTripReport)
      sp/base-url (sp/->SPScrapeTripReport))))

(defn get-trip-reports
  "implementation: type that implements scrape-trs.protocol/ScrapeTripReport
  list-page: page (as a string) that lists trip reports and corresponds to the given implementation

  Scrape all trip reports from the list-page, paging through pagination if
  necessary, and return a sequence of scrape-trs.protocol/TripReport instances."
  [implementation list-page]
  (let [pager-urls (.extract-pager-urls implementation list-page)
        list-pages (if pager-urls (map slurp pager-urls) [list-page])
        trip-report-urls (mapcat #(.extract-trip-report-urls implementation %)
                                 list-pages)
        trip-reports (map #(.extract-trip-report implementation %1 %2)
                          trip-report-urls
                          (map slurp trip-report-urls))]
    trip-reports))

(defn save-trip-reports!
  ""
  [f trip-reports])<commit_msg>Remove save-trip-reports! since scrape-trs is now a library.<commit_after>(ns scrape-trs.core
  (:require [scrape-trs.cascade-climbers.core :as cc]
            [scrape-trs.summitpost.core :as sp]))

(defn get-implementation
  [url]
  (let [base-url (re-find #"https?://.*?/" url)]
    (condp = base-url
      cc/base-url (cc/->CCScrapeTripReport)
      sp/base-url (sp/->SPScrapeTripReport))))

(defn get-trip-reports
  "implementation: type that implements scrape-trs.protocol/ScrapeTripReport
  list-page: page (as a string) that lists trip reports and corresponds to the given implementation

  Scrape all trip reports from the list-page, paging through pagination if
  necessary, and return a sequence of scrape-trs.protocol/TripReport instances."
  [implementation list-page]
  (let [pager-urls (.extract-pager-urls implementation list-page)
        list-pages (if pager-urls (map slurp pager-urls) [list-page])
        trip-report-urls (mapcat #(.extract-trip-report-urls implementation %)
                                 list-pages)
        trip-reports (map #(.extract-trip-report implementation %1 %2)
                          trip-report-urls
                          (map slurp trip-report-urls))]
    trip-reports))
{'fraction_non_alphanumeric': 0.12053571428571429, 'fraction_numerical': 0.0016233766233766235, 'mean_word_length': 2.717948717948718, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4543561', 'n_tokens_mistral': 812, 'n_tokens_neox': 773, 'n_words': 198}