<commit_before>(ns org.altlaw.extract.scrape.handler
  (:require [org.altlaw.util.jruby :as ruby]
            [clojure.contrib.walk :as walk]
            [clojure.contrib.singleton :as sing]))

(def scraper-handler
     (sing/per-thread-singleton
      (fn []
        (ruby/eval-jruby "require 'org/altlaw/extract/scrape/scraper_handler'")
        (ruby/eval-jruby "ScraperHandler.new"))))

(defn run-scrapers [download]
  (assert (map? download))
  (assert (contains? download :request_uri))
  (assert (contains? download :response_body_bytes))
  (map ruby/convert-jruby
       (ruby/eval-jruby "$handler.parse(Download.from_map($download))"
                        {:download (walk/stringify-keys download)
                         :handler (scraper-handler)})))

(defn all-requests []
  (map ruby/convert-jruby<commit_msg>handler.clj: don't assert :response_body_bytes key<commit_after>(ns org.altlaw.extract.scrape.handler
  (:require [org.altlaw.util.jruby :as ruby]
            [clojure.contrib.walk :as walk]
            [clojure.contrib.singleton :as sing]))

(def scraper-handler
     (sing/per-thread-singleton
      (fn []
        (ruby/eval-jruby "require 'org/altlaw/extract/scrape/scraper_handler'")
        (ruby/eval-jruby "ScraperHandler.new"))))

(defn run-scrapers [download]
  (assert (map? download))
  (assert (contains? download :request_uri))
  (map ruby/convert-jruby
       (ruby/eval-jruby "$handler.parse(Download.from_map($download))"
                        {:download (walk/stringify-keys download)
                         :handler (scraper-handler)})))

(defn all-requests []
  (map ruby/convert-jruby
{'fraction_non_alphanumeric': 0.1419828641370869, 'fraction_numerical': 0.0, 'mean_word_length': 2.9302884615384617, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3820605', 'n_tokens_mistral': 565, 'n_tokens_neox': 529, 'n_words': 95}