<commit_before>; Pulling together code from clojure core, clojure contrib, snippets on the web,
; and some of my own work to create an easy-to-use and concise html scraping
; api.

(ns clj-scrape.core
  (:use clj-scrape.utils))

(require '[clojure.xml :as xml]
         '[clojure.contrib.zip-filter :as zf]
         '[clojure.contrib.zip-filter.xml :as zfxml]
         '[clojure.zip :as zip])


(defn- startparse-tagsoup [s ch]
  "startparse fn required by xml/parse constructor for generating a tagsoup
  parse stream."
  (doto (org.ccil.cowan.tagsoup.Parser.)
      (.setContentHandler ch)
      (.parse s)))

(defn dom [stream]
  "Returns the parsed and zipped dom data strucutre that can be given as the
  first argument to xml->."
  (zip/xml-zip
    (xml/parse (org.xml.sax.InputSource. stream) startparse-tagsoup)))

(def xml->      zfxml/xml->)
(def xml1->     zfxml/xml1->)
(def desc       zf/descendants)
(def leftmost?  zf/leftmost?)
(def rightmost? zf/rightmost?)
(def ansc       zf/ancestors)
(def children   zf/children)
(def attr       zfxml/attr)
(def attr=      zfxml/attr=)
(def text       zfxml/text)

(defn class= [target-name]
  "Returns a query predicate that matches a node when has a class named
  target-name."
  (let [name-re (re-pattern (str "\\b" target-name "\\b"))]
   (fn [loc] (if-let [class-str (attr loc :class)]<commit_msg>few tweaks to scrape<commit_after>; Pulling together code from clojure core, clojure contrib, snippets on the web,
; and some of my own work to create an easy-to-use and concise html scraping
; api.

(ns clj-scrape.core
  (:use clj-scrape.utils))

(require '[clojure.xml :as xml]
         '[clojure.contrib.zip-filter :as zf]
         '[clojure.contrib.zip-filter.xml :as zfxml]
         '[clojure.zip :as zip])

; http://paste.lisp.org/display/70171
(defn- startparse-tagsoup [s ch]
  "startparse fn required by xml/parse constructor for generating a tagsoup
  parse stream."
  (doto (org.ccil.cowan.tagsoup.Parser.)
      (.setContentHandler ch)
      (.parse s)))

(defn dom [stream]
  "Returns the parsed and zipped dom data strucutre that can be given as the
  first argument to xml->."
  (zip/xml-zip
    (xml/parse (org.xml.sax.InputSource. stream) startparse-tagsoup)))

(def xml->       zfxml/xml->)
(def xml1->      zfxml/xml1->)
(def rightmost?  zf/leftmost?)
(def rightmost?  zf/rightmost?)
(def desc        zf/descendants)
(def ansc        zf/ancestors)
(def children    zf/children)
(def attr        zfxml/attr)
(def attr=       zfxml/attr=)
(def text        zfxml/text)

(defn class= [target-name]
  "Returns a query predicate that matches a node when has a class named
  target-name."
  (let [name-re (re-pattern (str "\\b" target-name "\\b"))]
   (fn [loc] (if-let [class-str (attr loc :class)]
{'fraction_non_alphanumeric': 0.1315028901734104, 'fraction_numerical': 0.0032514450867052024, 'mean_word_length': 3.0482456140350878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 36}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4246622', 'n_tokens_mistral': 1016, 'n_tokens_neox': 990, 'n_words': 311}