<commit_before>(ns org.altlaw.jobs.web.request
  (:require [org.altlaw.util.crawler :as crawler]
            [org.altlaw.db.download-log :as dl]
            [org.altlaw.util.log :as log]
            [org.altlaw.util.context :as context]
            [org.altlaw.util.files :as files]
            [org.altlaw.util.hadoop :as h]))

(h/setup-mapreduce)

(defn all-links [document]
  (mapcat val (:links document)))

(defn check-downloaded [url]
  (if (dl/downloaded? url)
    (do (h/counter "Link URLs" "Already downloaded")
        nil)
    (do (h/counter "Link URLs" "Requested")
        [{:request_uri url} nil])))

(defn my-map [record-type record]
  (when (= :document record-type)
    (filter identity
            (map check-downloaded (all-links record)))))

(defn mapper-configure [this jobconf]
  (context/use-hadoop-jobconf jobconf))

(def mapper-map (partial h/standard-map my-map))

(defn tool-run [this args]
  (let [job (h/default-jobconf this)
        inpath (Path. (h/job-path :web :scrape))
        outpath (Path. (h/job-path :web :request))
        hdfs (FileSystem/get job)]
    (.delete hdfs outpath true)
    (FileInputFormat/setInputPaths job (str inpath))
    (FileOutputFormat/setOutputPath job outpath)
    (FileOutputFormat/setCompressOutput job false)
    (.setMapperClass job org.altlaw.jobs.web.request_mapper)
    ;; (.setReducerClass job org.altlaw.jobs.web.request_reducer)
    (.setNumReduceTasks job 0)
    (.setOutputFormat job TextOutputFormat)
    (.setJobName job "web.request")
    (JobClient/runJob job))
  0)<commit_msg>jobs/web/request.clj: use reduce step to merge duplicate links<commit_after>(ns org.altlaw.jobs.web.request
  (:require [org.altlaw.util.crawler :as crawler]
            [org.altlaw.db.download-log :as dl]
            [org.altlaw.util.log :as log]
            [org.altlaw.util.context :as context]
            [org.altlaw.util.files :as files]
            [org.altlaw.util.hadoop :as h]))

(h/setup-mapreduce)

(defn all-links [document]
  (mapcat val (:links document)))

(defn check-downloaded [url]
  (if (dl/downloaded? url)
    (do (h/counter "Link URLs" "Already downloaded")
        nil)
    (do (h/counter "Link URLs" "Requested")
        [{:request_uri url} nil])))

(defn my-map [record-type record]
  (when (= :document record-type)
    (filter identity
            (map check-downloaded (all-links record)))))

(defn my-reduce [request nils]
  (h/counter "Link URLs" "Requested after removing duplicates")
  [[request nil]])

(defn mapper-configure [this jobconf]
  (context/use-hadoop-jobconf jobconf))

(def mapper-map (partial h/standard-map my-map))

(def reducer-reduce (partial h/standard-reduce my-reduce))

(defn tool-run [this args]
  (let [job (h/default-jobconf this)
        inpath (Path. (h/job-path :web :scrape))
        outpath (Path. (h/job-path :web :request))
        hdfs (FileSystem/get job)]
    (.delete hdfs outpath true)
    (FileInputFormat/setInputPaths job (str inpath))
    (FileOutputFormat/setOutputPath job outpath)
    (FileOutputFormat/setCompressOutput job false)
    (.setMapperClass job org.altlaw.jobs.web.request_mapper)
    (.setReducerClass job org.altlaw.jobs.web.request_reducer)
    (.setInputFormat job KeyValueTextInputFormat)
    (.setOutputFormat job TextOutputFormat)
    (.setJobName job "web.request")
    (JobClient/runJob job))
  0)
{'fraction_non_alphanumeric': 0.13595706618962433, 'fraction_numerical': 0.0008944543828264759, 'mean_word_length': 3.1419753086419755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4920804', 'n_tokens_mistral': 1182, 'n_tokens_neox': 1137, 'n_words': 278}