<commit_before>(ns forma.hadoop.jobs.forma-test
  "Check that each step yields an intermediate result for the local
  data set, which runs from the end of the preprocessing through the
  final probablities.

  We are sourcing from a selection of about 2,300 pixels in Indonesia,
  of which 1,359 are hit with probabilities above 50% by April 2012.

  Only certain functions are tested on the small-sample data set,
  since even with the small selection, some of the functions take a
  few minutes to finish -- let alone check certain values. TODO: for
  these functions, come up with a few tests on an _even smaller_
  number of pixels."
  (:use forma.hadoop.jobs.forma :reload)
  (:use cascalog.api
        [midje sweet cascalog]
        [clojure.string :only (join)]
        [forma.hadoop.pail :only (to-pail split-chunk-tap)]
        [forma.hadoop.predicate :as p]
        [forma.ops.classify :only (beta-dict)])
  (:require [forma.testing :as t]
            [forma.thrift :as thrift]
            [forma.utils :as u]))

(def test-map
  "Define estimation map for testing based on 500m-16day resolution.
  This is the estimation map that generated the small-sample test
  data."
  {:est-start "2005-12-31"
   :est-end "2012-04-22"
   :s-res "500"
   :t-res "16"
   :neighbors 1
   :window-dims [600 600]
   :vcf-limit 25
   :long-block 30
   :window 10
   :ridge-const 1e-8
   :convergence-thresh 1e-6
   :max-iterations 500
   :min-coast-dist 3
   :nodata -9999.0})

(def t-res "16")
(def s-res "500")
(def loc-vec [s-res 28 8 0 0])
(def pixel-loc (apply thrift/ModisPixelLocation* loc-vec))
(def vcf-src    [(conj loc-vec 26)]) 
(def hansen-src [(conj loc-vec 100)])
(def ecoid-src  [(conj loc-vec 10101)])
(def gadm-src   [(conj loc-vec 40132)])
(def border-src [(conj loc-vec 4)])

(def sample-ts-dc-src
  (let [ts (thrift/TimeSeries* 693 [1 2 3])]
    [[(thrift/DataChunk* "ndvi" pixel-loc ts "16")]
     [(thrift/DataChunk* "ndvi" (thrift/ModisPixelLocation* "500" 28 9 0 0) ts "16")]]))

(def sample-hansen-dc
  (thrift/DataChunk* "hansen" pixel-loc 100 t-res))

(defn sample-fire-series
  [start-idx length]
  (thrift/TimeSeries* start-idx (vec (repeat length (thrift/FireValue* 0 0 0 0)))))

(def sample-hansen-src
  (let [bad-loc (thrift/ModisPixelLocation* s-res 29 8 0 0)]
    [[(thrift/DataChunk* "hansen" pixel-loc 100 t-res)]
     [(thrift/DataChunk* "hansen" bad-loc 100 t-res)]]))

(fact
  "Test `static-tap`"
  (let [pixel-loc (thrift/ModisPixelLocation* "500" 28 8 0 0)
      dc-src [["pail-path" (thrift/DataChunk* "vcf" pixel-loc 25 "00")]]]
    (static-tap dc-src)) => (produces [["500" 28 8 0 0 25]]))

(fact
  "Checks that consolidate-static correctly merges static datasets."
  (consolidate-static (:vcf-limit test-map)
                      vcf-src gadm-src hansen-src ecoid-src border-src)
  => (produces [(conj loc-vec 26 40132 10101 100 4)]))

(fact
  "Checks that `screen-by-tileset` correctly screens out pixels not in tile-set"
  (let [tile-set #{[28 8]}
        bad-h 29
        src [["fake-path" sample-hansen-dc]
             ["fake-path" (thrift/DataChunk* "hansen"
                                 (thrift/ModisPixelLocation* s-res bad-h 8 0 0)
                                 100
                                 t-res)]]]
    (screen-by-tileset src tile-set)) => (produces [["fake-path" sample-hansen-dc]]))

(fact
  "Checks that fire-tap is trimming the fire timeseries correctly.
   Note that 828 is the period corresponding to 2006-01-01.
   Also note that the timeseries is a running sum."
  (let [src [[(thrift/DataChunk* "fire" pixel-loc (sample-fire-series 825 5) "01")]]
        est-start "2005-12-31"
        est-end "2006-01-01"]
    (fire-tap est-start est-end t-res src)) => (produces [[s-res 28 8 0 0 (sample-fire-series 827 2)]]))

(fact
  "Test `adjust-precl`"
  (let [base-t-res "32"
        target-t-res "16"
        precl-src [["500" 28 8 0 0 360 [1.5 2.5 3.4 4.7]]]]
    (adjust-precl base-t-res target-t-res precl-src))
  => (produces [["500" 28 8 0 0 690 [2 2 3 3 3 4 5]]]))

(fact
  "Check that `filter-query` properly screens out the pixel with VCF < 25, and keeps the one with VCF >= 25."
  (let [vcf-limit 25
        static-src [["500" 28 8 0 0 25 0 0 0 0]
                    ["500" 28 9 0 0 10 0 0 0 0]]
        ts-src sample-ts-dc-src]
    (filter-query static-src vcf-limit ts-src)) => (produces [["500" 28 8 0 0 693 [1 2 3]]]))

(fact
  "Check `training-3000s?`"
  (training-3000s? "16" 826 "2005-12-31" [1 2 3 4]) => false
  (training-3000s? "16" 826 "2005-12-31" [-3000 2 3 4]) => false
  (training-3000s? "16" 826 "2005-12-31" [-3000 -3000 3 4]) => true
  (training-3000s? "16" 826 "2005-12-31" [-3000 -3000 -3000 4]) => true)

(fact
  "Check that `dynamic-filter` properly truncates values where
   timeseries lengths don't line up, drops pixel with all -3000s in
   the training period, and replaces a nodata value with the value to
   the left of it."
  (let [nodata (:nodata test-map)
        ndvi-src [["500" 28 8 0 0 826 [1 2 3]]
                  ["500" 28 8 0 1 826 [-3000 -3000 3]]
                  ["500" 28 8 0 2 826 [-3000 2 3]]]
        rain-src [["500" 28 8 0 0 826 [0.1 0.2]]
                  ["500" 28 8 0 1 826 [0.1 0.2]]
                  ["500" 28 8 0 2 826 [0.1 0.2]]]]
    (dynamic-filter test-map ndvi-src rain-src))
  => (produces [["500" 28 8 0 0 826 [1 2] [0.1 0.2]]
                ["500" 28 8 0 2 826 [-3000 2] [0.1 0.2]]]))

(tabular
 (fact
   "Check `series-end`"
   (let [src [[[1 2 3]]]]
     (<- [?end]
         (src ?series)
         (series-end ?series ?start-idx :> ?end))) => (produces ?res))
?start-idx ?res
 2            [[4]]
 0            [[2]]
 50           [[52]])

(fact
 "Test `calculate-trends` mechanics - math is checked elsewhere"
  (let [{:keys [window long-block]} test-map]
    (calculate-trends window long-block 693 (range 300 600) (range 800 499 -1)))
  => (list 992 0.9999999999999906 nil nil 201.353677896894))

(fact
  "Test `telescoping-trends` defmapcatop - math is tested elsewhere"
  (let [src [["500" 28 8 0 0 693 (vec (range 300))
                    (vec (map #(/ % 100.) (range 1 301)))]]]
    (<- [?s-res ?mod-h ?mod-v ?sample ?line ?start ?end-l ?short-l ?long-l ?t-stat-l ?break-l]
        (src ?s-res ?mod-h ?mod-v ?sample ?line ?start ?ndvi ?precl)<commit_msg>Tweaked calculate-trends test to reflect optimized short trend calcs; fixed formatting<commit_after>(ns forma.hadoop.jobs.forma-test
  "Check that each step yields an intermediate result for the local
  data set, which runs from the end of the preprocessing through the
  final probablities.

  We are sourcing from a selection of about 2,300 pixels in Indonesia,
  of which 1,359 are hit with probabilities above 50% by April 2012.

  Only certain functions are tested on the small-sample data set,
  since even with the small selection, some of the functions take a
  few minutes to finish -- let alone check certain values. TODO: for
  these functions, come up with a few tests on an _even smaller_
  number of pixels."
  (:use forma.hadoop.jobs.forma :reload)
  (:use cascalog.api
        [midje sweet cascalog]
        [clojure.string :only (join)]
        [forma.hadoop.pail :only (to-pail split-chunk-tap)]
        [forma.hadoop.predicate :as p]
        [forma.ops.classify :only (beta-dict)])
  (:require [forma.testing :as t]
            [forma.thrift :as thrift]
            [forma.utils :as u]))

(def test-map
  "Define estimation map for testing based on 500m-16day resolution.
  This is the estimation map that generated the small-sample test
  data."
  {:est-start "2005-12-31" ;; period index 827
   :est-end "2012-04-22" ;; period index 973
   :s-res "500"
   :t-res "16"
   :neighbors 1
   :window-dims [600 600]
   :vcf-limit 25
   :long-block 30
   :window 10
   :ridge-const 1e-8
   :convergence-thresh 1e-6
   :max-iterations 500
   :min-coast-dist 3
   :nodata -9999.0})

(def t-res "16")
(def s-res "500")
(def loc-vec [s-res 28 8 0 0])
(def pixel-loc (apply thrift/ModisPixelLocation* loc-vec))
(def vcf-src    [(conj loc-vec 26)]) 
(def hansen-src [(conj loc-vec 100)])
(def ecoid-src  [(conj loc-vec 10101)])
(def gadm-src   [(conj loc-vec 40132)])
(def border-src [(conj loc-vec 4)])

(def sample-ts-dc-src
  (let [ts (thrift/TimeSeries* 693 [1 2 3])]
    [[(thrift/DataChunk* "ndvi" pixel-loc ts "16")]
     [(thrift/DataChunk* "ndvi" (thrift/ModisPixelLocation* "500" 28 9 0 0) ts "16")]]))

(def sample-hansen-dc
  (thrift/DataChunk* "hansen" pixel-loc 100 t-res))

(defn sample-fire-series
  [start-idx length]
  (thrift/TimeSeries* start-idx (vec (repeat length (thrift/FireValue* 0 0 0 0)))))

(def sample-hansen-src
  (let [bad-loc (thrift/ModisPixelLocation* s-res 29 8 0 0)]
    [[(thrift/DataChunk* "hansen" pixel-loc 100 t-res)]
     [(thrift/DataChunk* "hansen" bad-loc 100 t-res)]]))

(fact
  "Test `static-tap`"
  (let [pixel-loc (thrift/ModisPixelLocation* "500" 28 8 0 0)
      dc-src [["pail-path" (thrift/DataChunk* "vcf" pixel-loc 25 "00")]]]
    (static-tap dc-src)) => (produces [["500" 28 8 0 0 25]]))

(fact
  "Checks that consolidate-static correctly merges static datasets."
  (consolidate-static (:vcf-limit test-map)
                      vcf-src gadm-src hansen-src ecoid-src border-src)
  => (produces [(conj loc-vec 26 40132 10101 100 4)]))

(fact
  "Checks that `screen-by-tileset` correctly screens out pixels not in tile-set"
  (let [tile-set #{[28 8]}
        bad-h 29
        src [["fake-path" sample-hansen-dc]
             ["fake-path" (thrift/DataChunk* "hansen"
                                 (thrift/ModisPixelLocation* s-res bad-h 8 0 0)
                                 100
                                 t-res)]]]
    (screen-by-tileset src tile-set)) => (produces [["fake-path" sample-hansen-dc]]))

(fact
  "Checks that fire-tap is trimming the fire timeseries correctly.
   Note that 828 is the period corresponding to 2006-01-01.
   Also note that the timeseries is a running sum."
  (let [src [[(thrift/DataChunk* "fire" pixel-loc (sample-fire-series 825 5) "01")]]
        est-start "2005-12-31"
        est-end "2006-01-01"]
    (fire-tap est-start est-end t-res src)) => (produces [[s-res 28 8 0 0 (sample-fire-series 827 2)]]))

(fact
  "Test `adjust-precl`"
  (let [base-t-res "32"
        target-t-res "16"
        precl-src [["500" 28 8 0 0 360 [1.5 2.5 3.4 4.7]]]]
    (adjust-precl base-t-res target-t-res precl-src))
  => (produces [["500" 28 8 0 0 690 [2 2 3 3 3 4 5]]]))

(fact
  "Check that `filter-query` properly screens out the pixel with VCF < 25, and keeps the one with VCF >= 25."
  (let [vcf-limit 25
        static-src [["500" 28 8 0 0 25 0 0 0 0]
                    ["500" 28 9 0 0 10 0 0 0 0]]
        ts-src sample-ts-dc-src]
    (filter-query static-src vcf-limit ts-src)) => (produces [["500" 28 8 0 0 693 [1 2 3]]]))

(fact
  "Check `training-3000s?`"
  (training-3000s? "16" 826 "2005-12-31" [1 2 3 4]) => false
  (training-3000s? "16" 826 "2005-12-31" [-3000 2 3 4]) => false
  (training-3000s? "16" 826 "2005-12-31" [-3000 -3000 3 4]) => true
  (training-3000s? "16" 826 "2005-12-31" [-3000 -3000 -3000 4]) => true)

(fact
  "Check that `dynamic-filter` properly truncates values where
   timeseries lengths don't line up, drops pixel with all -3000s in
   the training period, and replaces a nodata value with the value to
   the left of it."
  (let [nodata (:nodata test-map)
        ndvi-src [["500" 28 8 0 0 826 [1 2 3]]
                  ["500" 28 8 0 1 826 [-3000 -3000 3]]
                  ["500" 28 8 0 2 826 [-3000 2 3]]]
        rain-src [["500" 28 8 0 0 826 [0.1 0.2]]
                  ["500" 28 8 0 1 826 [0.1 0.2]]
                  ["500" 28 8 0 2 826 [0.1 0.2]]]]
    (dynamic-filter test-map ndvi-src rain-src))
  => (produces [["500" 28 8 0 0 826 [1 2] [0.1 0.2]]
                ["500" 28 8 0 2 826 [-3000 2] [0.1 0.2]]]))

(tabular
 (fact
   "Check `series-end`"
   (let [src [[[1 2 3]]]]
     (<- [?end]
         (src ?series)
         (series-end ?series ?start-idx :> ?end))) => (produces ?res))
?start-idx ?res
 2            [[4]]
 0            [[2]]
 50           [[52]])

(fact
 "Test `calculate-trends` mechanics - math is checked elsewhere."
  (let [{:keys [window long-block]} test-map]
    (calculate-trends window long-block 693 (range 300 600) (range 800 499 -1)))
  => (list 992 nil nil 201.353677896894))

(fact
  "Test `telescoping-trends` defmapcatop - math is tested elsewhere."
  (let [src [["500" 28 8 0 0 693 (vec (range 300))
                    (vec (map #(/ % 100.) (range 1 301)))]]]
    (<- [?s-res ?mod-h ?mod-v ?sample ?line ?start ?end-l ?short-l ?long-l ?t-stat-l ?break-l]
        (src ?s-res ?mod-h ?mod-v ?sample ?line ?start ?ndvi ?precl)
{'fraction_non_alphanumeric': 0.15082612116443744, 'fraction_numerical': 0.0988985051140834, 'mean_word_length': 2.5406685236768802, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5551716', 'n_tokens_mistral': 5648, 'n_tokens_neox': 4616, 'n_words': 1584}