<commit_before>(ns forma.source.hdf-test
  (:use [forma.source.hdf] :reload)
  (:use cascalog.api
        forma.testing
        [midje sweet cascalog])
  (:require [forma.hadoop.io :as io]
            [cascalog.ops :as c]))

(def hdf-path
  (dev-path "/testdata/MOD13A3/MOD13A3.A2000032.h03v11.005.2006271174459.hdf"))

(fact "metadata test."
  (let [meta (with-gdal-open [dataset hdf-path]
               (metadata dataset))]
    meta => map?
    meta => (contains {"ASSOCIATEDPLATFORMSHORTNAME" "Terra"})))


(fact "modis-subsets test. All keys in `modis-subsets` should be found
inside the HDF file located at `hdf-path`."
  (map subdataset-key
       (subdataset-names hdf-path)) => (just (keys modis-subsets) :in-any-order))

(facts "filter checker."
  (let [test-keys (fn [keys]
                    (filter (dataset-filter keys)
                            (subdataset-names hdf-path)))]
    "If the user supplies a sequence of appropriate keys, the function
will return a sequence of subdataset names corresponding to the
supplied keys (in order)."
    (test-keys #{:ndvi :evi}) => #(= (count %) 2)
    (test-keys [:ndvi :evi])  => #(= (count %) 2)

    "If a key is missing, the generated function throws an assertion
error."
    (test-keys #{:ndv}) => (throws AssertionError)))

(tabular
 (let [src (io/hfs-wholefile hdf-path)
       [datasets n] ((juxt identity count) ?dataset-seq)]
   (fact?<- "Test that our cascalog queries are returning
             the proper subdatasets."
            ?result [?count]
            (src ?filename ?hdf)
            (unpack-modis [datasets] ?hdf :> ?dataset ?freetile)
            (c/count ?count)))
 ?dataset-seq      ?result
 [:ndvi :evi]      [[2]]
 [:evi :reli :mir] [[3]])

(tabular
 (fact "Test ensuring that raster-chunks can be serialized, and that
 they produce the proper number of chunks for the supplied
 chunk-size. (By pushing raster-chunks down into a subquery, we force
 serialization of tuples before they're utilized inside of the final
 `fact?<-` query.)"<commit_msg>Made test key less ambiguous.<commit_after>(ns forma.source.hdf-test
  (:use [forma.source.hdf] :reload)
  (:use cascalog.api
        forma.testing
        [midje sweet cascalog])
  (:require [forma.hadoop.io :as io]
            [cascalog.ops :as c]))

(def hdf-path
  (dev-path "/testdata/MOD13A3/MOD13A3.A2000032.h03v11.005.2006271174459.hdf"))

(fact "metadata test."
  (let [meta (with-gdal-open [dataset hdf-path]
               (metadata dataset))]
    meta => map?
    meta => (contains {"ASSOCIATEDPLATFORMSHORTNAME" "Terra"})))


(fact "modis-subsets test. All keys in `modis-subsets` should be found
inside the HDF file located at `hdf-path`."
  (map subdataset-key
       (subdataset-names hdf-path)) => (just (keys modis-subsets) :in-any-order))

(facts "filter checker."
  (let [test-keys (fn [keys]
                    (filter (dataset-filter keys)
                            (subdataset-names hdf-path)))]
    "If the user supplies a sequence of appropriate keys, the function
will return a sequence of subdataset names corresponding to the
supplied keys (in order)."
    (test-keys #{:ndvi :evi}) => #(= (count %) 2)
    (test-keys [:ndvi :evi])  => #(= (count %) 2)

    "If a key is missing, the generated function throws an assertion
error."
    (test-keys #{:missingkey}) => (throws AssertionError)))

(tabular
 (let [src (io/hfs-wholefile hdf-path)
       [datasets n] ((juxt identity count) ?dataset-seq)]
   (fact?<- "Test that our cascalog queries are returning
             the proper subdatasets."
            ?result [?count]
            (src ?filename ?hdf)
            (unpack-modis [datasets] ?hdf :> ?dataset ?freetile)
            (c/count ?count)))
 ?dataset-seq      ?result
 [:ndvi :evi]      [[2]]
 [:evi :reli :mir] [[3]])

(tabular
 (fact "Test ensuring that raster-chunks can be serialized, and that
 they produce the proper number of chunks for the supplied
 chunk-size. (By pushing raster-chunks down into a subquery, we force
 serialization of tuples before they're utilized inside of the final
 `fact?<-` query.)"
{'fraction_non_alphanumeric': 0.13566365191884625, 'fraction_numerical': 0.018088486922512834, 'mean_word_length': 2.904580152671756, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5899330', 'n_tokens_mistral': 1413, 'n_tokens_neox': 1344, 'n_words': 427}