<commit_before>(ns forma.hdf
  (:use (forma hadoop [conversion :only (to-period)])
        (cascalog api [io :only (temp-dir)])
        (clojure.contrib [seq-utils :only (find-first indexed)]))
  (:require (clojure.contrib [string :as s]
                             [io :as io]))
  (:import [java.util Hashtable]
           [java.io File]
           [org.gdal.gdal gdal Dataset Band]
           [org.gdal.gdalconst gdalconstConstants]
           [org.apache.hadoop.io BytesWritable]))

(set! *warn-on-reflection* true)

;; ## Constants

(def
  ^{:doc "Map between MODIS dataset identifiers (arbitrarily chosen
by the REDD team) and corresponding unique substrings, culled from
values in the SUBDATASETS hashtable of MOD13A3 HDF file. No simple,
descriptive tag exists within the metadata, as of 2/05/2011, so this
ends up being the best solution."}
  modis-subsets
  {:evi "monthly EVI"
   :qual "VI Quality"
   :red "red reflectance"
   :nir "NIR reflectance"
   :azi "azimuth"
   :blue "blue reflectance"
   :reli "monthly pixel"
   :viewz "view zenith"
   :ndvi "NDVI"
   :sunz "sun zenith"})

;; ## MODIS Unpacking

(defn metadata
  "Returns the metadata hashtable for the supplied MODIS Dataset."
  ([^Dataset modis key]
     (.GetMetadata_Dict modis key))
  ([modis]
     (metadata modis "")))

;; TODO -- describe why we only take the names business. It's because we're
;; skipping the descriptions.
;; The MODIS SUBDATASET dictionary holds entries of the form:
;; Key: SUBDATASET_2_NAME
;; Val: HDF4_EOS:EOS_GRID:"/path/to/modis.hdf":MOD_Grid_monthly_1km_VI:1 km
;; monthly EVI
;; Key: SUBDATASET_4_DESC

(defn subdataset-names
  "Returns the NAME entries of the SUBDATASETS metadata Hashtable for
  a given filepath."
  [hdf-file]
  (let [path (str hdf-file)
        dataset (gdal/Open path)]
    (try
      (vals (filter #(s/substring? "_NAME" (key %))
                    (metadata dataset "SUBDATASETS")))
      (finally (.delete dataset)))))

;; TODO --update docs, since we now don't unpack the entry.

(defn subdataset-key
  "Takes a long-form path to a MODIS subdataset, and checks to see if
  any of the values of the modis-subsets map can be found as
  substrings. If we find one, we return the associated key, cast to a
  String."
  [path]
  (s/as-str (find-first #(s/substring? (% modis-subsets) path)
                        (keys modis-subsets))))

;;  TODO-- check docs, since we now don't check against _NAME
(defn dataset-filter
  "Generates a predicate function that checks a name from the
   SUBDATASETS metadata dictionary against a supplied set of
   acceptable datasets."
  [good-keys]
  (let [substrings (map modis-subsets good-keys)]
    (fn [name]
      (some #(s/substring? % name) substrings))))

(defn make-subdataset
  "Accepts a filepath the SUBDATASETS dictionary of a MODIS Dataset,<commit_msg>Documentation updates inside of HDF.<commit_after>(ns forma.hdf
  (:use (forma hadoop [conversion :only (to-period)])
        (cascalog api [io :only (temp-dir)])
        (clojure.contrib [seq-utils :only (find-first indexed)]))
  (:require (clojure.contrib [string :as s]
                             [io :as io]))
  (:import [java.util Hashtable]
           [java.io File]
           [org.gdal.gdal gdal Dataset Band]
           [org.gdal.gdalconst gdalconstConstants]
           [org.apache.hadoop.io BytesWritable]))

(set! *warn-on-reflection* true)

;; ## Constants

(def
  ^{:doc "Map between MODIS dataset identifiers (arbitrarily chosen by
  the REDD team) and corresponding unique substrings, culled from
  values in the \"NAMES\" entries of the SUBDATASETS hashtable of
  MOD13A3 HDF file. No simple, descriptive tag exists within the
  metadata, as of 2/05/2011, so this ends up being the best
  solution."}
  modis-subsets {:evi "monthly EVI"
   :qual "VI Quality"
   :red "red reflectance"
   :nir "NIR reflectance"
   :azi "azimuth"
   :blue "blue reflectance"
   :reli "monthly pixel"
   :viewz "view zenith"
   :ndvi "NDVI"
   :sunz "sun zenith"})

;; ## MODIS Unpacking

(defn metadata
  "Returns the metadata hashtable for the supplied MODIS Dataset."
  ([^Dataset modis key]
     (.GetMetadata_Dict modis key))
  ([modis]
     (metadata modis "")))


;; Every MODIS file (archived in HDF4) packages a number of different
;; datasets; 9, in the case of the MOD13A3 product. The paths to these
;; are contained within the "SUBDATASETS" metadata dictionary of the
;; wrapping archive. This dictionary contains 18 groups of K-V pairs,
;; or 2 per dataset:
;;
;; Key: SUBDATASET_2_NAME
;; Val: HDF4_EOS:EOS_GRID:"/path/to/modis.hdf":MOD_Grid_monthly_1km_VI:1 km
;; monthly EVI
;; Key: SUBDATASET_4_DESC
;; Val: [1200x1200] 1 km monthly EVI MOD_Grid_monthly_1km_VI (16-bit
;; integer)
;;
;; We're only interested in the names, as these alone allow us to
;; uniquely identify each dataset.

(defn subdataset-names
  "Returns the NAME entries of the SUBDATASETS metadata Hashtable for
  the dataset at a given filepath."
  [hdf-file]
  (let [path (str hdf-file)
        dataset (gdal/Open path)]
    (try
      (vals (filter #(s/substring? "_NAME" (key %))
                    (metadata dataset "SUBDATASETS")))
      (finally (.delete dataset)))))

;; TODO --update docs, since we now don't unpack the entry.

(defn subdataset-key
  "Takes a long-form path to a MODIS subdataset, and checks to see if
  any of the values of the modis-subsets map can be found as
  substrings. If we find one, we return the associated key, cast to a
  string."
  [path]
  (s/as-str (find-first #(s/substring? (% modis-subsets) path)
                        (keys modis-subsets))))

;;  TODO-- check docs, since we now don't check against _NAME
(defn dataset-filter
  "Generates a predicate function that checks a name from the
   SUBDATASETS metadata dictionary against a supplied set of
   acceptable datasets."
  [good-keys]
  (let [substrings (map modis-subsets good-keys)]
    (fn [name]
      (some #(s/substring? % name) substrings))))

(defn make-subdataset
  "Accepts a filepath the SUBDATASETS dictionary of a MODIS Dataset,
{'fraction_non_alphanumeric': 0.10885700148441366, 'fraction_numerical': 0.008246742536698005, 'mean_word_length': 3.258426966292135, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1740064', 'n_tokens_mistral': 2057, 'n_tokens_neox': 1991, 'n_words': 706}