<commit_before>;; This namespace is a hub for all others, and not really complete in
;; its own right. We define various forma constants, and hold some of
;; the test queries in here for now. Some of these bad boys need to
;; get moved over to tests.

(ns forma.core
  (:use cascalog.api
        (forma [hadoop :only (all-files)]))
  (:require (cascalog [vars :as v]
                      [ops :as c])
            (forma [hdf :as h]
                   [rain :as r])))

;; ### FORMA Constants
;;
;; Aside from our EC2 information, this is the only data we need to
;; supply for the first phase of operations. Everything else should be
;; able to remain local to individual modules.

(def forma-subsets #{:ndvi :evi :qual :reli})

;; Arbitrary number of pixels slurped at a time off of a MODIS raster
;; band. For 1km data, each MODIS tile is 1200x1200 pixels; dealing
;; with each pixel individually would incur unacceptable IO costs
;; within hadoop. We currently fix the chunk size at 24,000, resulting
;; in 60 chunks per 1km data. Sharper resolution -> more chunks!

(def chunk-size 24000)

;; ### Demonstration Queries

(defn chunk-test
  "Cascalog query that takes a directory containing MODIS HDF files, or
  a link directly to such a file, totals up the # of chunks per file,
  and displays the count alongside some other nice metadata. This
  method is a proof of concept for the hadoop chunking system; The
  fact that it works shows that chunking is occurring, and the
  individual chunks are being serialized over multiple hadoop jobs."
  [dir]
  (let [source (all-files dir)
        chunks (h/modis-chunks source forma-subsets chunk-size)]
    (?<- (stdout)
         [?dataset ?res ?tile-h ?tile-v ?period ?count]
         (chunks ?dataset ?res ?tile-h ?tile-v ?period ?chunkid ?chunk)
         (c/count ?count))))

(defn rain-test
  "Like chunk-test, but works for NOAA PRECL data files."
  [m-res ll-res c-size tile-seq dir]
  (let [source (all-files dir)
        chunks (r/rain-chunks m-res ll-res c-size tile-seq source)]
    (?<- (stdout) [?dataset ?res ?period]
         (chunks ?dataset ?res ?mod-h ?mod-v ?period ?chunkid ?chunk)
         (c/count ?count))))<commit_msg>Updated rain-chunks in core, for showing actual data.<commit_after>;; This namespace is a hub for all others, and not really complete in
;; its own right. We define various forma constants, and hold some of
;; the test queries in here for now. Some of these bad boys need to
;; get moved over to tests.

(ns forma.core
  (:use cascalog.api
        (forma [hadoop :only (all-files)]))
  (:require (cascalog [vars :as v]
                      [ops :as c])
            (forma [hdf :as h]
                   [rain :as r])))

;; ### FORMA Constants
;;
;; Aside from our EC2 information, this is the only data we need to
;; supply for the first phase of operations. Everything else should be
;; able to remain local to individual modules.

(def forma-subsets #{:ndvi :evi :qual :reli})

;; Arbitrary number of pixels slurped at a time off of a MODIS raster
;; band. For 1km data, each MODIS tile is 1200x1200 pixels; dealing
;; with each pixel individually would incur unacceptable IO costs
;; within hadoop. We currently fix the chunk size at 24,000, resulting
;; in 60 chunks per 1km data. Sharper resolution -> more chunks!

(def chunk-size 24000)

;; ### Demonstration Queries

(defn chunk-test
  "Cascalog query that takes a directory containing MODIS HDF files, or
  a link directly to such a file, totals up the # of chunks per file,
  and displays the count alongside some other nice metadata. This
  method is a proof of concept for the hadoop chunking system; The
  fact that it works shows that chunking is occurring, and the
  individual chunks are being serialized over multiple hadoop jobs."
  [dir]
  (let [source (all-files dir)
        chunks (h/modis-chunks source forma-subsets chunk-size)]
    (?<- (stdout)
         [?dataset ?res ?mod-h ?mod-v ?period ?count]
         (chunks ?dataset ?res ?mod-h ?mod-v ?period ?chunkid ?chunk)
         (c/count ?count))))

(defn rain-test
  "Like chunk-test, but works for NOAA PRECL data files."
  [m-res ll-res c-size tile-seq dir]
  (let [source (all-files dir)
        chunks (r/rain-chunks m-res ll-res c-size tile-seq source)]
    (?<- (stdout) [?dataset ?res ?mod-h ?mod-v ?period ?count]
         (chunks ?dataset ?res ?mod-h ?mod-v ?period ?chunkid ?chunk)
         (c/count ?count))))
{'fraction_non_alphanumeric': 0.1009691232814965, 'fraction_numerical': 0.010367365336939374, 'mean_word_length': 3.11306765523633, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2812838', 'n_tokens_mistral': 1390, 'n_tokens_neox': 1321, 'n_words': 615}