<commit_before>(ns forma.trends.analysis-test
  (:use [forma.trends.analysis] :reload)
  (:use [midje sweet]
        [forma.trends.data :only (ndvi rain reli)])
  (:require [incanter.core :as i]))

(facts "Test that `long-stats` yields the trend coefficient and t-test
 statistic on `ndvi`"

  ;; Long-term statistics on NDVI alone
  (let [[coeff t-test] (long-stats ndvi)]
    coeff  => (roughly -1.14300)
    t-test => (roughly -0.91826))

  ;; Long-term statistics on NDVI, with rain as a cofactor.  Note that
  ;; the trends change slightly.
  (let [[coeff t-test] (long-stats ndvi rain)]
    coeff  => (roughly -1.23824)
    t-test => (roughly -0.99763)))

(fact "Test first-order conditions."
  (ffirst
   (first-order-conditions (i/matrix ndvi))) => (roughly -1094.50817))

(defn- shift-down-end
  "Returns a transformed collection, where the last half is shifted
  down by 50%; used to test that the hansen-stat identifies
  down-shifts in a time-series (see next test)."
  [coll]
  (let [half-count (Math/floor (/ (count ndvi) 2))
        second-half (drop half-count ndvi)]
    (concat (take half-count ndvi)
            (map #(- % (/ % 2)) second-half))))<commit_msg>added tests for new long-term stats<commit_after>(ns forma.trends.analysis-test
  (:use [forma.trends.analysis] :reload)
  (:use [midje sweet]
        [forma.trends.data :only (ndvi rain reli)])
  (:require [incanter.core :as i]))

(facts "Test that `long-stats` yields the trend coefficient and t-test
 statistic on `ndvi`"

  ;; Long-term statistics on NDVI alone
  (let [[coeff t-test] (long-stats ndvi)]
    coeff  => (roughly -1.14300)
    t-test => (roughly -0.91826))

  ;; Long-term statistics on NDVI, with rain as a cofactor.  Note that
  ;; the trends change slightly.
  (let [[coeff t-test] (long-stats ndvi rain)]
    coeff  => (roughly -1.23824)
    t-test => (roughly -0.99763))

  ;; Returns a pair of nil values in the presence of an exception
  (let [[coeff t-test] (long-stats [0])]
    coeff  => nil
    t-test => nil))

(fact "Test first-order conditions."
  (ffirst
   (first-order-conditions (i/matrix ndvi))) => (roughly -1094.50817))

(defn- shift-down-end
  "Returns a transformed collection, where the last half is shifted
  down by 50%; used to test that the hansen-stat identifies
  down-shifts in a time-series (see next test)."
  [coll]
  (let [half-count (Math/floor (/ (count ndvi) 2))
        second-half (drop half-count ndvi)]
    (concat (take half-count ndvi)
            (map #(- % (/ % 2)) second-half))))
{'fraction_non_alphanumeric': 0.1466508125247721, 'fraction_numerical': 0.029726516052318668, 'mean_word_length': 3.137704918032787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3506027', 'n_tokens_mistral': 940, 'n_tokens_neox': 864, 'n_words': 285}