<commit_before>(ns clj-simple-chart.ncs.production-cumulative-yearly-fields
  (:require [clj-simple-chart.csv.csvmap :as csv]
            [clj-http.client :as client]
            [clojure.test :as test]
            [clojure.string :as str]))

(def url "http://factpages.npd.no/ReportServer?/FactPages/TableView/field_production_yearly&rs:Command=Render&rc:Toolbar=false&rc:Parameters=f&rs:Format=CSV&Top100=false&IpAddress=172.16.17.32&CultureCode=en")

(defonce raw (-> url
                 (client/get)
                 (:body)
                 (csv/csv-map)))

(def columns (:columns raw))
(def data (:data raw))

(test/is (= [:prfInformationCarrier
             :prfYear
             :prfPrdOilNetMillSm3
             :prfPrdGasNetBillSm3
             :prfPrdNGLNetMillSm3
             :prfPrdCondensateNetMillSm3
             :prfPrdOeNetMillSm3
             :prfPrdProducedWaterInFieldMillSm3
             :prfNpdidInformationCarrier]
            columns))

(def bad-data (->> data
                   (filter #(str/starts-with? (:prfPrdNGLNetMillSm3 %) "("))))
(test/is (= 1 (count bad-data)))

(defn parens-to-negative [v]
  (if (str/starts-with? v "(")
    (str "-" (str/replace v #"\(|\)" ""))
    v))

(def parsed (->> data
                 (map #(update % :prfPrdNGLNetMillSm3 parens-to-negative))
                 (map #(update % :prfPrdOilNetMillSm3 parens-to-negative))
                 (csv/read-number-or-throw-columns [:prfYear
                                                    :prfPrdOilNetMillSm3
                                                    :prfPrdGasNetBillSm3
                                                    :prfPrdNGLNetMillSm3
                                                    :prfPrdCondensateNetMillSm3
                                                    :prfPrdOeNetMillSm3])
                 (map #(assoc % :prfPrdLiquidsNetMillSm3
                                (+ (:prfPrdOilNetMillSm3 %)
                                   (:prfPrdNGLNetMillSm3 %)
                                   (:prfPrdCondensateNetMillSm3 %))))))

(def start-year (->> parsed
                     (map :prfYear)
                     (apply min)))

(def stop-year (->> parsed
                    (map :prfYear)
                    (apply max)))

(def field-names (map :prfInformationCarrier parsed))

(defn cumulative-production
  [fields year kind]
  {:pre [(every? #(some #{%} field-names) fields)
         (pos? year)
         (some #{kind} [:prfPrdLiquidsNetMillSm3 :prfPrdGasNetBillSm3 :liquids :gas])]}
  (cond (= kind :liquids)
        (recur fields year :prfPrdLiquidsNetMillSm3)

        (= kind :gas)
        (recur fields year :prfPrdGasNetBillSm3)

        :else (->> parsed
                   (filter #(<= (:prfYear %) year))
                   (filter #(some #{(:prfInformationCarrier %)} fields))
                   (map kind)
                   (reduce + 0)
                   (double))))

(test/is (< (cumulative-production ["STATFJORD"] 1990 :prfPrdLiquidsNetMillSm3)
            (cumulative-production ["STATFJORD"] 2000 :prfPrdLiquidsNetMillSm3)))<commit_msg>Verify data<commit_after>(ns clj-simple-chart.ncs.production-cumulative-yearly-fields
  (:require [clj-simple-chart.csv.csvmap :as csv]
            [clj-http.client :as client]
            [clojure.test :as test]
            [clojure.string :as str]))

(def url "http://factpages.npd.no/ReportServer?/FactPages/TableView/field_production_yearly&rs:Command=Render&rc:Toolbar=false&rc:Parameters=f&rs:Format=CSV&Top100=false&IpAddress=172.16.17.32&CultureCode=en")

(defonce raw (-> url
                 (client/get)
                 (:body)
                 (csv/csv-map)))

(def columns (:columns raw))
(def data (:data raw))

(test/is (= [:prfInformationCarrier
             :prfYear
             :prfPrdOilNetMillSm3
             :prfPrdGasNetBillSm3
             :prfPrdNGLNetMillSm3
             :prfPrdCondensateNetMillSm3
             :prfPrdOeNetMillSm3
             :prfPrdProducedWaterInFieldMillSm3
             :prfNpdidInformationCarrier]
            columns))

(def bad-data (->> data
                   (filter #(str/starts-with? (:prfPrdNGLNetMillSm3 %) "("))))
(test/is (= 1 (count bad-data)))

(defn parens-to-negative [v]
  (if (str/starts-with? v "(")
    (str "-" (str/replace v #"\(|\)" ""))
    v))

(def parsed (->> data
                 (map #(update % :prfPrdNGLNetMillSm3 parens-to-negative))
                 (map #(update % :prfPrdOilNetMillSm3 parens-to-negative))
                 (csv/read-number-or-throw-columns [:prfYear
                                                    :prfPrdOilNetMillSm3
                                                    :prfPrdGasNetBillSm3
                                                    :prfPrdNGLNetMillSm3
                                                    :prfPrdCondensateNetMillSm3
                                                    :prfPrdOeNetMillSm3])
                 (map #(assoc % :prfPrdLiquidsNetMillSm3
                                (+ (:prfPrdOilNetMillSm3 %)
                                   (:prfPrdNGLNetMillSm3 %)
                                   (:prfPrdCondensateNetMillSm3 %))))))

(def start-year (->> parsed
                     (map :prfYear)
                     (apply min)))

(def stop-year (->> parsed
                    (map :prfYear)
                    (apply max)))

(def field-names (->> (map :prfInformationCarrier parsed)
                      (distinct)
                      (sort)
                      (vec)))

(defn cumulative-production
  [fields year kind]
  {:pre [(every? #(some #{%} field-names) fields)
         (pos? year)
         (some #{kind} [:prfPrdLiquidsNetMillSm3 :prfPrdGasNetBillSm3 :liquids :gas])]}
  (cond (= kind :liquids)
        (recur fields year :prfPrdLiquidsNetMillSm3)

        (= kind :gas)
        (recur fields year :prfPrdGasNetBillSm3)

        :else (->> parsed
                   (filter #(<= (:prfYear %) year))
                   (filter #(some #{(:prfInformationCarrier %)} fields))
                   (map kind)
                   (reduce + 0)
                   (double))))

(test/is (= (Math/round 4650.93)
            ; This value 4650.93 from
            ; Sokkelåret 2016 Excel Sheet Figur 2-1
            ; Summing column I (without forecast)
            (Math/round (cumulative-production field-names 2016 :liquids))))

(test/is (= (Math/round 2193.00)
            ; This value 2193.00 (2188.14) from
            ; Sokkelåret 2016 Excel Sheet Figur 2-1
            ; Summing column G (without forecast)
            (Math/round (cumulative-production field-names 2016 :gas))))

(test/is (< (cumulative-production ["STATFJORD"] 1990 :prfPrdLiquidsNetMillSm3)
            (cumulative-production ["STATFJORD"] 2000 :prfPrdLiquidsNetMillSm3)))
{'fraction_non_alphanumeric': 0.12180052956751986, 'fraction_numerical': 0.020888496616651958, 'mean_word_length': 1.6180207932229496, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5043774', 'n_tokens_mistral': 2267, 'n_tokens_neox': 2098, 'n_words': 370}