<commit_before>(ns statspop.detection
  "attempt to detect tabular data in non-table formats by finding continuous sets of rows of at least size x with at least some y proportion of numbers.  data coming in from chrome extension as documentElement.innerText

 INCOMPLETE"
  (:require [clojure.string :as str]))


;; to filter by digit proportions
(defn lines [text] (str/split text #"\n"))

(defn spaceless [line] (str/replace line #"\s" ""))
;; need to verify this works right with tabs too.  could always have the regex match tabs too, but I think \s gets all whitespace

(defn just-digits [line] (str/join (re-seq #"\d" line)))

(defn identify-numeric
  "float (0,1), string --> bool
  test line to see if at least proportion of its non-space characters are numeric"
  [proportion line]
  (let [nospace (spaceless line)
        digits (just-digits nospace)]
    (>= (/ (count digits) (count nospace)) proportion)))

(defn make-index-maps
"make vector of vectors into map with vectors and indexes"
  [vov]
  (map-indexed #(identity {:index %1 :vals %2}) vov))

(defn get-indexes-of-matches
  "filter the index-maps by testing values against identify-numeric
  this could really be optimized with transducer. too much unnecessary intermediate collection"
  [indexed-maps proportion]
  (let [matches (filter #(identify-numeric proportion (:vals %)) indexed-maps)]
  (mapv :index matches)))

(defn find-runs
  [indices minsize]
  (loop [all-regions []
         active-region []
         cur-idx (first indices)
         remaining (rest indices)]
    (do
      (println (str "INCOMING item " cur-idx))
      (cond
        (= 0 (count remaining))
        (if (>= (count active-region) minsize)
          (conj all-regions active-region)
          all-regions)
        (= active-region [])
        (do
          (println {:stage "appending to empty region" :all all-regions :active active-region :item cur-idx :rest remaining})
          (recur all-regions (conj active-region cur-idx) (first remaining) (rest remaining)))
        (= (inc (last active-region)) cur-idx)
        (do
          (println {:stage "appending to unbroken run" :all all-regions :active active-region :item cur-idx :rest remaining})
          (recur all-regions (conj active-region cur-idx) (first remaining) (rest remaining)))
        :else (if (>= (count active-region) minsize)
                (do
                  (println {:stage "run broken, appending to all-regions" :all all-regions :active active-region :item cur-idx :rest remaining})
                  (recur (conj all-regions active-region) (conj [] cur-idx) (first remaining) (rest remaining)))
                (do (println {:stage "run-broken, not appending (too small)" :all all-regions :active active-region :item cur-idx :rest remaining})
                    (recur all-regions (conj [] cur-idx) (first remaining) (rest remaining))))))))

(def test-lines "123 abc
  foo bar baa
  1234 a
  1245 ab<commit_msg>now it is correct yay!<commit_after>(ns statspop.detection
  "attempt to detect tabular data in non-table formats by finding continuous sets of rows of at least size x with at least some y proportion of numbers.  data coming in from chrome extension as documentElement.innerText

 INCOMPLETE"
  (:require [clojure.string :as str]))


;; to filter by digit proportions
(defn lines [text] (str/split text #"\n"))

(defn spaceless [line] (str/replace line #"\s" ""))
;; need to verify this works right with tabs too.  could always have the regex match tabs too, but I think \s gets all whitespace

(defn just-digits [line] (str/join (re-seq #"\d" line)))

(defn identify-numeric
  "float (0,1), string --> bool
  test line to see if at least proportion of its non-space characters are numeric"
  [proportion line]
  (let [nospace (spaceless line)
        digits (just-digits nospace)]
    (>= (/ (count digits) (count nospace)) proportion)))

(defn make-index-maps
"make vector of vectors into map with vectors and indexes"
  [vov]
  (map-indexed #(identity {:index %1 :vals %2}) vov))

(defn get-indexes-of-matches
  "filter the index-maps by testing values against identify-numeric
  this could really be optimized with transducer. too much unnecessary intermediate collection"
  [indexed-maps proportion]
  (let [matches (filter #(identify-numeric proportion (:vals %)) indexed-maps)]
  (mapv :index matches)))

(defn find-runs
  [indices minsize]
  (loop [all-regions []
         active-region []
         cur-idx (first indices)
         remaining (rest indices)]
    (do
      (println (str "INCOMING item " cur-idx))
      (cond
        (= 0 (count remaining))
        (if (= (inc (last active-region)) cur-idx)
          (if (>= (count (conj active-region cur-idx)) minsize)
            (conj all-regions (conj active-region cur-idx))
            all-regions)
          (if (>= (count active-region) minsize)
            (conj all-regions active-region)
            all-regions)
          )
        (= active-region [])
        (do
          (println {:stage "appending to empty region" :all all-regions :active active-region :item cur-idx :rest remaining})
          (recur all-regions (conj active-region cur-idx) (first remaining) (rest remaining)))
        (= (inc (last active-region)) cur-idx)
        (do
          (println {:stage "appending to unbroken run" :all all-regions :active active-region :item cur-idx :rest remaining})
          (recur all-regions (conj active-region cur-idx) (first remaining) (rest remaining)))
        :else (if (>= (count active-region) minsize)
                (do
                  (println {:stage "run broken, appending to all-regions" :all all-regions :active active-region :item cur-idx :rest remaining})
                  (recur (conj all-regions active-region) (conj [] cur-idx) (first remaining) (rest remaining)))
                (do (println {:stage "run-broken, not appending (too small)" :all all-regions :active active-region :item cur-idx :rest remaining})
                    (recur all-regions (conj [] cur-idx) (first remaining) (rest remaining))))))))

(def test-lines "123 abc
  foo bar baa
  1234 a
  1245 ab
{'fraction_non_alphanumeric': 0.11860807057670315, 'fraction_numerical': 0.0052279039372651525, 'mean_word_length': 2.972744970798183, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2632078', 'n_tokens_mistral': 1878, 'n_tokens_neox': 1767, 'n_words': 676}