<commit_before>;
; Copyright © 2013,2014 <NAME> (<EMAIL>)
;
; This work is licensed under the Creative Commons Attribution-ShareAlike 3.0
; Unported License. To view a copy of this license, visit
; http://creativecommons.org/licenses/by-sa/3.0/ or send a letter to Creative
; Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA.
;

(ns alfresco-technical-validation.source-indexer
  (:require [clojure.string        :as s]
            [clojure.tools.logging :as log]
            [clojure.java.io       :as io]
            ))

(def ^:private file-types-of-interest {
  #"module\.properties"   :module-properties
  #".*\.java"             :java
  #".*\.js"               :javascript
  #".*\.ftl"              :freemarker
  #".*\.xml"              :xml
  #".*\.desc\.xml"        :web-script-descriptors
  #".*-context\.xml"      :spring-app-context
  #"build\.xml"           :ant
  #"pom\.xml"             :maven
  #"build\.gradle"        :gradle                 
  #"project\.clj"         :leiningen
  #"build\.sbt"           :sbt
  #"[mM]akefile"          :make
  })

(defn- build-file-type-index
  [files file-regex k]
  { k (filter #(re-matches file-regex %) files) })

(defn- build-file-types-index
  [files]
  (merge (map #(build-file-type-index files % (val %)) file-types-of-interest)))

(defn index-source
  "Indexes the source code in the given location."
  [source]
  (let [files (file-seq (io/file source))
        ]
    (comment "####TODO: implement me!")
    ))<commit_msg>Added code to index source files by type.<commit_after>;
; Copyright © 2013,2014 <NAME> (<EMAIL>.com)
;
; This work is licensed under the Creative Commons Attribution-ShareAlike 3.0
; Unported License. To view a copy of this license, visit
; http://creativecommons.org/licenses/by-sa/3.0/ or send a letter to Creative
; Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA.
;

(ns alfresco-technical-validation.source-indexer
  (:require [clojure.string        :as s]
            [clojure.tools.logging :as log]
            [clojure.java.io       :as io]
            ))

(def ^:private file-types-of-interest {
  #"module\.properties"   :module-properties
  #".*\.java"             :java
  #".*\.js"               :javascript
  #".*\.ftl"              :freemarker
  #".*\.xml"              :xml
  #".*\.desc\.xml"        :web-script-descriptors
  #".*-context\.xml"      :spring-app-context
  #"build\.xml"           :ant
  #"pom\.xml"             :maven
  #"build\.gradle"        :gradle                 
  #"project\.clj"         :leiningen
  #"build\.sbt"           :sbt
  #"[mM]akefile"          :make
  })

(defn- build-file-type-index
  [files file-regex k]
  { k (filter #(re-matches file-regex (.getName %)) files) })

(defn- build-file-types-index
  [files]
  (into {} (map #(build-file-type-index files (key %) (val %)) file-types-of-interest)))

(defn index-source
  "Indexes the source code in the given location."
  [source]
  (let [files      (file-seq (io/file source))
        file-index (build-file-types-index files)]
    (comment "####TODO: implement me!")
    ))
{'fraction_non_alphanumeric': 0.16019261637239166, 'fraction_numerical': 0.014767255216693418, 'mean_word_length': 2.1828396322778345, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2270072', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1042, 'n_words': 260}