<commit_before>":";exec java -cp "$HOME/.m2/repository/org/clojure/clojure/1.5.1/clojure-1.5.1.jar" clojure.main $0 $*

(require '[clojure.string :as s])

(doseq [c (take 25 
            (sort-by val > 
              (frequencies
                (remove 
                  #(some (partial = %) (s/split (slurp "../stop_words.txt") #","))
                  (re-seq #"[a-z]{2,}" (s/lower-case (slurp (first *command-line-args*))))))))]
  (printf "%s - %d\n" (first c) (nth c 1)))
<commit_msg>Adds Java interop and changes remove to checking set<commit_after>":";exec java -cp "$HOME/.m2/repository/org/clojure/clojure/1.5.1/clojure-1.5.1.jar" clojure.main $0 $*

(doseq [c (take 25 
            (sort-by val > 
              (frequencies
                (remove 
                  #(contains? (set (.split (slurp "../stop_words.txt") ",")) %)
                  (re-seq #"[a-z]{2,}" (.toLowerCase (slurp (first *command-line-args*))))))))]
  (printf "%s - %d\n" (first c) (nth c 1)))

{'fraction_non_alphanumeric': 0.21100917431192662, 'fraction_numerical': 0.024464831804281346, 'mean_word_length': 2.0783699059561127, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '198447', 'n_tokens_mistral': 392, 'n_tokens_neox': 367, 'n_words': 78}