<commit_before>;; Copyright © 2015, JUXT LTD.

(ns yada.core
  (:require
   [clojure.set :as set]
   [clojure.string :as str]
   [clojure.tools.logging :refer :all :exclude [trace]]
   [clojure.walk :refer (keywordize-keys)]
   [byte-streams :as bs]
   [bidi.bidi :as bidi]
   [hiccup.core :as h]
   [manifold.deferred :as d]
   ring.middleware.basic-authentication
   [ring.middleware.params :refer (assoc-query-params)]
   [ring.swagger.schema :as rs]
   ring.util.codec
   [ring.util.request :as req]
   ring.util.time
   schema.utils
   [yada.body :as body]
   [yada.charset :as charset]
   [yada.journal :as journal]
   [yada.methods :as methods]
   [yada.representation :as rep]
   [yada.protocols :as p]
   [yada.response :refer (->Response)]<commit_msg>Remove unnecessary dependency<commit_after>;; Copyright © 2015, JUXT LTD.

(ns yada.core
  (:require
   [clojure.set :as set]
   [clojure.string :as str]
   [clojure.tools.logging :refer :all :exclude [trace]]
   [clojure.walk :refer (keywordize-keys)]
   [byte-streams :as bs]
   [bidi.bidi :as bidi]
   [manifold.deferred :as d]
   ring.middleware.basic-authentication
   [ring.middleware.params :refer (assoc-query-params)]
   [ring.swagger.schema :as rs]
   ring.util.codec
   [ring.util.request :as req]
   ring.util.time
   schema.utils
   [yada.body :as body]
   [yada.charset :as charset]
   [yada.journal :as journal]
   [yada.methods :as methods]
   [yada.representation :as rep]
   [yada.protocols :as p]
   [yada.response :refer (->Response)]
{'fraction_non_alphanumeric': 0.1438943894389439, 'fraction_numerical': 0.005280528052805281, 'mean_word_length': 3.4327485380116958, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1501095', 'n_tokens_mistral': 597, 'n_tokens_neox': 580, 'n_words': 136}