<commit_before>(ns birdwatch.twitter-client
  (:gen-class)
  (:use
   [twitter.callbacks]
   [twitter.callbacks.handlers]
   [twitter.api.streaming])
  (:require
   [clojure.edn :as edn]
   [clojure.string :as str]
   [clojure.data.json :as json]
   [clj-time.core :as t]
   [clojure.pprint :as pp]
   [clojure.tools.logging :as log]
   [http.async.client :as ac]
   [twitter.oauth :as oauth]
   [twitter-streaming-client.core :as client]
   [clojure.core.async :as async :refer [<! <!! >! >!! chan put! alts! timeout go go-loop close!]]
   [com.stuartsierra.component :as component])
  (:import
   (twitter.callbacks.protocols AsyncStreamingCallback)))

(defn- creds [config] (oauth/make-oauth-creds (:consumer-key config) (:consumer-secret config)
                                              (:user-access-token config) (:user-access-token-secret config)))

;; channels
(def ^:private chunk-chan (chan))
(def ^:private msg-chan (chan))

;; atoms for keeping track of counts, incomplete chunk and last received timestamp
(def ^:private last-received (atom (t/epoch)))
(def ^:private chunk-buff (atom ""))
(def ^:private counter (atom 0))

(defn- parse [jstr tweets-chan]
  (try
    (let [c @counter
          json (json/read-json jstr)]
      (when (== (mod c 1000) 0) (log/info "processed" c "since startup"))
      (if (:text json)
        (>!! tweets-chan json)<commit_msg>removing unnecessary dependency<commit_after>(ns birdwatch.twitter-client
  (:gen-class)
  (:use
   [twitter.callbacks]
   [twitter.callbacks.handlers]
   [twitter.api.streaming])
  (:require
   [clojure.edn :as edn]
   [clojure.string :as str]
   [clojure.data.json :as json]
   [clj-time.core :as t]
   [clojure.pprint :as pp]
   [clojure.tools.logging :as log]
   [http.async.client :as ac]
   [twitter.oauth :as oauth]
   [clojure.core.async :as async :refer [<! <!! >! >!! chan put! alts! timeout go go-loop close!]]
   [com.stuartsierra.component :as component])
  (:import
   (twitter.callbacks.protocols AsyncStreamingCallback)))

(defn- creds [config] (oauth/make-oauth-creds (:consumer-key config) (:consumer-secret config)
                                              (:user-access-token config) (:user-access-token-secret config)))

;; channels
(def ^:private chunk-chan (chan))
(def ^:private msg-chan (chan))

;; atoms for keeping track of counts, incomplete chunk and last received timestamp
(def ^:private last-received (atom (t/epoch)))
(def ^:private chunk-buff (atom ""))
(def ^:private counter (atom 0))

(defn- parse [jstr tweets-chan]
  (try
    (let [c @counter
          json (json/read-json jstr)]
      (when (== (mod c 1000) 0) (log/info "processed" c "since startup"))
      (if (:text json)
        (>!! tweets-chan json)
{'fraction_non_alphanumeric': 0.15604395604395604, 'fraction_numerical': 0.004395604395604396, 'mean_word_length': 3.2538940809968846, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6267929', 'n_tokens_mistral': 973, 'n_tokens_neox': 927, 'n_words': 248}