<commit_before>import os
import sys
import json
import argparse
from auth import auth
from databasehandler import CollectionDatabase
from tweepy import OAuthHandler
from tweepy import Stream
from tweepy.streaming import StreamListener
from httplib import IncompleteRead

DATABASE_PATH = os.path.join(os.path.dirname(__file__), 'database/')

auth = auth('credentials.csv')

consumer_key = auth['consumer_key']
consumer_secret = auth['consumer_secret']

access_token = auth['access_token']
access_token_secret = auth['access_secret']


class CollectListener(StreamListener):<commit_msg>Add a StdOutListener for debugging<commit_after>import os
import sys
import json
import argparse
from auth import auth
from databasehandler import CollectionDatabase
from tweepy import OAuthHandler
from tweepy import Stream
from tweepy.streaming import StreamListener
from httplib import IncompleteRead

DATABASE_PATH = os.path.join(os.path.dirname(__file__), 'database/')

auth = auth('credentials.csv')

consumer_key = auth['consumer_key']
consumer_secret = auth['consumer_secret']

access_token = auth['access_token']
access_token_secret = auth['access_secret']


class StdOutListener(StreamListener):
    # Std Out Listener meant for debugging/testing
    def on_data(self, data):
        print data
        return True

    def on_error(self, status):
        print status


class CollectListener(StreamListener):
{'fraction_non_alphanumeric': 0.07489300998573467, 'fraction_numerical': 0.0, 'mean_word_length': 6.158163265306122, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1108829', 'n_tokens_mistral': 406, 'n_tokens_neox': 391, 'n_words': 117}