<commit_before>#!/usr/bin/env python
from twitter import Twitter, OAuth, TwitterHTTPError, TwitterStream
import sys
import os
import logging
import requests
from time import sleep

try:
    import json
except ImportError:
    import simplejson as json


class TwitterConsumer:
    def __init__(self):
        self.logger = logging.getLogger()
        handler = logging.StreamHandler()
        formatter = logging.Formatter(
            '%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
        handler.setFormatter(formatter)
        self.logger.addHandler(handler)

        loglevel = os.getenv('LOG_LEVEL', 'info')

        self.logger.setLevel(loglevel)

        needConfig = True
        while needConfig:
            #self.topics = 'docker,devops,#F1,coreos,#AWS,@Docker,@awscloud,@LewisHamilton'
            self.topics = os.getenv('TOPCIS', False)
            self.logger.debug("TOPICS: %s" % self.topics)
            self.twitter_access_token = os.getenv('TWITTER_ACCESS_TOKEN', False)
            self.logger.debug("twitter_access_token: %s" % self.twitter_access_token)
            self.twitter_access_secret = os.getenv('TWITTER_ACCESS_SECRET', False)
            self.logger.debug("twitter_access_secret: %s" % self.twitter_access_secret)
            self.twitter_consumer_key = os.getenv('TWITTER_CONSUMER_KEY', False)
            self.logger.debug("twitter_consumer_key: %s" % self.twitter_consumer_key)
            self.twitter_consumer_secret = os.getenv('TWITTER_CONSUMER_SECRET', False)
            self.logger.debug("twitter_consumer_secret: %s" % self.twitter_consumer_secret)
            self.giles_endpoint = os.getenv("GILES_ENDPOINT", 'http://giles/v1/')
            self.logger.debug("giles_endpoint: %s" % self.giles_endpoint)
            err = self._check_health()
            if len(err) > 0:
                for error in err:
                    self.logger.critical("Error: %s" % error)
                sleep(15)
            else:
                needConfig = False
                self.logger.info("Configuration Checks out, Starting up....")


    def _check_health(self):
        """
        Application health check
        Report Missing Settings to logger
        """
        err = []
        if self.topics:
            err.append("Missing Topics to Monitor")
        if self.twitter_access_token:
            err.append("Missing twitter_access_token")
        if self.twitter_access_secret:
            err.append("Missing twitter_access_secret")
        if self.twitter_consumer_key:
            err.append("Missing twitter_consumer_key")
        if self.twitter_consumer_secret:
            err.append("Missing twitter_consumer_secret")
        if self.giles_endpoint:
            err.append("Missing Giles API Endpoint")
        return err


    def _save_mongo(self, tweet):
        """
        Save Tweet to MongoDB via Giles API
        """
        headers = {'user-agent': 'inkslinger/0.0.1'}
        self.logger.debug("Saving Tweet....")
        self.logger.debug(tweet)
        url = "%s/%s" % (self.giles_endpoint, 'posts')
        self.logger.debug("Sending data to %s [PUT]" % url)
        r = requests.post(url, json=tweet, headers=headers)
        if r.status_code == 200:<commit_msg>removing one of the checks for the giles endpoint<commit_after>#!/usr/bin/env python
from twitter import Twitter, OAuth, TwitterHTTPError, TwitterStream
import sys
import os
import logging
import requests
from time import sleep

try:
    import json
except ImportError:
    import simplejson as json


class TwitterConsumer:
    def __init__(self):
        self.logger = logging.getLogger()
        handler = logging.StreamHandler()
        formatter = logging.Formatter(
            '%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
        handler.setFormatter(formatter)
        self.logger.addHandler(handler)

        loglevel = os.getenv('LOG_LEVEL', 'info')

        self.logger.setLevel(loglevel)

        needConfig = True
        while needConfig:
            #self.topics = 'docker,devops,#F1,coreos,#AWS,@Docker,@awscloud,@LewisHamilton'
            self.topics = os.getenv('TOPCIS', False)
            self.logger.debug("TOPICS: %s" % self.topics)
            self.twitter_access_token = os.getenv('TWITTER_ACCESS_TOKEN', False)
            self.logger.debug("twitter_access_token: %s" % self.twitter_access_token)
            self.twitter_access_secret = os.getenv('TWITTER_ACCESS_SECRET', False)
            self.logger.debug("twitter_access_secret: %s" % self.twitter_access_secret)
            self.twitter_consumer_key = os.getenv('TWITTER_CONSUMER_KEY', False)
            self.logger.debug("twitter_consumer_key: %s" % self.twitter_consumer_key)
            self.twitter_consumer_secret = os.getenv('TWITTER_CONSUMER_SECRET', False)
            self.logger.debug("twitter_consumer_secret: %s" % self.twitter_consumer_secret)
            self.giles_endpoint = os.getenv("GILES_ENDPOINT", 'http://giles/v1/')
            self.logger.debug("giles_endpoint: %s" % self.giles_endpoint)
            err = self._check_health()
            if len(err) > 0:
                for error in err:
                    self.logger.critical("Error: %s" % error)
                sleep(15)
            else:
                needConfig = False
                self.logger.info("Configuration Checks out, Starting up....")


    def _check_health(self):
        """
        Application health check
        Report Missing Settings to logger
        """
        err = []
        if self.topics:
            err.append("Missing Topics to Monitor")
        if self.twitter_access_token:
            err.append("Missing twitter_access_token")
        if self.twitter_access_secret:
            err.append("Missing twitter_access_secret")
        if self.twitter_consumer_key:
            err.append("Missing twitter_consumer_key")
        if self.twitter_consumer_secret:
            err.append("Missing twitter_consumer_secret")
        return err


    def _save_mongo(self, tweet):
        """
        Save Tweet to MongoDB via Giles API
        """
        headers = {'user-agent': 'inkslinger/0.0.1'}
        self.logger.debug("Saving Tweet....")
        self.logger.debug(tweet)
        url = "%s/%s" % (self.giles_endpoint, 'posts')
        self.logger.debug("Sending data to %s [PUT]" % url)
        r = requests.post(url, json=tweet, headers=headers)
        if r.status_code == 200:
{'fraction_non_alphanumeric': 0.11021296440230063, 'fraction_numerical': 0.004352557127312296, 'mean_word_length': 2.4314666666666667, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '24820', 'n_tokens_mistral': 1974, 'n_tokens_neox': 1867, 'n_words': 375}