<commit_before>from __future__ import unicode_literals
import logging

import tweepy

import models


LOGGER = logging.getLogger(__name__)

SERVICES = [
    'API', 'JS', 'DASH'
]

STATUSES = [
    '', 'UP', 'DOWN', 'ISSUE'
]


class TwitterStatusProcessor(object):
    SEPARATOR = ':'

    def __init__(self, **auth):
        if auth:
            self.auth = tweepy.OAuthHandler(auth['consumer_key'],
                                            auth['consumer_secret'])
            self.auth.set_access_token(auth['token'], auth['token_secret'])

            self.twitter = tweepy.API(
                self.auth,
                api_root='/1.1',
            )

    def _parse_tweet(self, message, created_at, tweet_id, tweet_id_str):
        spec, _, message = message.partition(self.SEPARATOR)
        if not spec:
            return

        spec = spec.split('-')
        service = spec[0]
        if len(spec) > 1:
            state = spec[1]
        else:
            state = ''
        if service not in SERVICES or state not in STATUSES:
            return

        self._insert(service, created_at, message, state, tweet_id_str)

        self._set_last_updated(service, tweet_id)

    def _get_tw_key(self, service=None):
        if not service:
            service = '__general'
        return service + '-timestamp'

    def _set_last_updated(self, service=None, tweet_id=None):
        key = self._get_tw_key(service)
        kv = models.KV(k=key, value=str(tweet_id), key_name=key)
        kv.put()<commit_msg>Use a case-insensitive check of the status string<commit_after>from __future__ import unicode_literals
import logging

import tweepy

import models


LOGGER = logging.getLogger(__name__)

SERVICES = [
    'API', 'JS', 'DASH'
]

STATUSES = [
    '', 'UP', 'DOWN', 'ISSUE'
]


class TwitterStatusProcessor(object):
    SEPARATOR = ':'

    def __init__(self, **auth):
        if auth:
            self.auth = tweepy.OAuthHandler(auth['consumer_key'],
                                            auth['consumer_secret'])
            self.auth.set_access_token(auth['token'], auth['token_secret'])

            self.twitter = tweepy.API(
                self.auth,
                api_root='/1.1',
            )

    def _parse_tweet(self, message, created_at, tweet_id, tweet_id_str):
        spec, _, message = message.partition(self.SEPARATOR)
        if not spec:
            return

        spec = spec.split('-')
        service = spec[0]
        if len(spec) > 1:
            state = spec[1].upper()
        else:
            state = ''
        if service not in SERVICES or state not in STATUSES:
            return

        self._insert(service, created_at, message, state, tweet_id_str)

        self._set_last_updated(service, tweet_id)

    def _get_tw_key(self, service=None):
        if not service:
            service = '__general'
        return service + '-timestamp'

    def _set_last_updated(self, service=None, tweet_id=None):
        key = self._get_tw_key(service)
        kv = models.KV(k=key, value=str(tweet_id), key_name=key)
        kv.put()
{'fraction_non_alphanumeric': 0.11269430051813471, 'fraction_numerical': 0.003238341968911917, 'mean_word_length': 2.1014056224899598, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '728236', 'n_tokens_mistral': 983, 'n_tokens_neox': 934, 'n_words': 208}