<commit_before>#!/usr/bin/env python

import os
import sys
import logging
import time
import datetime
import tweepy
import simplejson as json
import utils

import args
from listener import StreamListener
logging.basicConfig()
logger = logging.getLogger(__name__)

RETRY_LIMIT = 10

"""
    List of named location query (--location-query) names and their
    associated bounding boxes (longitude, latitude) (what is the name of that standard?  geoJSON?)

    Recursive lookups:
    If a value is string, it's a reference to a named entry in the table.  Use the remainder of the
    value string as the new lookup key.
"""
LOCATION_QUERY_MACROS = {
    'any': [-180,-90,180,90],
    'all': 'any',
    'global': 'any',
    'usa': [-124.848974,24.396308,-66.885444,49.384358], # http://www.openstreetmap.org/?box=yes&bbox=-124.848974,24.396308,-66.885444,49.384358
    'contintental_usa': 'usa'
}


def lookup_location_query_macro(name):
    """
    Look up location query name in macro table.
    Return list of coordinates of bounding box as floating point numbers, or None if
    not found.
    """
    resolved = LOCATION_QUERY_MACROS.get(name.lower())
    if isinstance(resolved, basestring):
        return lookup_location_query_macro(resolved)
    return resolved



def get_version():
    from __init__ import __version__
    return __version__


def location_query_to_location_filter(tweepy_auth, location_query):
    t = lookup_location_query_macro(location_query)
    if t:
        return t
    api = tweepy.API(tweepy_auth)
    # Normalize whitespace to single spaces.
    places = api.geo_search(query=location_query)
    normalized_location_query = location_query.replace(' ', '')
    for place in places:
        logger.debug('Considering place "%s"' % place.full_name)
        # Normalize spaces
        if place.full_name.replace(' ', '').lower() == normalized_location_query.lower():
            logger.info('Found matching place: full_name=%(full_name)s id=%(id)s url=%(url)s' % place.__dict__)
            if place.bounding_box is not None:
                t = [x for x in place.bounding_box.origin()]
                t.extend([x for x in place.bounding_box.corner()])
                logger.info('  location box: %s' % t)
                return t
            else:
                raise ValueError("Place '%s' does not have a bounding box." % place.full_name)

    # Nothing found, try for matching macro
    raise ValueError("'%s': No such place." % location_query)


def make_filter_args(opts, tweepy_auth):
    kwargs = {}
    if opts.track:
        kwargs['track'] = opts.track
    if opts.stall_warnings:
        kwargs['stall_warnings'] = True
    if opts.locations:
        kwargs['locations'] = map(float, opts.locations)
    if opts.location_query:
        kwargs['locations'] = location_query_to_location_filter(tweepy_auth, opts.location_query)
    if opts.follow:
        kwargs['follow'] = opts.follow
    return kwargs


def process_tweets(opts):
    """Set up and process incoming streams."""
    try:
        consumer_key = os.environ['CONSUMER_KEY']
        consumer_secret = os.environ['CONSUMER_SECRET']
        access_key = os.environ['ACCESS_KEY']
        access_secret = os.environ['ACCESS_SECRET']
        logger.debug('consumer_key={consumer_key}, consumer_secret={consumer_secret}'.format(**locals()))
        logger.debug('access_key={access_key}, access_secret={access_secret}'.format(**locals()))
        auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_key, access_secret)
    except KeyError as e:
        logger.error('You must set the %s API key environment variable.', e)
        return

    logger.debug('Init tweepy.Stream()')
    logger.debug(opts)
    listener = StreamListener(opts=opts, logger=logger)
    streamer = tweepy.Stream(auth=auth, listener=listener, retry_count=9999,
        retry_time=1, buffer_size=16000)

    try:
        kwargs = make_filter_args(opts, auth)
    except ValueError as e:
        listener.running = False
        sys.stderr.write("%s: error: %s\n" % (__file__, e.message))
        return

    while listener.running:
        try:
            try:
                logger.debug('streamer.filter(%s)', kwargs)
                streamer.filter(**kwargs)
            except TypeError as e:
                if 'stall_warnings' in e.message:
                    logger.warn("Installed Tweepy version doesn't support stall_warnings parameter.  Restarting without stall_warnings parameter.")
                    del kwargs['stall_warnings']
                    streamer.filter(**kwargs)
                else:
                    raise

            logger.debug('Returned from streaming...')
        except IOError:
            if opts.terminate_on_error:
                listener.running = False
            logger.exception('Caught IOError')
        except KeyboardInterrupt:
            # Stop the listener loop.
            listener.running = False
        except Exception:
            listener.running = False
            logger.exception("Unexpected exception.")

        if listener.running:
            logger.debug('Sleeping...')
            time.sleep(5)<commit_msg>Adding log output on KeyboardInterrupt exception<commit_after>#!/usr/bin/env python

import os
import sys
import logging
import time
import datetime
import tweepy
import simplejson as json
import utils

import args
from listener import StreamListener
logging.basicConfig()
logger = logging.getLogger(__name__)

RETRY_LIMIT = 10

"""
    List of named location query (--location-query) names and their
    associated bounding boxes (longitude, latitude) (what is the name of that standard?  geoJSON?)

    Recursive lookups:
    If a value is string, it's a reference to a named entry in the table.  Use the remainder of the
    value string as the new lookup key.
"""
LOCATION_QUERY_MACROS = {
    'any': [-180,-90,180,90],
    'all': 'any',
    'global': 'any',
    'usa': [-124.848974,24.396308,-66.885444,49.384358], # http://www.openstreetmap.org/?box=yes&bbox=-124.848974,24.396308,-66.885444,49.384358
    'contintental_usa': 'usa'
}


def lookup_location_query_macro(name):
    """
    Look up location query name in macro table.
    Return list of coordinates of bounding box as floating point numbers, or None if
    not found.
    """
    resolved = LOCATION_QUERY_MACROS.get(name.lower())
    if isinstance(resolved, basestring):
        return lookup_location_query_macro(resolved)
    return resolved



def get_version():
    from __init__ import __version__
    return __version__


def location_query_to_location_filter(tweepy_auth, location_query):
    t = lookup_location_query_macro(location_query)
    if t:
        return t
    api = tweepy.API(tweepy_auth)
    # Normalize whitespace to single spaces.
    places = api.geo_search(query=location_query)
    normalized_location_query = location_query.replace(' ', '')
    for place in places:
        logger.debug('Considering place "%s"' % place.full_name)
        # Normalize spaces
        if place.full_name.replace(' ', '').lower() == normalized_location_query.lower():
            logger.info('Found matching place: full_name=%(full_name)s id=%(id)s url=%(url)s' % place.__dict__)
            if place.bounding_box is not None:
                t = [x for x in place.bounding_box.origin()]
                t.extend([x for x in place.bounding_box.corner()])
                logger.info('  location box: %s' % t)
                return t
            else:
                raise ValueError("Place '%s' does not have a bounding box." % place.full_name)

    # Nothing found, try for matching macro
    raise ValueError("'%s': No such place." % location_query)


def make_filter_args(opts, tweepy_auth):
    kwargs = {}
    if opts.track:
        kwargs['track'] = opts.track
    if opts.stall_warnings:
        kwargs['stall_warnings'] = True
    if opts.locations:
        kwargs['locations'] = map(float, opts.locations)
    if opts.location_query:
        kwargs['locations'] = location_query_to_location_filter(tweepy_auth, opts.location_query)
    if opts.follow:
        kwargs['follow'] = opts.follow
    return kwargs


def process_tweets(opts):
    """Set up and process incoming streams."""
    try:
        consumer_key = os.environ['CONSUMER_KEY']
        consumer_secret = os.environ['CONSUMER_SECRET']
        access_key = os.environ['ACCESS_KEY']
        access_secret = os.environ['ACCESS_SECRET']
        logger.debug('consumer_key={consumer_key}, consumer_secret={consumer_secret}'.format(**locals()))
        logger.debug('access_key={access_key}, access_secret={access_secret}'.format(**locals()))
        auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_key, access_secret)
    except KeyError as e:
        logger.error('You must set the %s API key environment variable.', e)
        return

    logger.debug('Init tweepy.Stream()')
    logger.debug(opts)
    listener = StreamListener(opts=opts, logger=logger)
    streamer = tweepy.Stream(auth=auth, listener=listener, retry_count=9999,
        retry_time=1, buffer_size=16000)

    try:
        kwargs = make_filter_args(opts, auth)
    except ValueError as e:
        listener.running = False
        sys.stderr.write("%s: error: %s\n" % (__file__, e.message))
        return

    while listener.running:
        try:
            try:
                logger.debug('streamer.filter(%s)', kwargs)
                streamer.filter(**kwargs)
            except TypeError as e:
                if 'stall_warnings' in e.message:
                    logger.warn("Installed Tweepy version doesn't support stall_warnings parameter.  Restarting without stall_warnings parameter.")
                    del kwargs['stall_warnings']
                    streamer.filter(**kwargs)
                else:
                    raise

            logger.debug('Returned from streaming...')
        except IOError:
            if opts.terminate_on_error:
                listener.running = False
            logger.exception('Caught IOError')
        except KeyboardInterrupt:
            logger.warning("KeyboardInterrupt")
            # Stop the listener loop.
            listener.running = False
        except Exception:
            listener.running = False
            logger.exception("Unexpected exception.")

        if listener.running:
            logger.debug('Sleeping...')
            time.sleep(5)
{'fraction_non_alphanumeric': 0.10426404655155967, 'fraction_numerical': 0.016979872173995995, 'mean_word_length': 2.7177304964539006, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6233903', 'n_tokens_mistral': 3202, 'n_tokens_neox': 2998, 'n_words': 767}