# Path: tweetget/settings.py
# TWITTER_API_KEY = os.environ['TWITTER_API_KEY']
# TWITTER_API_SECRET = os.environ['TWITTER_API_SECRET']
# RATE_LIMIT = 1  # requests every 15 minutes, max is 450 for app twitter api
# RATE_LIMIT_WINDOW = 900  # 15 minutes * 60
# DATA_PATH = './data'
# QUERIES = ['python -monty', '#sarcasm', '#happy', '#sad']
# 
# Path: tweetget/single.py
# def get_tweets(query, count=TWEETS_PER_REQUEST, lang='en', result_type='recent', twitter=None, **kwargs):
#     query = query
#     twitter = twitter or get_twitter()
# 
#     params = {
#         'q': query,
#         'lang': lang,
#         'count': str(int(count)),
#         'result_type': result_type,
#     }
#     params.update(kwargs)
# 
#     # twython.exceptions.TwythonError: ('Connection aborted.', error(104, 'Connection reset by peer'))
#     resp = twitter.search(**params)
# 
#     return resp['statuses']
# 
# def save_tweets(tweets, query='', verbosity=1):
#     now = datetime.datetime.utcnow()
#     filename = os.path.join(DATA_PATH, '{}-{}.json.gz'.format(query, now.isoformat()).replace(':', '_'))
#     with gzip.open(filename, 'a') as f:
#         f.write(json.dumps(tweets, indent=2))
#     if verbosity > 0:
#         print('{} tweets written to {}'.format(len(tweets), filename))
# 
# def get_twitter():
#     twitter = Twython(TWITTER_API_KEY, TWITTER_API_SECRET, oauth_version=2)
#     return Twython(TWITTER_API_KEY, access_token=twitter.obtain_access_token())


import os
import sys
from .settings import DATA_PATH, TWEETS_PER_SEARCH, QUERIES
from .single import get_tweets, save_tweets, get_twitter
        # use search api to request 100 tweets. Twitter returns the most recent (max_id) first
        if oldest_id <= newest_id:
            tweets = get_tweets(query=query, max_id=oldest_id - 1, count=TWEETS_PER_SEARCH, twitter=twitter)
        else:
            tweets = get_tweets(query=query, max_id=oldest_id - 1, since_id=newest_id, count=TWEETS_PER_SEARCH, twitter=twitter)
        rate_limit_remaining = twitter.get_lastfunction_header('x-rate-limit-remaining')
        rate_limit_reset = twitter.get_lastfunction_header('x-rate-limit-reset')

        if not len(tweets):
            # not rate limitted, just no tweets returned by query
            oldest_id = oldest_id + ((newest_id or oldest_id) - oldest_id + 1) * 10000
            break
        elif isinstance(tweets, dict):
            # rate limit hit, or other twython response error
            print(tweets)
            break

        all_tweets.extend(tweets)

        # determine new oldest id
        tweet_ids = {t['id'] for t in tweets}
        if oldest_id:
            tweet_ids.add(oldest_id)
        oldest_id, newest_id = min(tweet_ids), max(tweet_ids)
        if rate_limit_remaining == 1:
            time.sleep(rate_limit_reset - time.time())

    save_tweets(all_tweets, query=query)

    # set id to start from for next time

# Path: tweetget/settings.py
# TWITTER_API_KEY = os.environ['TWITTER_API_KEY']
# TWITTER_API_SECRET = os.environ['TWITTER_API_SECRET']
# RATE_LIMIT = 1  # requests every 15 minutes, max is 450 for app twitter api
# RATE_LIMIT_WINDOW = 900  # 15 minutes * 60
# DATA_PATH = './data'
# QUERIES = ['python -monty', '#sarcasm', '#happy', '#sad']
# 
# Path: tweetget/single.py
# def get_tweets(query, count=TWEETS_PER_REQUEST, lang='en', result_type='recent', twitter=None, **kwargs):
#     query = query
#     twitter = twitter or get_twitter()
# 
#     params = {
#         'q': query,
#         'lang': lang,
#         'count': str(int(count)),
#         'result_type': result_type,
#     }
#     params.update(kwargs)
# 
#     # twython.exceptions.TwythonError: ('Connection aborted.', error(104, 'Connection reset by peer'))
#     resp = twitter.search(**params)
# 
#     return resp['statuses']
# 
# def save_tweets(tweets, query='', verbosity=1):
#     now = datetime.datetime.utcnow()
#     filename = os.path.join(DATA_PATH, '{}-{}.json.gz'.format(query, now.isoformat()).replace(':', '_'))
#     with gzip.open(filename, 'a') as f:
#         f.write(json.dumps(tweets, indent=2))
#     if verbosity > 0:
#         print('{} tweets written to {}'.format(len(tweets), filename))
# 
# def get_twitter():
#     twitter = Twython(TWITTER_API_KEY, TWITTER_API_SECRET, oauth_version=2)
#     return Twython(TWITTER_API_KEY, access_token=twitter.obtain_access_token())


# Path: tweetget/core.py
import os
import sys
from .settings import DATA_PATH, TWEETS_PER_SEARCH, QUERIES
from .single import get_tweets, save_tweets, get_twitter

        # use search api to request 100 tweets. Twitter returns the most recent (max_id) first
        if oldest_id <= newest_id:
            tweets = get_tweets(query=query, max_id=oldest_id - 1, count=TWEETS_PER_SEARCH, twitter=twitter)
        else:
            tweets = get_tweets(query=query, max_id=oldest_id - 1, since_id=newest_id, count=TWEETS_PER_SEARCH, twitter=twitter)
        rate_limit_remaining = twitter.get_lastfunction_header('x-rate-limit-remaining')
        rate_limit_reset = twitter.get_lastfunction_header('x-rate-limit-reset')

        if not len(tweets):
            # not rate limitted, just no tweets returned by query
            oldest_id = oldest_id + ((newest_id or oldest_id) - oldest_id + 1) * 10000
            break
        elif isinstance(tweets, dict):
            # rate limit hit, or other twython response error
            print(tweets)
            break

        all_tweets.extend(tweets)

        # determine new oldest id
        tweet_ids = {t['id'] for t in tweets}
        if oldest_id:
            tweet_ids.add(oldest_id)
        oldest_id, newest_id = min(tweet_ids), max(tweet_ids)
        if rate_limit_remaining == 1:
            time.sleep(rate_limit_reset - time.time())

    save_tweets(all_tweets, query=query)

    # set id to start from for next time

    _set_oldest_id(oldest_id, newest_id, query=query)