<commit_before>#!/usr/bin/env python

from __future__ import print_function

import os
import re
import sys
import json
import time
import yaml
import logging
import argparse
import requests

from requests_oauthlib import OAuth1Session


def main():
    """
    The twarc command line.
    """
    parser = argparse.ArgumentParser("twarc")
    parser.add_argument("--search", dest="search",
                        help="search for tweets matching a query")
    parser.add_argument("--max_id", dest="max_id",
                        help="maximum tweet id to search for")
    parser.add_argument("--since_id", dest="since_id",
                        help="smallest id to search for")
    parser.add_argument("--stream", dest="stream",
                        help="stream tweets matching filter")
    parser.add_argument("--hydrate", dest="hydrate",
                        help="rehydrate tweets from a file of tweet ids")
    parser.add_argument("--log", dest="log",
                        default="twarc.log", help="log file")
    parser.add_argument("--consumer_key",
                        default=None, help="Twitter API consumer key")
    parser.add_argument("--consumer_secret",
                        default=None, help="Twitter API consumer secret")
    parser.add_argument("--access_token",
                        default=None, help="Twitter API access key")
    parser.add_argument("--access_token_secret",
                        default=None, help="Twitter API access token secret")
    parser.add_argument('-y', '--yaml',
                        default='twarc.yaml',
                        help="YAML file containing Twitter keys and secrets")
    args = parser.parse_args()

    logging.basicConfig(
        filename=args.log,
        level=logging.INFO,
        format="%(asctime)s %(levelname)s %(message)s"
    )

    consumer_key = args.consumer_key or os.environ.get('CONSUMER_KEY')
    consumer_secret = args.consumer_secret or os.environ.get('CONSUMER_SECRET')
    access_token = args.access_token or os.environ.get('ACCESS_TOKEN')
    access_token_secret = args.access_token_secret or os.environ.get("ACCESS_TOKEN_SECRET")

    if not (consumer_key and consumer_secret and
            access_token and access_token_secret):
        credentials = load_yaml(args.yaml)
        if credentials:
            consumer_key = credentials['consumer_key']
            consumer_secret = credentials['consumer_secret']
            access_token = credentials['access_token']
            access_token_secret = credentials['access_token_secret']
        else:
            raise argparse.ArgumentTypeError("Please make sure to use command "
                "line arguments to set the Twitter API keys or set the "
                "CONSUMER_KEY, CONSUMER_SECRET ACCESS_TOKEN and "
                "ACCESS_TOKEN_SECRET environment variables or use a YAML file")

    t = Twarc(consumer_key=consumer_key,
              consumer_secret=consumer_secret,
              access_token=access_token,
              access_token_secret=access_token_secret)

    if args.search:
        tweets = t.search(
            args.search,
            since_id=args.since_id,
            max_id=args.max_id
        )
    elif args.stream:
        tweets = t.stream(args.stream)
    elif args.hydrate:
        tweets = t.hydrate(open(args.hydrate, 'rU'))
    else:
        raise argparse.ArgumentTypeError("must supply one of: --search --stream or --hydrate")

    # iterate through the tweets and write them to stdout
    for tweet in tweets:
        if "id_str" in tweet:
            logging.info("archived %s", tweet["id_str"])
            print(json.dumps(tweet))


def load_yaml(filename):
    """
    File should contain:
    consumer_key: TODO_ENTER_YOURS
    consumer_secret: TODO_ENTER_YOURS
    access_token: TODO_ENTER_YOURS
    access_token_secret: TODO_ENTER_YOURS
    """
    if not os.path.isfile(filename):
        return None
    f = open(filename)
    data = yaml.safe_load(f)
    f.close()
    if not data.viewkeys() >= {
            'access_token', 'access_token',
            'consumer_key', 'consumer_secret'}:
        sys.exit("Twitter credentials missing from YAML: " + filename)
    return data


def rate_limit(f):
    """
    A decorator to handle rate limiting from the Twitter API. If
    a rate limit error is encountered we will sleep until we can
    issue the API call again.<commit_msg>If no keys, ask for them and save to file<commit_after>#!/usr/bin/env python

from __future__ import print_function

import os
import sys
import json
import time
import yaml
import logging
import argparse
import requests

from requests_oauthlib import OAuth1Session

if sys.version_info[:2] <= (2, 7):
    # Python 2
    get_input = raw_input
else:
    # Python 3
    get_input = input


def main():
    """
    The twarc command line.
    """
    parser = argparse.ArgumentParser("twarc")
    parser.add_argument("--search", dest="search",
                        help="search for tweets matching a query")
    parser.add_argument("--max_id", dest="max_id",
                        help="maximum tweet id to search for")
    parser.add_argument("--since_id", dest="since_id",
                        help="smallest id to search for")
    parser.add_argument("--stream", dest="stream",
                        help="stream tweets matching filter")
    parser.add_argument("--hydrate", dest="hydrate",
                        help="rehydrate tweets from a file of tweet ids")
    parser.add_argument("--log", dest="log",
                        default="twarc.log", help="log file")
    parser.add_argument("--consumer_key",
                        default=None, help="Twitter API consumer key")
    parser.add_argument("--consumer_secret",
                        default=None, help="Twitter API consumer secret")
    parser.add_argument("--access_token",
                        default=None, help="Twitter API access key")
    parser.add_argument("--access_token_secret",
                        default=None, help="Twitter API access token secret")
    parser.add_argument('-y', '--yaml',
                        default='twarc.yaml',
                        help="YAML file containing Twitter keys and secrets")
    args = parser.parse_args()

    logging.basicConfig(
        filename=args.log,
        level=logging.INFO,
        format="%(asctime)s %(levelname)s %(message)s"
    )

    consumer_key = args.consumer_key or os.environ.get('CONSUMER_KEY')
    consumer_secret = args.consumer_secret or os.environ.get('CONSUMER_SECRET')
    access_token = args.access_token or os.environ.get('ACCESS_TOKEN')
    access_token_secret = args.access_token_secret or os.environ.get("ACCESS_TOKEN_SECRET")

    if not (consumer_key and consumer_secret and
            access_token and access_token_secret):
        credentials = load_yaml(args.yaml)
        if credentials:
            consumer_key = credentials['consumer_key']
            consumer_secret = credentials['consumer_secret']
            access_token = credentials['access_token']
            access_token_secret = credentials['access_token_secret']
        else:
            print("Please enter Twitter keys and secrets.")
            consumer_key = get_input('Consumer key: ')
            consumer_secret = get_input('Consumer secret: ')
            access_token = get_input('Access_token: ')
            access_token_secret = get_input('Access token secret: ')
            save_keys(consumer_key, consumer_secret,
                      access_token, access_token_secret)

    t = Twarc(consumer_key=consumer_key,
              consumer_secret=consumer_secret,
              access_token=access_token,
              access_token_secret=access_token_secret)

    if args.search:
        tweets = t.search(
            args.search,
            since_id=args.since_id,
            max_id=args.max_id
        )
    elif args.stream:
        tweets = t.stream(args.stream)
    elif args.hydrate:
        tweets = t.hydrate(open(args.hydrate, 'rU'))
    else:
        raise argparse.ArgumentTypeError("must supply one of: --search --stream or --hydrate")

    # iterate through the tweets and write them to stdout
    for tweet in tweets:
        if "id_str" in tweet:
            logging.info("archived %s", tweet["id_str"])
            print(json.dumps(tweet))


def load_yaml(filename):
    """
    File should contain:
    consumer_key: TODO_ENTER_YOURS
    consumer_secret: TODO_ENTER_YOURS
    access_token: TODO_ENTER_YOURS
    access_token_secret: <PASSWORD>_ENTER_YOURS
    """
    if not os.path.isfile(filename):
        return None
    f = open(filename)
    data = yaml.safe_load(f)
    f.close()
    if not data.viewkeys() >= {
            'access_token', 'access_token',
            'consumer_key', 'consumer_secret'}:
        sys.exit("Twitter credentials missing from YAML: " + filename)
    return data


def save_yaml(filename, data):
    """
    Save data to filename in YAML format
    """
    with open(filename, 'w') as yaml_file:
        yaml_file.write(yaml.safe_dump(data, default_flow_style=False))


def save_keys(consumer_key, consumer_secret,
              access_token, access_token_secret):
    """
    Save keys to ~/.twarc.yaml
    """
    credentials = {'consumer_key': consumer_key,
                   'consumer_secret': consumer_secret,
                   'access_token': access_token,
                   'access_token_secret': access_token_secret
                   }
    home = os.path.expanduser("~")
    filename = os.path.join(home, ".twarc.yaml")
    save_yaml(filename, credentials)
    print("Keys saved to", filename)


def rate_limit(f):
    """
    A decorator to handle rate limiting from the Twitter API. If
    a rate limit error is encountered we will sleep until we can
    issue the API call again.
{'fraction_non_alphanumeric': 0.09977601303196905, 'fraction_numerical': 0.0007126858073712074, 'mean_word_length': 2.4515108924806746, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3030422', 'n_tokens_mistral': 2870, 'n_tokens_neox': 2715, 'n_words': 679}