<commit_before>#!/usr/bin/env python

import sys
import json
import time
import config
import oauth2
import logging


consumer = oauth2.Consumer(key=config.consumer_key, secret=config.consumer_secret)
token = oauth2.Token(config.access_token, config.access_token_secret)
client = oauth2.Client(consumer, token)

def sleep_till(t):
    now = time.time()
    if now > t:
        return 
    secs = t - now + 5 # padded with 5 seconds to be safe
    logging.info("sleeping %s seconds for rate limiting" % secs)
    time.sleep(secs)

def search(q, url=None, rate_limit_remaining=None, rate_limit_reset=None):
    if rate_limit_remaining != None and rate_limit_remaining == 0:
        sleep_till(rate_limit_reset)
        rate_limit_remaining = None
        rate_limit_reset = None

    # fetch some results
    if not url:
        url = "https://api.twitter.com/1.1/search/tweets.json?q=%s" % q

    logging.info("fetching %s", url)
    resp, content = client.request(url)

    # set rate limit info if not known
    if rate_limit_remaining == None:
        rate_limit_remaining = int(resp["x-rate-limit-remaining"])
        rate_limit_reset = int(resp["x-rate-limit-reset"])
    else:
        rate_limit_remaining -= 1

    # return an generator for each result
    results = json.loads(content)
    for status in results["statuses"]:
        yield status
   
    # look for the next set of results
    next_url = results["search_metadata"].get("next_results", None)
    if next_url:
        logging.info("fetching next page of results %s", next_url)
        next_url = "https://api.twitter.com/1.1/search/tweets.json" + next_url
        for status in search(q, next_url, rate_limit_remaining, rate_limit_reset):
            yield status

def archive(statuses, filename):
    fh = open(filename, "a")<commit_msg>try again when twitter api returns {"errors":[{"message":"Internal error","code":131}]}<commit_after>#!/usr/bin/env python

import sys
import json
import time
import config
import oauth2
import logging


consumer = oauth2.Consumer(key=config.consumer_key, secret=config.consumer_secret)
token = oauth2.Token(config.access_token, config.access_token_secret)
client = oauth2.Client(consumer, token)

def sleep_till(t):
    now = time.time()
    if now > t:
        return 
    secs = t - now + 5 # padded with 5 seconds to be safe
    logging.info("sleeping %s seconds for rate limiting" % secs)
    time.sleep(secs)

def search(q, url=None, rate_limit_remaining=None, rate_limit_reset=None):
    if rate_limit_remaining != None and rate_limit_remaining == 0:
        sleep_till(rate_limit_reset)
        rate_limit_remaining = None
        rate_limit_reset = None

    # fetch some results
    if not url:
        url = "https://api.twitter.com/1.1/search/tweets.json?q=%s" % q

    # twitter search api sometimes throws a 500, try 5 times and then give up
    count = 0
    while count <= 5:
        logging.info("fetching %s", url)
        resp, content = client.request(url)
        if resp.status == 200:
            break
        count += 1
        secs = count * 2
        logging.error("got error when fetching %s sleeping %s secs", url, secs)
        time.sleep(count * 2)

    if resp.status != 200:
        logging.fatal("couldn't get search results for %s" % url)
        sys.exit(1)

    # set rate limit info if not known
    if rate_limit_remaining == None:
        rate_limit_remaining = int(resp["x-rate-limit-remaining"])
        rate_limit_reset = int(resp["x-rate-limit-reset"])
    else:
        rate_limit_remaining -= 1

    # return an generator for each result
    results = json.loads(content)

    if not results.has_key("statuses"):
        print json.dumps(results, indent=2)
    for status in results["statuses"]:
        yield status
   
    # look for the next set of results
    next_url = results["search_metadata"].get("next_results", None)
    if next_url:
        logging.info("fetching next page of results %s", next_url)
        next_url = "https://api.twitter.com/1.1/search/tweets.json" + next_url
        for status in search(q, next_url, rate_limit_remaining, rate_limit_reset):
            yield status

def archive(statuses, filename):
    fh = open(filename, "a")
{'fraction_non_alphanumeric': 0.1060246679316888, 'fraction_numerical': 0.010436432637571158, 'mean_word_length': 3.0587102983638115, 'pattern_counts': {'":': 3, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3325688', 'n_tokens_mistral': 1378, 'n_tokens_neox': 1323, 'n_words': 361}