<commit_before>#!/usr/bin/env python
import sys, os
import re
import argparse
import dataset
import logging
from datetime import datetime
from time import sleep
from getpass import getpass
from search import Search
from persistence import get_last_id, get_first_id, store_response
import settings

logging.basicConfig()
logger = logging.getLogger("twick")

DEFAULT_DB = "sqlite:///twick.sqlite"

CREDENTIAL_NAMES = [
    "TWICK_API_KEY",
    "TWICK_API_SECRET",
    "TWICK_ACCESS_TOKEN",
    "TWICK_ACCESS_TOKEN_SECRET"
]

def try_credential(name):
    try: return os.environ[name]
    except: raise Exception("Missing environment variable ${}".format(name))

def get_credentials(credential_array=None):
    if credential_array: return credential_array
    return map(try_credential, CREDENTIAL_NAMES)

linebreak_pattern = re.compile(r"[\n\r]+")
def format_tweet(t):
    timestamp = t.timestamp.strftime(settings.date_format)
    screen_name = t.raw["user"]["screen_name"]
    text = t.raw["text"].strip()
    formatted = u"Tweet @ {}: <{}>: {}".format(timestamp, screen_name, text)
    newlines_removed = re.sub(linebreak_pattern, " ", formatted)
    return newlines_removed

def log_response(response):
    logger.info(u"Response @ {}: Found {} tweet(s).".format(
        response.timestamp.strftime(settings.date_format),
        len(response.tweets)))
    map(logger.info, map(format_tweet, response.tweets))
    pass

def cmd_fetch(args):
    search = Search(args.credentials)
    while True:
        last_id = get_last_id(args.db)
        response = search.query(args.query, since_id=last_id)
        log_response(response)
        store_response(args.db, response, args.store_raw)
        sleep(args.throttle)

def cmd_backfill(args):
    search = Search(args.credentials)
    while True:
        # Note: Unlike since_id, max_id is inclusive
        # Cf.: https://dev.twitter.com/docs/working-with-timelines
        first_id = get_first_id(args.db)
        max_id = (first_id - 1) if first_id else None
        response = search.query(args.query, max_id=max_id)
        log_response(response)
        store_response(args.db, response, args.store_raw)
        if not len(response.tweets): break <commit_msg>Use sys.maxint for first backfill max_id<commit_after>#!/usr/bin/env python
import sys, os
import re
import argparse
import dataset
import logging
from datetime import datetime
from time import sleep
from getpass import getpass
from search import Search
from persistence import get_last_id, get_first_id, store_response
import settings

logging.basicConfig()
logger = logging.getLogger("twick")

DEFAULT_DB = "sqlite:///twick.sqlite"

CREDENTIAL_NAMES = [
    "TWICK_API_KEY",
    "TWICK_API_SECRET",
    "TWICK_ACCESS_TOKEN",
    "TWICK_ACCESS_TOKEN_SECRET"
]

def try_credential(name):
    try: return os.environ[name]
    except: raise Exception("Missing environment variable ${}".format(name))

def get_credentials(credential_array=None):
    if credential_array: return credential_array
    return map(try_credential, CREDENTIAL_NAMES)

linebreak_pattern = re.compile(r"[\n\r]+")
def format_tweet(t):
    timestamp = t.timestamp.strftime(settings.date_format)
    screen_name = t.raw["user"]["screen_name"]
    text = t.raw["text"].strip()
    formatted = u"Tweet @ {}: <{}>: {}".format(timestamp, screen_name, text)
    newlines_removed = re.sub(linebreak_pattern, " ", formatted)
    return newlines_removed

def log_response(response):
    logger.info(u"Response @ {}: Found {} tweet(s).".format(
        response.timestamp.strftime(settings.date_format),
        len(response.tweets)))
    map(logger.info, map(format_tweet, response.tweets))
    pass

def cmd_fetch(args):
    search = Search(args.credentials)
    while True:
        last_id = get_last_id(args.db)
        response = search.query(args.query, since_id=last_id)
        log_response(response)
        store_response(args.db, response, args.store_raw)
        sleep(args.throttle)

def cmd_backfill(args):
    search = Search(args.credentials)
    while True:
        # Note: Unlike since_id, max_id is inclusive
        # Cf.: https://dev.twitter.com/docs/working-with-timelines
        first_id = get_first_id(args.db)
        max_id = (first_id or sys.maxint) - 1
        response = search.query(args.query, max_id=max_id)
        log_response(response)
        store_response(args.db, response, args.store_raw)
        if not len(response.tweets): break 
{'fraction_non_alphanumeric': 0.11333483247132899, 'fraction_numerical': 0.00044974139869574995, 'mean_word_length': 3.9642857142857144, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1297998', 'n_tokens_mistral': 1462, 'n_tokens_neox': 1451, 'n_words': 299}