<commit_before>'''
Created on Sep 26, 2014

@author: tim
'''
import tweepy
from tweepy import OAuthHandler
import time
import itertools
import os
import cPickle as pickle
import sys
import re
import traceback
os.environ['http_proxy']= ''
import logging 
import os

t0 = time.time()

DEBUG = True

config_name = 'debug.txt'

# determine if application is a script file or frozen exe
if getattr(sys, 'frozen', False):
    application_path = os.path.dirname(sys.executable)
elif __file__:
    application_path = os.path.dirname(__file__)

config_path = os.path.join(application_path, config_name)


logging.basicConfig(filename=config_path, format='%(message)s', level=logging.DEBUG)

def log(s): 
    if DEBUG: logging.log(logging.DEBUG, s)
        
ckeys = []
csecrets = []
atokens = []
asecrets = []
        

ckeys.append('Go2BKKaGREBEioh1BEeRhA')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

#debug!
ckeys.append('h554bvyP67an9o98A0uZl8D5i')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>0McxYgXM')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('872274950-LhxYgTpUtLEsVAn6TAMPWAYhwEIuZEOrsFa5593I')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('2VPJEUQ9j1U1RBifjLzoORHPz')
csecrets.append('kTSS1DsYp5awkjCWUndEoKileEDmgLouHXixAWItCtPRaaafES')
atokens.append('<KEY>')
asecrets.append('<KEY>')

current_api = 0

base_path = sys.argv[0].replace('hashtag_crawler.py','')
data_de = []

print base_path

def cast_status_to_dict(tweepy_status):
    ret = {}
    ret['text'] = tweepy_status.text
    ret['follower_count'] = tweepy_status.author.followers_count
    ret['friends_count'] = tweepy_status.author.friends_count
    ret['retweet_count'] = tweepy_status.retweet_count
    return ret

auth = OAuthHandler(ckeys[0], csecrets[0])
auth.set_access_token(atokens[0], asecrets[0])
api = tweepy.API(auth)

def process_hashtag(query):
    global current_api
    global processed_hashtags    
    global data_de
    global api
    processed_hashtags[query] = True
    items = tweepy.Cursor(api.search, q=hashtag + query,lang='de').items(max_tweets)
    i = 0
    last_error = 0
    while True:
        try: 
            i += 1
            status = next(items)
            if status.text not in unique_tweets:   
                unique_tweets[status.text] = True     
                data_de.append(cast_status_to_dict(status))
                matches = hashtag_rex.findall(status.text)  
                for match in matches:
                    match = match.lower()
                    if match not in processed_hashtags and match not in next_wave:
                        next_wave.append(match)                 
        except tweepy.TweepError:
            print  traceback.format_exc()
            #print  sys.exc_info()[0]
            if i > 0 and last_error + 10 > i: 
                time.sleep(60)
                log('Recurrent rate error. Going to sleep now...')
            last_error = i
            log('changed to next api key: {}'.format(current_api))
            current_api += 1 
            if current_api >= len(ckeys): current_api = 0
            auth = OAuthHandler(ckeys[i], csecrets[i])
            auth.set_access_token(atokens[i], asecrets[i])
            api = tweepy.API(auth)            
            continue
        except StopIteration:
            print i
            break

def process_current_wave():
    global current_wave
    i = 0
    while len(current_wave) > 0:
        if i % 10 == 0 and i > 0:     
            log('Dumping data...')       
            pickle.dump(processed_hashtags, open(base_path + 'hashtags.p','wb'))
            pickle.dump(data_de, open(base_path + 'data_de2.p','wb'))
        process_hashtag(current_wave[0])
        processed_hashtags[current_wave[0]] = True<commit_msg>Fixed index bug.<commit_after>'''
Created on Sep 26, 2014

@author: tim
'''
import tweepy
from tweepy import OAuthHandler
import time
import itertools
import os
import cPickle as pickle
import sys
import re
import traceback
os.environ['http_proxy']= ''
import logging 
import os

t0 = time.time()

DEBUG = True

config_name = 'debug.txt'

# determine if application is a script file or frozen exe
if getattr(sys, 'frozen', False):
    application_path = os.path.dirname(sys.executable)
elif __file__:
    application_path = os.path.dirname(__file__)

config_path = os.path.join(application_path, config_name)


logging.basicConfig(filename=config_path, format='%(message)s', level=logging.DEBUG)

def log(s): 
    if DEBUG: logging.log(logging.DEBUG, s)
        
ckeys = []
csecrets = []
atokens = []
asecrets = []
        

ckeys.append('Go2BKKaGREBEioh1BEeRhA')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('napyrVTJtsycJ9sTQOdQw')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

#debug!
ckeys.append('h554bvyP67an9o98A0uZl8D5i')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('872274950-LhxYgTpUtLEsVAn6TAMPWAYhwEIuZEOrsFa5593I')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('<KEY>')
csecrets.append('<KEY>')
atokens.append('<KEY>')
asecrets.append('<KEY>')

ckeys.append('2VPJEUQ9j1U1RBifjLzoORHPz')
csecrets.append('kTSS1DsYp5awkjCWUndEoKileEDmgLouHXixAWItCtPRaaafES')
atokens.append('<KEY>')
asecrets.append('<KEY>')

current_api = 0

base_path = sys.argv[0].replace('hashtag_crawler.py','')
data_de = []

print base_path

def cast_status_to_dict(tweepy_status):
    ret = {}
    ret['text'] = tweepy_status.text
    ret['follower_count'] = tweepy_status.author.followers_count
    ret['friends_count'] = tweepy_status.author.friends_count
    ret['retweet_count'] = tweepy_status.retweet_count
    return ret

auth = OAuthHandler(ckeys[0], csecrets[0])
auth.set_access_token(atokens[0], asecrets[0])
api = tweepy.API(auth)

def process_hashtag(query):
    global current_api
    global processed_hashtags    
    global data_de
    global api
    processed_hashtags[query] = True
    items = tweepy.Cursor(api.search, q=hashtag + query,lang='de').items(max_tweets)
    i = 0
    last_error = 0
    while True:
        try: 
            i += 1
            status = next(items)
            if status.text not in unique_tweets:   
                unique_tweets[status.text] = True     
                data_de.append(cast_status_to_dict(status))
                matches = hashtag_rex.findall(status.text)  
                for match in matches:
                    match = match.lower()
                    if match not in processed_hashtags and match not in next_wave:
                        next_wave.append(match)                 
        except tweepy.TweepError:
            print  traceback.format_exc()
            #print  sys.exc_info()[0]
            if i > 0 and last_error + 10 > i: 
                time.sleep(60)
                log('Recurrent rate error. Going to sleep now...')
            last_error = i
            current_api += 1 
            log('changed to next api key: {}'.format(current_api))
            if current_api >= len(ckeys): current_api = 0
            auth = OAuthHandler(ckeys[current_api], csecrets[current_api])
            auth.set_access_token(atokens[current_api], asecrets[current_api])
            api = tweepy.API(auth)            
            continue
        except StopIteration:
            print i
            break

def process_current_wave():
    global current_wave
    i = 0
    while len(current_wave) > 0:
        if i % 10 == 0 and i > 0:     
            log('Dumping data...')       
            pickle.dump(processed_hashtags, open(base_path + 'hashtags.p','wb'))
            pickle.dump(data_de, open(base_path + 'data_de2.p','wb'))
        process_hashtag(current_wave[0])
        processed_hashtags[current_wave[0]] = True
{'fraction_non_alphanumeric': 0.15802930474972957, 'fraction_numerical': 0.013177303569672534, 'mean_word_length': 3.5605381165919283, 'pattern_counts': {'":': 0, '<': 128, '<?xml version=': 0, '>': 138, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4089077', 'n_tokens_mistral': 3834, 'n_tokens_neox': 3622, 'n_words': 607}