<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import ConfigParser
import hashlib
import inspect
import os

import tweepy

path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))

config = ConfigParser.SafeConfigParser()
config.read(os.path.join(path, 'config'))

search_query = config.get('settings', 'search_query')
tweet_language = config.get('settings', 'tweet_language')

user_blacklist = []
word_blacklist = ['RT', u'♺']

# Build savepoint path + file
hashed_search_query = hashlib.md5(search_query).hexdigest()
last_id_filename = 'last_id_search_query_%s' % hashed_search_query
rt_bot_path = os.path.dirname(os.path.abspath(__file__))
last_id_file = os.path.join(rt_bot_path, last_id_filename)

auth = tweepy.OAuthHandler(config.get('twitter', 'consumer_key'), config.get('twitter', 'consumer_secret'))
auth.set_access_token(config.get('twitter', 'access_token'), config.get('twitter', 'access_token_secret'))
api = tweepy.API(auth)

# Retrieve last savepoint if available
try:
    with open(last_id_file, 'r') as file:
        savepoint = file.read()
except IOError:
    savepoint = ''
    print 'No savepoint found. Trying to get as many results as possible.'

# Search query
timelineIterator = tweepy.Cursor(api.search, q=search_query, since_id=savepoint, lang=tweet_language).items()

timeline = list(timelineIterator)

try:
    last_tweet_id = timeline[0].id
except IndexError:
    last_tweet_id = savepoint

# Filter @replies/blacklisted words & users out and reverse timeline<commit_msg>Removed unused blacklisted word.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import ConfigParser
import hashlib
import inspect
import os

import tweepy

path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))

config = ConfigParser.SafeConfigParser()
config.read(os.path.join(path, 'config'))

search_query = config.get('settings', 'search_query')
tweet_language = config.get('settings', 'tweet_language')

user_blacklist = []
word_blacklist = ['RT']

# Build savepoint path + file
hashed_search_query = hashlib.md5(search_query).hexdigest()
last_id_filename = 'last_id_search_query_%s' % hashed_search_query
rt_bot_path = os.path.dirname(os.path.abspath(__file__))
last_id_file = os.path.join(rt_bot_path, last_id_filename)

auth = tweepy.OAuthHandler(config.get('twitter', 'consumer_key'), config.get('twitter', 'consumer_secret'))
auth.set_access_token(config.get('twitter', 'access_token'), config.get('twitter', 'access_token_secret'))
api = tweepy.API(auth)

# Retrieve last savepoint if available
try:
    with open(last_id_file, 'r') as file:
        savepoint = file.read()
except IOError:
    savepoint = ''
    print 'No savepoint found. Trying to get as many results as possible.'

# Search query
timelineIterator = tweepy.Cursor(api.search, q=search_query, since_id=savepoint, lang=tweet_language).items()

timeline = list(timelineIterator)

try:
    last_tweet_id = timeline[0].id
except IndexError:
    last_tweet_id = savepoint

# Filter @replies/blacklisted words & users out and reverse timeline
{'fraction_non_alphanumeric': 0.12762520193861066, 'fraction_numerical': 0.0019386106623586429, 'mean_word_length': 5.972972972972973, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1483641', 'n_tokens_mistral': 1048, 'n_tokens_neox': 1034, 'n_words': 215}