<commit_before>#!/usr/bin/env python

import tweepy
import numpy as np
import pandas as pd
import json
import csv
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from textblob import TextBlob

# import twitter keys and tokens
from config import *

#Authenticate via Twitter
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)

#Set the query variable
query = "Congress"
#query = raw_input("Twitter search term: ")

#Initialize tweets matrix
tweet_matrix = []

#Retrieve historical tweets, parse for selected information and append vectors to matrix
historical_tweets = tweepy.Cursor(api.search, q=query).items(10)
for tweet in historical_tweets:
    try:
        if tweet.retweeted_status:
            tweet_vector = [tweet.retweeted, tweet.author.screen_name, tweet.retweeted_status.text, tweet.coordinates]
            print tweet.retweeted_status.text
    except:
        tweet_vector = [tweet.retweeted, tweet.author.screen_name, tweet.text, tweet.coordinates]
        print tweet.text
    tweet_matrix.append(tweet_vector)

with open("tweets.csv", "wb") as f:
    writer = csv.writer(f)
    for row in tweet_matrix:
        try:
            writer.writerow(row)
        except Exception, e:
            pass

def polarity(s):
    blob = TextBlob(s)
    print blob
'''
    print data.text
    tweet = TextBlob(data.text)

    print tweet.sentiment.polarity
<commit_msg>Created process_tweet method<commit_after>#!/usr/bin/env python

import tweepy
import numpy as np
import pandas as pd
import json
import csv
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from textblob import TextBlob
import time

# import twitter keys and tokens
from config import *

#Authenticate via Twitter
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)

#Set the query variable
query = "@Betterment"
#query = raw_input("Twitter search term: ")

#Initialize tweets matrix
tweet_matrix = []

def process_tweet(t):
    return [t.created_at, t.user.screen_name, t.text, t.coordinates, t.is_quote_status]

#Retrieve historical tweets, parse for selected information and append vectors to matrix
raw_tweet_matrix = []
processed_tweet_matrix = []
tweets = tweepy.Cursor(api.search, q=query).items(10)
for tweet in tweets:
    raw_tweet_matrix.append(tweet)
    processed_tweet_matrix.append(process_tweet(tweet))

'''
historical_tweets = tweepy.Cursor(api.search, q=query).items(10)
for tweet in historical_tweets:
    try:
        if tweet.retweeted_status:
            tweet_vector = [tweet.retweeted, tweet.author.screen_name, tweet.retweeted_status.text, tweet.coordinates]
            print tweet.retweeted_status.text
    except:
        tweet_vector = [tweet.retweeted, tweet.author.screen_name, tweet.text, tweet.coordinates]
        print tweet.text
    tweet_matrix.append(tweet_vector)

with open("tweets.csv", "wb") as f:
    writer = csv.writer(f)
    for row in tweet_matrix:
        try:
            writer.writerow(row)
        except Exception, e:
            pass
'''
def polarity(s):
    blob = TextBlob(s)
    return blob.sentiment.polarity

def sentiment(s):
    if polarity(s) < 0:
        return "neg"
    return "pos"
'''
    print data.text
    tweet = TextBlob(data.text)

    print tweet.sentiment.polarity

{'fraction_non_alphanumeric': 0.08683151236342726, 'fraction_numerical': 0.0020126509488211618, 'mean_word_length': 3.8589385474860336, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2188548', 'n_tokens_mistral': 1155, 'n_tokens_neox': 1097, 'n_words': 294}