<commit_before># helper functions for twitter data analysis

import numpy as np
import pandas as pd
import nltk
from nltk.corpus import stopwords
import string
from operator import itemgetter
import seaborn as sns
import matplotlib.pyplot as plt
import qgrid
from math import log
import got
import time
import timeit


class Twords(object):
    """ Object that takes in tweets from Java twitter search engine and allows
    manipulation, analysis and visualization.
    """

    def __init__(self):
        self.data_path = ''
        self.background_path = ''
        self.background_dict = {}
        self.search_terms = []
        self.tweets_df = pd.DataFrame()
        self.word_bag = []
        self.freq_dist = nltk.FreqDist(self.word_bag)
        self.word_freq_df = pd.DataFrame()

    def __repr__(self):
        return "Twitter word analysis object"

    #############################################################
    # Methods to set attributes
    #############################################################

    def set_Data_path(self, data_path):
        """ data_path is path to data set from java twitter search"""
        self.data_path = data_path

    def set_Background_path(self, background_path):
        """ background_path is path to background data set from java
        twitter search"""
        self.background_path = background_path

    def set_Search_terms(self, search_terms):
        """ search_terms is a list of strings that were used in twitter search
        to obtain data in tweets_df
        """
        assert type(search_terms) == list
        for term in search_terms:
            assert type(term) == str
            assert term
        self.search_terms = search_terms

    def create_Background_dict(self):
        """ Create the dictionary of background word rates from file in the
        background data path
        """
        sample_rates = pd.read_csv(self.background_path, sep=",")
        self.background_dict = sample_rates[["word", "frequency"]].set_index("word")["frequency"].to_dict()


    #############################################################
    # Methods to gather tweets with Python GetOldTweets
    #############################################################

    def collect_one_run(self, search_terms, end_date, call_size):
        """ Does one twitter search with GetOldTweets python library.
        search_terms: string of terms to search on
        end_date: the date at which search should end (e.g., if you want
                    the most recent tweets, set this to today)
        call_size: number of tweets to return.

        Returns a list of tweets (packaged as dictionaries) and the most recent
        date that was searched. The most recent date is used when making
        repeated calls with the python library, which is necessary because
        searches of more than around 50,000 tweets get hung up on website.
        """
        tweetCriteria = got.manager.TweetCriteria()
        tweetCriteria.setQuerySearch(search_terms)
        tweetCriteria.setUntil(end_date)
        tweetCriteria.setMaxTweets(call_size)
        tweets = got.manager.TweetManager.getTweets(tweetCriteria)

        if len(tweets) == 0: # catches cases when twitter blocks search
            print "Could not retrieve any tweets"
            return None

        rows_list = []
        for tweet in tweets:
            row_dict = {"username": tweet.username,
                        "date": str(tweet.date.date()),
                        "retweets": tweet.retweets,
                        "favorites": tweet.favorites,
                        "text": tweet.text,
                        "mentions": tweet.mentions,
                        "hashtags": tweet.hashtags,
                        "id": tweet.id,
                        "permalink": tweet.permalink}
            rows_list.append(row_dict)
        return rows_list, row_dict["date"]

    def create_tweets_dataframe(self, search_terms, final_end_date,
                                num_tweets, call_size):
        """ Performs repeated calls to collect_one_run to get num_tweets into
        a dataframe. Each call to collect_one_run takes call_size.

        call_size should be on the order of several thousand for best results

        Each time a new call to collect_one_run is made, the date is
        incremented backward by one day. This means that if seven calls are
        made and each call only takes tweets from one day, seven different
        days (starting with final_end_date and moving backward in time) will
        be sampled.

        final_end_date should be a string in form "2015-12-31"
        """
        total_row_list = []
        search_date = final_end_date
        num_tweets_searched = 0

        starttime = time.time()

        while num_tweets_searched < num_tweets:
            row_list, last_date = self.collect_one_run(search_terms,
                                                       search_date,
                                                       call_size)

            total_row_list += row_list
            search_date = last_date
            num_tweets_searched += call_size

        # once all tweets are collected, combine them all into dataframe
        column_names = ["username",
                        "date",
                        "retweets",
                        "favorites",
                        "text",
                        "mentions",
                        "hashtags",
                        "id",
                        "permalink"]

        tweets_df = pd.DataFrame(total_row_list, columns = column_names)

        print "Time to collect ",  str(num_tweets), " tweets: ", time.time() - starttime, "seconds"

        self.tweets_df = tweets_df

    #############################################################
    # Methods to gather tweets and prune (done every time)
    #############################################################

    def get_tweets(self):
        """ Takes path to twitter data obtained with java tweet search library
        and returns a dataframe of the tweets and their accompanying
        information. Dataframe has columns for username, date, retweets,
        favorites, text, mentions, and hashtag. The dataframe is stored under
        the attribute tweets_pd.
        """
        # Read in csv file with many columns to account for people who put many
        # semicolons in tweets, then keep only the rows that don't have
        # semicolons in a tweet by dropping rows with too many columns.
        # (Semicolons are the delimeter in the java twitter search library.)
        tweets = pd.read_csv(self.data_path, sep=";",
                             names=list('abcdefghijklmno'))
        tweets = tweets[tweets.k.isnull()]
<commit_msg>added a comment line that explains how to output file to csv. The output function is a built-in one from pandas, but I always forget to set index=False, so I'm recording it in the comment for posterity.<commit_after># helper functions for twitter data analysis

import numpy as np
import pandas as pd
import nltk
from nltk.corpus import stopwords
import string
from operator import itemgetter
import seaborn as sns
import matplotlib.pyplot as plt
import qgrid
from math import log
import got
import time
import timeit


class Twords(object):
    """ Object that takes in tweets from Java twitter search engine and allows
    manipulation, analysis and visualization.
    """

    def __init__(self):
        self.data_path = ''
        self.background_path = ''
        self.background_dict = {}
        self.search_terms = []
        self.tweets_df = pd.DataFrame()
        self.word_bag = []
        self.freq_dist = nltk.FreqDist(self.word_bag)
        self.word_freq_df = pd.DataFrame()

    def __repr__(self):
        return "Twitter word analysis object"

    #############################################################
    # Methods to set attributes
    #############################################################

    def set_Data_path(self, data_path):
        """ data_path is path to data set from java twitter search"""
        self.data_path = data_path

    def set_Background_path(self, background_path):
        """ background_path is path to background data set from java
        twitter search"""
        self.background_path = background_path

    def set_Search_terms(self, search_terms):
        """ search_terms is a list of strings that were used in twitter search
        to obtain data in tweets_df
        """
        assert type(search_terms) == list
        for term in search_terms:
            assert type(term) == str
            assert term
        self.search_terms = search_terms

    def create_Background_dict(self):
        """ Create the dictionary of background word rates from file in the
        background data path
        """
        sample_rates = pd.read_csv(self.background_path, sep=",")
        self.background_dict = sample_rates[["word", "frequency"]].set_index("word")["frequency"].to_dict()


    #############################################################
    # Methods to gather tweets with Python GetOldTweets
    #############################################################

    def collect_one_run(self, search_terms, end_date, call_size):
        """ Does one twitter search with GetOldTweets python library.
        search_terms: string of terms to search on
        end_date: the date at which search should end (e.g., if you want
                    the most recent tweets, set this to today)
        call_size: number of tweets to return.

        Returns a list of tweets (packaged as dictionaries) and the most recent
        date that was searched. The most recent date is used when making
        repeated calls with the python library, which is necessary because
        searches of more than around 50,000 tweets get hung up on website.
        """
        tweetCriteria = got.manager.TweetCriteria()
        tweetCriteria.setQuerySearch(search_terms)
        tweetCriteria.setUntil(end_date)
        tweetCriteria.setMaxTweets(call_size)
        tweets = got.manager.TweetManager.getTweets(tweetCriteria)

        if len(tweets) == 0: # catches cases when twitter blocks search
            print "Could not retrieve any tweets"
            return None

        rows_list = []
        for tweet in tweets:
            row_dict = {"username": tweet.username,
                        "date": str(tweet.date.date()),
                        "retweets": tweet.retweets,
                        "favorites": tweet.favorites,
                        "text": tweet.text,
                        "mentions": tweet.mentions,
                        "hashtags": tweet.hashtags,
                        "id": tweet.id,
                        "permalink": tweet.permalink}
            rows_list.append(row_dict)
        return rows_list, row_dict["date"]

    def create_tweets_dataframe(self, search_terms, final_end_date,
                                num_tweets, call_size):
        """ Performs repeated calls to collect_one_run to get num_tweets into
        a dataframe. Each call to collect_one_run takes call_size.

        call_size should be on the order of several thousand for best results

        Each time a new call to collect_one_run is made, the date is
        incremented backward by one day. This means that if seven calls are
        made and each call only takes tweets from one day, seven different
        days (starting with final_end_date and moving backward in time) will
        be sampled.

        final_end_date should be a string in form "2015-12-31"
        """
        total_row_list = []
        search_date = final_end_date
        num_tweets_searched = 0

        starttime = time.time()

        while num_tweets_searched < num_tweets:
            row_list, last_date = self.collect_one_run(search_terms,
                                                       search_date,
                                                       call_size)

            total_row_list += row_list
            search_date = last_date
            num_tweets_searched += call_size

        # once all tweets are collected, combine them all into dataframe
        column_names = ["username",
                        "date",
                        "retweets",
                        "favorites",
                        "text",
                        "mentions",
                        "hashtags",
                        "id",
                        "permalink"]

        tweets_df = pd.DataFrame(total_row_list, columns = column_names)

        print "Time to collect ",  str(num_tweets), " tweets: ", time.time() - starttime, "seconds"

        self.tweets_df = tweets_df

    """ To save the dataframe to a csv file, use the pandas method without
    the index:

    twit.tweets_df.to_csv("output.csv", index=False)

    """

    #############################################################
    # Methods to gather tweets and prune (done every time)
    #############################################################

    def get_tweets(self):
        """ Takes path to twitter data obtained with java tweet search library
        and returns a dataframe of the tweets and their accompanying
        information. Dataframe has columns for username, date, retweets,
        favorites, text, mentions, and hashtag. The dataframe is stored under
        the attribute tweets_pd.
        """
        # Read in csv file with many columns to account for people who put many
        # semicolons in tweets, then keep only the rows that don't have
        # semicolons in a tweet by dropping rows with too many columns.
        # (Semicolons are the delimeter in the java twitter search library.)
        tweets = pd.read_csv(self.data_path, sep=";",
                             names=list('abcdefghijklmno'))
        tweets = tweets[tweets.k.isnull()]

{'fraction_non_alphanumeric': 0.11580684508861304, 'fraction_numerical': 0.0021525435890076775, 'mean_word_length': 2.207825086306099, 'pattern_counts': {'":': 18, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1237103', 'n_tokens_mistral': 3713, 'n_tokens_neox': 3458, 'n_words': 1211}