<jupyter_start><jupyter_text>Define tweet cleaning function<jupyter_code>def preprocess_tweet(tweet):
    
    #Initalize output string
    output = ''
    
    # Remove leading 'b' if present
    if tweet[0]=='b':
        tweet = tweet[1:]
    
    # Clean each word in tweet
    for word in tweet.split():
        
        # Remove punctuation
        word = word.strip('\'"?!,.():;..')
        
        # Convert more than 2 letter repetitions to 2 letter
        #word = re.sub(r'(.)\1+', r'\1\1', word)
        
        # Remove - & '
        word = re.sub(r'(-|\')', '', word)
        
        # Remove 2 or more dots
        #word = re.sub(r'(..)', '', word)
        
        #replace consecutive non-ASCII characters with a space
        word = re.sub(r'[^\x00-\x7F]+',' ', word)
        
        # Remove twitter handles (words with leading '@')
        if len(word)==0 or word[0] == '@' :
            word = ''
       
        # Remove words containing escape character ('\')
        if '\\' in word:
            word = ''
            
        # Remove urls (words starting with 'http')
        if word[0:4] == 'http':
            word = ''
            
        # Expand acronyms
        if word == 'wtf':
            word = 'what the fuck'
        if word == 'wth':
            word = 'what the hell'
        if word == 'wtaf':
            word = 'what the actual fuck'
        if word == 'tf':
            word = 'the fuck'
        
        # Add cleaned word to output
        output += word + ' '
        
    # Remove leading spaces
    output = output.lstrip()
        
    return output<jupyter_output><empty_output><jupyter_text>Define tweet scoring functions<jupyter_code>def compute_sentiment_score(my_tweet):
    blob = TextBlob(my_tweet)
    return blob.sentiment.polarity

def assess_sentiment(my_sentiment_score):

    # Assign sentiment bucket
    if my_sentiment_score > 0:
        sentiment = 'Positive'
    elif my_sentiment_score == 0.0:
        sentiment = 'Neutral'
    else:
        sentiment = 'Negative'
    
    return sentiment<jupyter_output><empty_output><jupyter_text>Import tweets and do basic cleaning/parsing<jupyter_code># Define candidate list
candidates = ['bennet', 'biden', 'booker', 'buttigieg', 'castro', 
              'delaney', 'gabbard', 'gillibrand', 'gravel',
              'harris', 'hickenlooper', 'inslee', 'klobuchar',
              'messam', 'moulton', 'ryan', 'sanders', 'swalwell',
             'trump', 'warren', 'weld', 'williamson', 'yang'] 

# Iterate through candidate list loading data and applying parsing
data = pd.DataFrame()
for candidate in candidates:
    
    # Read in data
    file_name = f'./Resources/{candidate}.csv'
    raw_data = pd.read_csv(file_name)
    print(f'Loaded {file_name}.  {len(raw_data)} records loaded.')

    # Drop unneeded columns
    df = raw_data.drop(columns = ['User Name', 'Retweeted'])

    # Add candidate name
    df['Candidate'] = candidate

    # For Date column, convert string to normalized datetime
    df['Date'] = pd.to_datetime(df['Date'])
    df['Date'] = df['Date'].apply(lambda x: x.normalize())
    
    # Append candidate-specific, parsed records into 'data' which holds tweets from all candidates
    data = pd.concat([data, df])

print(f'Completed loading data for all candidates.  {len(data)} total records.')

# Limit to target_date and inspect counts to verify each candidate has tweets on that date.
target_date = '2019-05-02'
data = data.loc[data['Date'] == target_date]
print(f'\nLimiting to {target_date}.  {len(data)} records remaining.\n')
      
print('Inspect counts by candidate and date to validate:')
data.groupby(['Candidate', 'Date']).count()<jupyter_output>Loaded ./Resources/bennet.csv.  3013 records loaded.
Loaded ./Resources/biden.csv.  11999 records loaded.
Loaded ./Resources/booker.csv.  6443 records loaded.
Loaded ./Resources/buttigieg.csv.  12000 records loaded.
Loaded ./Resources/castro.csv.  1749 records loaded.
Loaded ./Resources/delaney.csv.  704 records loaded.
Loaded ./Resources/gabbard.csv.  5598 records loaded.
Loaded ./Resources/gillibrand.csv.  1836 records loaded.
Loaded ./Resources/gravel.csv.  1092 records loaded.
Loaded ./Resources/harris.csv.  12000 records loaded.
Loaded ./Resources/hickenlooper.csv.  743 records loaded.
Loaded ./Resources/inslee.csv.  12000 records loaded.
Loaded ./Resources/klobuchar.csv.  1428 records loaded.
Loaded ./Resources/messam.csv.  172 records loaded.
Loaded ./Resources/moulton.csv.  1150 records loaded.
Loaded ./Resources/ryan.csv.  1277 records loaded.
Loaded ./Resources/sanders.csv.  12000 records loaded.
Loaded ./Resources/swalwell.csv.  4128 records loaded.
Loaded ./Resources/trump.[...]<jupyter_text>Compute sentiment scores<jupyter_code># Apply cleaning function to tweets
data['Cleaned Tweet Text'] = data['Tweet Text'].apply(preprocess_tweet)

# Compute score for each tweet
data['Sentiment Score'] = data['Cleaned Tweet Text'].apply(compute_sentiment_score)

# Compute sentiment bucket for each tweet
data['Sentiment Bucket'] = data['Sentiment Score'].apply(assess_sentiment)

# Save to file
data.to_csv('all_candidates.csv')

data.head()<jupyter_output><empty_output><jupyter_text>Inspect various summary views and statistics to validate<jupyter_code># Compute average sentiment for each candidate
average_sentiment_df = pd.DataFrame(data.groupby(['Candidate']).mean()['Sentiment Score'])
average_sentiment_df.sort_values(by = ['Sentiment Score'], ascending = False)
# Count tweets in each sentiment bucket
snowman_counts = pd.DataFrame(data.groupby(['Candidate', 'Sentiment Bucket']).count()['Sentiment Score'])
snowman_mean = pd.DataFrame(data.groupby(['Candidate', 'Sentiment Bucket']).mean())

snowman_plot = pd.DataFrame()
snowman_plot['Count of Tweets'] = snowman_counts['Sentiment Score']
snowman_plot['Average Sentiment Score'] = snowman_mean['Sentiment Score']
snowman_plot<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12877127928535312, 'fraction_numerical': 0.016854879487611663, 'mean_word_length': 3.1409630146545706, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '606918', 'license': 'MIT', 'n_tokens_mistral': 1920, 'n_tokens_neox': 1821, 'n_words': 491, 'path': 'tweet_sentiment_analysis/compute-sentiment-scores.ipynb', 'repo_name': 'elanderos318/Tweetocracy', 'size': '5933'}