<jupyter_start><jupyter_text>Gather:<jupyter_code>folder_name= 'files'
if not os.path.exists(folder_name):
    os.makedirs(folder_name)
url ='https://d17h27t6h515a5.cloudfront.net/topher/2017/August/59a4e958_twitter-archive-enhanced/twitter-archive-enhanced.csv'
response= requests.get(url)
with open(folder_name+"/twitter-archive-enhanced.csv", mode="wb") as file:
    file.write(response.content)
twitter_archive= pd.read_csv(folder_name+'/twitter-archive-enhanced.csv')
twitter_archive.head()
url1="https://d17h27t6h515a5.cloudfront.net/topher/2017/August/599fd2ad_image-predictions/image-predictions.tsv"
response=requests.get(url1)
with open(folder_name+"/image-predictions.tsv",mode='wb') as file:
    file.write(response.content)
image_predictions = pd.read_csv(folder_name+"/image-predictions.tsv",sep='\t')
image_predictions.head()
consumer_key = ''
consumer_secret = ''
access_token = ''
access_secret = ''

auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify = True)
tweet_ids = twitter_archive.tweet_id.values
len(tweet_ids)
fails_dict = {}
count = 0 
start = timer()
with open(folder_name+'/tweet_json.txt', 'w', encoding='utf8') as f:
    for tweet_id in twitter_archive['tweet_id']:
        count += 1
        print(str(count) + ": " + str(tweet_id))
        try:
            tweet = api.get_status(tweet_id, tweet_mode='extended')
            print('success')
            json.dump(tweet._json, f)
            f.write('\n')
            
        except tweepy.TweepError as e:
            print("Fail")
            fails_dict[tweet_id] = e
            continue
end = timer()
print(end - start)
print(fails_dict)
# For loop to append each tweet into a list
tweets_data = []

with open(folder_name+'/tweet_json.txt', "r") as tweet_file:
    for line in tweet_file:
        try:
            tweet = json.loads(line)
            tweets_data.append(tweet)
        except:
            continue
        
# Create tweet_info DataFrame
tweet_info = pd.DataFrame()
# Add selected variables to tweet_info DataFrame
tweet_info['tweet_id'] = list(map(lambda tweet: tweet['id'], tweets_data))
tweet_info['retweet_count'] = list(map(lambda tweet: tweet['retweet_count'], tweets_data))
tweet_info['favorite_count'] = list(map(lambda tweet: tweet['favorite_count'], tweets_data))<jupyter_output><empty_output><jupyter_text>ASSESS:<jupyter_code>twitter_archive_assess = twitter_archive.copy()
image_predictions_assess = image_predictions.copy()
tweet_info_assess = tweet_info.copy()
# View first 10 rows of twitter_archive DataFrame
twitter_archive_assess.head(10)
# View last 10 rows of twitter_archive DataFrame
twitter_archive_assess.tail(10)
# View info of twitter_archive DataFrame
twitter_archive_assess.info()
# View descriptive statistics of twitter_archive DataFrame
twitter_archive_assess.describe()
# View number of entries for each source
twitter_archive_assess.source.value_counts()
# View first 5 rows of image_predictions DataFrame
image_predictions_assess.head()
# View last 5 rows of image_predictions DataFrame
image_predictions_assess.tail()
# View info of image_predictions DataFrame
image_predictions_assess.info()
# View descriptive statistics of image_predictions DataFrame
image_predictions_assess.describe()
# View first 5 rows of tweet_info DataFrame
tweet_info_assess.head(10)
# View last 5 rows of tweet_info DataFrame
tweet_info_assess.tail(10)
# View info of tweet_info DataFrame
tweet_info_assess.info()
# View descriptive statistics of tweet_info DataFrame
tweet_info_assess.describe()<jupyter_output><empty_output><jupyter_text>Tidiness- Join 'tweet_info' and 'image_predictions' Dataframes to 'twitter_archive' DataFrame- Combine  the four columns: doggo, floofer, pupper, puppo into one column 'dog_stage'  Quality- Remove rows where there are no images- Remove 'retweeted_status_id', 'retweeted_status_user_id', 'retweeted_status_timestamp'- Change incorrect dog names.- Missing values in 'name' and dog stages showing as 'None'- Rating numerators with decimals not showing full float- Tweet with more than one / sometimes have the first occurence erroneously used for the rating numerators and denominators- Tweet ID 810984652412424192 doesn't contain a rating- Extra characters after '&'- Change sources to more readable categories.- Erroneous datatypes (timestamp, source, dog stages, tweet_id, in_reply_to_status_id, in_reply_to_user_id)  Clean:  Tidiness:<jupyter_code># Create copies of original DataFrames to work off of
twitter_archive_clean = twitter_archive.copy()
image_predictions_clean = image_predictions.copy()
tweet_info_clean = tweet_info.copy()<jupyter_output><empty_output><jupyter_text>Define: Create dog stage variable and remove the individual dog stage columns.  Code:<jupyter_code># Create 'dog_stage' variable which is made by extracting the dog stage variables from the text column when available 
twitter_archive_clean['dog_stage'] = twitter_archive_clean['text'].str.extract('(puppo|pupper|floofer|doggo)', expand=True)
# Create variable of columns that are no longer needed and drop them from the DataFrame 
twitter_archive_clean = twitter_archive_clean.drop(['doggo', 'floofer', 'pupper', 'puppo'], axis=1)<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean.head()<jupyter_output><empty_output><jupyter_text>Define: Merge tweet_info and image_predictions to twitter_archive table.  Code:<jupyter_code>twitter_archive_clean = pd.merge(twitter_archive_clean,tweet_info_clean, on=['tweet_id','tweet_id'], how='inner')
twitter_archive_clean = twitter_archive_clean.merge(image_predictions_clean, on='tweet_id', how='inner')<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 2067 entries, 0 to 2066
Data columns (total 27 columns):
tweet_id                      2067 non-null int64
in_reply_to_status_id         23 non-null float64
in_reply_to_user_id           23 non-null float64
timestamp                     2067 non-null object
source                        2067 non-null object
text                          2067 non-null object
retweeted_status_id           75 non-null float64
retweeted_status_user_id      75 non-null float64
retweeted_status_timestamp    75 non-null object
expanded_urls                 2067 non-null object
rating_numerator              2067 non-null int64
rating_denominator            2067 non-null int64
name                          2067 non-null object
dog_stage                     338 non-null object
retweet_count                 2067 non-null int64
favorite_count                2067 non-null int64
jpg_url                       2067 non-null object
img_num                       2067 non[...]<jupyter_text>Quality:  Define: Remove rows where there are no images (expanded_urls).  Code:<jupyter_code>twitter_archive_clean = twitter_archive_clean.dropna(subset=['expanded_urls'])<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>sum(twitter_archive_clean['expanded_urls'].isnull())
twitter_archive_clean.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 2067 entries, 0 to 2066
Data columns (total 27 columns):
tweet_id                      2067 non-null int64
in_reply_to_status_id         23 non-null float64
in_reply_to_user_id           23 non-null float64
timestamp                     2067 non-null object
source                        2067 non-null object
text                          2067 non-null object
retweeted_status_id           75 non-null float64
retweeted_status_user_id      75 non-null float64
retweeted_status_timestamp    75 non-null object
expanded_urls                 2067 non-null object
rating_numerator              2067 non-null int64
rating_denominator            2067 non-null int64
name                          2067 non-null object
dog_stage                     338 non-null object
retweet_count                 2067 non-null int64
favorite_count                2067 non-null int64
jpg_url                       2067 non-null object
img_num                       2067 non[...]<jupyter_text>Define: Remove retweets.  Code:<jupyter_code>len(twitter_archive_clean['retweeted_status_id'].isnull())
# Select rows where 'retweeted_status_id' is null to save to twitter_archive_clean
twitter_archive_clean = twitter_archive_clean[twitter_archive_clean['retweeted_status_id'].isnull()]<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1992 entries, 0 to 2066
Data columns (total 27 columns):
tweet_id                      1992 non-null int64
in_reply_to_status_id         23 non-null float64
in_reply_to_user_id           23 non-null float64
timestamp                     1992 non-null object
source                        1992 non-null object
text                          1992 non-null object
retweeted_status_id           0 non-null float64
retweeted_status_user_id      0 non-null float64
retweeted_status_timestamp    0 non-null object
expanded_urls                 1992 non-null object
rating_numerator              1992 non-null int64
rating_denominator            1992 non-null int64
name                          1992 non-null object
dog_stage                     326 non-null object
retweet_count                 1992 non-null int64
favorite_count                1992 non-null int64
jpg_url                       1992 non-null object
img_num                       1992 non-nu[...]<jupyter_text>Define: Remove retweeted columns.  Code:<jupyter_code># drop columns related to retweets
twitter_archive_clean = twitter_archive_clean.drop(['retweeted_status_id', 'retweeted_status_user_id', 'retweeted_status_timestamp'], axis=1)<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1992 entries, 0 to 2066
Data columns (total 24 columns):
tweet_id                 1992 non-null int64
in_reply_to_status_id    23 non-null float64
in_reply_to_user_id      23 non-null float64
timestamp                1992 non-null object
source                   1992 non-null object
text                     1992 non-null object
expanded_urls            1992 non-null object
rating_numerator         1992 non-null int64
rating_denominator       1992 non-null int64
name                     1992 non-null object
dog_stage                326 non-null object
retweet_count            1992 non-null int64
favorite_count           1992 non-null int64
jpg_url                  1992 non-null object
img_num                  1992 non-null int64
p1                       1992 non-null object
p1_conf                  1992 non-null float64
p1_dog                   1992 non-null bool
p2                       1992 non-null object
p2_conf                  1992[...]<jupyter_text>Define: Display full content of 'text' column.  Code:<jupyter_code># to increase the visibility of columns and set column width to infinite so entire content of 'text' column is displayed
pd.set_option("display.max_columns",24)
pd.set_option("display.max_colwidth",-1)<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean.sample(5)<jupyter_output><empty_output><jupyter_text>Define: Change incorrect dog names.  Code:<jupyter_code># Save locations where 'name' column is lowercase, lowercase and 'text' column contains 'named' and lowercase and 'text'
# column contains the words 'name is'
named_to_replace = twitter_archive_clean.loc[(twitter_archive_clean['name'].str.islower()) & (twitter_archive_clean['text'].str.contains('named'))]
name_is_to_replace = twitter_archive_clean.loc[(twitter_archive_clean['name'].str.islower()) & (twitter_archive_clean['text'].str.contains('name is'))]
not_named_to_replace = twitter_archive_clean.loc[(twitter_archive_clean['name'].str.islower())]
# Save these locations as lists
named_to_replace_list = named_to_replace['text'].tolist()
name_is_to_replace_list = name_is_to_replace['text'].tolist()
not_named_to_replace_list = not_named_to_replace['text'].tolist()<jupyter_output><empty_output><jupyter_text>if var2 == var3:    var1 = Trueelse:    var1 = False<jupyter_code># For loop to iterate through locations where name is lowercase and the words 'named' appear in 'text' and set the 'name' 
# value to be the word that appears after 'named'
for entry in named_to_replace_list:
    mask = twitter_archive_clean.text == entry
    name_column = 'name'
    twitter_archive_clean.loc[mask, name_column] = re.findall(r"named\s(\w+)", entry)

# For loop to iterate through locations where name is lowercase and the words 'name is' appear in 'text' and set the 'name' 
# value to be the word that appears after 'name is'    
for entry in name_is_to_replace_list:
    mask = twitter_archive_clean.text == entry
    name_column = 'name'
    twitter_archive_clean.loc[mask, name_column] = re.findall(r"name is\s(\w+)", entry)    

# For loop to iterate through locations where name is lowercase and replace the name value with the word "None"
for entry in not_named_to_replace_list:
    mask = twitter_archive_clean.text == entry
    name_column = 'name'
    twitter_archive_clean.loc[mask, name_column] = "None"
# Replace the occurence of "O" with "O'Malley"
twitter_archive_clean.name = twitter_archive_clean.name.replace("O", "O'Malley")<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean.name.sort_values()
twitter_archive_clean.loc[(twitter_archive_clean.name.str.islower())]
twitter_archive_clean[twitter_archive_clean.name == "O'Malley"]<jupyter_output><empty_output><jupyter_text>Define: Change missing values in 'name' from 'None' to NaN (dog stages already covered).  Code:<jupyter_code>twitter_archive_clean.name = twitter_archive_clean.name.replace('None', 'NaN')<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1992 entries, 0 to 2066
Data columns (total 24 columns):
tweet_id                 1992 non-null int64
in_reply_to_status_id    23 non-null float64
in_reply_to_user_id      23 non-null float64
timestamp                1992 non-null object
source                   1992 non-null object
text                     1992 non-null object
expanded_urls            1992 non-null object
rating_numerator         1992 non-null int64
rating_denominator       1992 non-null int64
name                     1992 non-null object
dog_stage                326 non-null object
retweet_count            1992 non-null int64
favorite_count           1992 non-null int64
jpg_url                  1992 non-null object
img_num                  1992 non-null int64
p1                       1992 non-null object
p1_conf                  1992 non-null float64
p1_dog                   1992 non-null bool
p2                       1992 non-null object
p2_conf                  1992[...]<jupyter_text>Define: Change sources to more readable categories.  Code:<jupyter_code># Remove url from sources
twitter_archive_clean.source = twitter_archive_clean.source.str.replace('<a href="http://twitter.com/download/iphone" rel="nofollow">Twitter for iPhone</a>', 'Twitter for iPhone')
twitter_archive_clean.source = twitter_archive_clean.source.str.replace('<a href="http://vine.co" rel="nofollow">Vine - Make a Scene</a>', 'Vine')
twitter_archive_clean.source = twitter_archive_clean.source.str.replace('<a href="http://twitter.com" rel="nofollow">Twitter Web Client</a>', 'Twitter Web Client')
twitter_archive_clean.source = twitter_archive_clean.source.str.replace('<a href="https://about.twitter.com/products/tweetdeck" rel="nofollow">TweetDeck</a>', 'TweetDeck')
# Change datatype to category
twitter_archive_clean.source = twitter_archive_clean.source.astype('category')<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean.source.value_counts()
twitter_archive_clean.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1992 entries, 0 to 2066
Data columns (total 24 columns):
tweet_id                 1992 non-null int64
in_reply_to_status_id    23 non-null float64
in_reply_to_user_id      23 non-null float64
timestamp                1992 non-null object
source                   1992 non-null category
text                     1992 non-null object
expanded_urls            1992 non-null object
rating_numerator         1992 non-null int64
rating_denominator       1992 non-null int64
name                     1992 non-null object
dog_stage                326 non-null object
retweet_count            1992 non-null int64
favorite_count           1992 non-null int64
jpg_url                  1992 non-null object
img_num                  1992 non-null int64
p1                       1992 non-null object
p1_conf                  1992 non-null float64
p1_dog                   1992 non-null bool
p2                       1992 non-null object
p2_conf                  19[...]<jupyter_text>Define: Change datatypes of timestamp to datetime, dog_stage to categorical, and tweet_id, in_reply_to_status_id, and in_reply_to_user_id to strings.  Code:<jupyter_code>twitter_archive_clean.dog_stage = twitter_archive_clean.dog_stage.astype('category')
twitter_archive_clean.timestamp = pd.to_datetime(twitter_archive_clean['timestamp'])
twitter_archive_clean.tweet_id = twitter_archive_clean.tweet_id.astype('str')
twitter_archive_clean.in_reply_to_status_id = twitter_archive_clean.in_reply_to_status_id.astype('str')
twitter_archive_clean.in_reply_to_user_id = twitter_archive_clean.in_reply_to_user_id.astype('str')<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1992 entries, 0 to 2066
Data columns (total 24 columns):
tweet_id                 1992 non-null object
in_reply_to_status_id    1992 non-null object
in_reply_to_user_id      1992 non-null object
timestamp                1992 non-null datetime64[ns]
source                   1992 non-null category
text                     1992 non-null object
expanded_urls            1992 non-null object
rating_numerator         1992 non-null int64
rating_denominator       1992 non-null int64
name                     1992 non-null object
dog_stage                326 non-null category
retweet_count            1992 non-null int64
favorite_count           1992 non-null int64
jpg_url                  1992 non-null object
img_num                  1992 non-null int64
p1                       1992 non-null object
p1_conf                  1992 non-null float64
p1_dog                   1992 non-null bool
p2                       1992 non-null object
p2_conf       [...]<jupyter_text>Define: Remove tweet without rating.  Code:<jupyter_code>twitter_archive_clean = twitter_archive_clean[twitter_archive_clean.tweet_id != 810984652412424192]<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean[twitter_archive_clean.tweet_id == 810984652412424192]<jupyter_output><empty_output><jupyter_text>Define: Fix rating numerator and denominators that are not actually ratings and removing the duplicate of tweet_id 740373189193256964.  Code:<jupyter_code># View all occurences where there are more than one #/# in 'text' column and disable warnings
warnings.simplefilter('ignore')
twitter_archive_clean[twitter_archive_clean['text'].str.contains( r"(\d+\.?\d*\/\d+\.?\d*\D+\d+\.?\d*\/\d+\.?\d*)")]
# Save the text where the rating numerator and denomiq1nators were incorrectly extracted
ratings_to_fix =['This is Bookstore and Seaweed. Bookstore is tired and Seaweed is an asshole. 10/10 and 7/10 respectively https://t.co/eUGjGjjFVJ',
                 'After so many requests, this is Bretagne. She was the last surviving 9/11 search dog, and our second ever 14/10. RIP https://t.co/XAVDNDaVgQ',
                 'Happy 4/20 from the squad! 13/10 for all https://t.co/eV1diwds8a',
                 'This is Bluebert. He just saw that both #FinalFur match ups are split 50/50. Amazed af. 11/10 https://t.co/Kky1DPG4iq',
                 'This is Socks. That water pup w the super legs just splashed him. Socks did not appreciate that. 9/10 and 2/10 https://t.co/8rc5I22bBf',
                'Meet Fynn &amp; Taco. Fynn is an all-powerful leaf lord and Taco is in the wrong place at the wrong time. 11/10 &amp; 10/10 https://t.co/MuqHPvtL8c',
                'This is Darrel. He just robbed a 7/11 and is in a high speed police chase. Was just spotted by the helicopter 10/10 https://t.co/7EsP8LmSp5',
                'Meet Tassy &amp; Bee. Tassy is pretty chill, but Bee is convinced the Ruffles are haunted. 10/10 &amp; 11/10 respectively https://t.co/fgORpmTN9C',
                'These two pups just met and have instantly bonded. Spectacular scene. Mesmerizing af. 10/10 and 7/10 for blue dog https://t.co/gwryaJO4tC',
                'Meet Rufio. He is unaware of the pink legless pupper wrapped around him. Might want to get that checked 10/10 &amp; 4/10 https://t.co/KNfLnYPmYh',
                'Two gorgeous dogs here. Little waddling dog is a rebel. Refuses to look at camera. Must be a preteen. 5/10 &amp; 8/10 https://t.co/YPfw7oahbD',
                'Meet Eve. She"'"s a raging alcoholic 8/10 (would b 11/10 but pupper alcoholism is a tragic issue that I can"'"t condone) https://t.co/U36HYQIijg',
                '10/10 for dog. 7/10 for cat. 12/10 for human. Much skill. Would pet all https://t.co/uhx5gfpx5k',
                 'Meet Holly. She"'"s trying to teach small human-like pup about blocks but he"'"s not paying attention smh. 11/10 &amp; 8/10 https://t.co/RcksaUrGNu',
                 "This is an Albanian 3 1/2 legged Episcopalian. Loves well-polished hardwood flooring. Penis on the collar. 9/10 https://t.co/d9NcXFKwLv"
                 
                ]
# Loop through the list of ratings to fix and extract the second occurence of #/ to save as the rating numerator. As all the
# occurences of the actual ratings in the ratings to fix list have a denominator of 10, we will set that value for each 
#entry instead of extracting it.
for entry in ratings_to_fix:
    mask = twitter_archive_clean.text == entry
    column_name1 = 'rating_numerator'
    column_name2 = 'rating_denominator'
    twitter_archive_clean.loc[mask, column_name1] = re.findall(r"\d+\.?\d*\/\d+\.?\d*\D+(\d+\.?\d*)\/\d+\.?\d*", entry)
    twitter_archive_clean.loc[mask, column_name2] = 10<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean[twitter_archive_clean['text'].str.contains( r"(\d+\.?\d*\/\d+\.?\d*\D+\d+\.?\d*\/\d+\.?\d*)")]
twitter_archive_clean.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1992 entries, 0 to 2066
Data columns (total 24 columns):
tweet_id                 1992 non-null object
in_reply_to_status_id    1992 non-null object
in_reply_to_user_id      1992 non-null object
timestamp                1992 non-null datetime64[ns]
source                   1992 non-null category
text                     1992 non-null object
expanded_urls            1992 non-null object
rating_numerator         1992 non-null object
rating_denominator       1992 non-null int64
name                     1992 non-null object
dog_stage                326 non-null category
retweet_count            1992 non-null int64
favorite_count           1992 non-null int64
jpg_url                  1992 non-null object
img_num                  1992 non-null int64
p1                       1992 non-null object
p1_conf                  1992 non-null float64
p1_dog                   1992 non-null bool
p2                       1992 non-null object
p2_conf      [...]<jupyter_text>Define: Remove extra characters after '&' in twitter_archive_clean['text'].  Code:<jupyter_code>twitter_archive_clean['text'] = twitter_archive_clean['text'].str.replace('&amp;', '&')<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean[twitter_archive_clean.text.str.contains('&amp;')]<jupyter_output><empty_output><jupyter_text>Define: Fix rating numerator that have decimals.  Code:<jupyter_code># View tweets with decimals in rating in 'text' column
twitter_archive_clean[twitter_archive_clean.text.str.contains(r"(\d+\.\d*\/\d+)")]
# Set correct numerators for specific tweets
twitter_archive_clean.loc[40, ['rating_numerator']]=13.5
twitter_archive_clean.loc[553, ['rating_numerator']]=9.75
twitter_archive_clean.loc[609, ['rating_numerator']]=11.27
twitter_archive_clean.loc[1446, ['rating_numerator']]=11.26
# Change datatype of rating_numerator and denominator to float
twitter_archive_clean['rating_numerator'] = twitter_archive_clean['rating_numerator'].astype('float')
twitter_archive_clean['rating_denominator'] = twitter_archive_clean['rating_denominator'].astype('float')<jupyter_output><empty_output><jupyter_text>Test:<jupyter_code>twitter_archive_clean[twitter_archive_clean.text.str.contains(r"(\d+\.\d*\/\d+)")]<jupyter_output><empty_output><jupyter_text>Store:<jupyter_code># Save the clean DataFrame to folder files
twitter_archive_clean.to_csv('files/twitter_archive_master.csv')<jupyter_output><empty_output><jupyter_text>Analyze:<jupyter_code># using specific columns for analysis
df = twitter_archive_clean[['timestamp', 'retweet_count', 'favorite_count', 'rating_numerator', 'rating_denominator']].copy()
df.set_index('timestamp', inplace=True)
sns.distplot(df.favorite_count, color = 'green')
sns.distplot(df.retweet_count, color = 'red')
plt.xlabel('favorite_count/retweet_count')
plt.ylabel('Count')
plt.title('Retweets and favorites counts')
plt.savefig('files/retweets_favorites_count.png')
plt.show()

sns.distplot(df.favorite_count.apply(np.log), color = 'green', axlabel='Favorite Count (Normalized)')
plt.title('log scale of favorites counts')
plt.savefig('files/log_favorite_count.png')
sns.distplot(df.retweet_count.apply(np.log), color = 'blue',axlabel='Retweet Count (Normalized)')
plt.title('log scale of Retweets counts')
plt.savefig('files/log_retweet_count.png')
#plotting Retweets and favorites over time
df.retweet_count.plot(color = 'red', label='Retweets')
df.favorite_count.plot(color = 'blue', label='Favorites')
plt.style.use('seaborn-darkgrid')
plt.legend(loc='upper left')
plt.xlabel('Tweet timestamp')
plt.ylabel('Count')
plt.title('Retweets and favorites over time')
plt.savefig('files/retweets_favorites.png')
plt.show()
#will give the relation between columns
df.corr()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09128802102891476, 'fraction_numerical': 0.04202027788208787, 'mean_word_length': 3.6290630975143405, 'pattern_counts': {'":': 1, '<': 145, '<?xml version=': 0, '>': 145, 'https://': 18, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '34', 'dir': 'jupyter-structured-clean-dedup', 'id': '93436', 'license': 'MIT', 'n_tokens_mistral': 9154, 'n_tokens_neox': 8176, 'n_words': 2038, 'path': 'Data_Wrangling_WeRatedogs/wrangle_act.ipynb', 'repo_name': 'mvenkata11/Udacity_Projects', 'size': '26630'}