<jupyter_start><jupyter_text>Getting the data<jupyter_code>def get_statistics(video_id):

    url = "https://www.googleapis.com/youtube/v3/videos"
    pms = {'key': api_key, 'id': video_id, 'part':'contentDetails,statistics'}
    res = requests.get(url, params = pms)
    data = res.json()

    return(data)
def get_channel_videos(channel_id='UC-2Y8dQb0S6DtpxNgAKoJKA'):

	url = "https://www.googleapis.com/youtube/v3/search"

	pms = {'type': 'video', 'id' : id, 'key': api_key, 'channelId': channel_id, 'part':'snippet', 'order':'viewCount','maxResults':50}

	res = requests.get(url, params = pms)

	print("Connection status: %s" % res)

	data = res.json()

	#print(data)

	lst = []

	for video in data['items']:
		video_stats = get_statistics(video['id']['videoId'])
		#print(video['snippet']['title'])
		results_json = {
			'channelTitle' : video['snippet']['channelTitle'],
			'title' : video['snippet']['title'],
			'publishedAt' : video['snippet']['publishedAt'],
			'videoId' : video['id']['videoId'],
			'viewCount' : video_stats['items'][0]['statistics']['viewCount'],
			'commentCount' : video_stats['items'][0]['statistics']['commentCount'],
			'likeCount' : video_stats['items'][0]['statistics']['likeCount'],
			'dislikeCount' : video_stats['items'][0]['statistics']['dislikeCount'],
		}

		lst.append(results_json)

	df = pd.read_json(json.dumps(lst))

	return(df)

import requests

url = 'https://www.googleapis.com/youtube/v3/videos'

pms = {'part': 'snippet, statistics', 'id' : 'YQUpg795iBo', 'key': 'xxx'}

res = requests.get(url, params = pms)

data = res.json()
url = 'https://www.googleapis.com/youtube/v3/commentThreads'

full_data = []			# return list
page = ''				# init paging

while True:

	pms = {'part': 'snippet', 'videoId' : id, 'maxResults' : 100, 'key': api_key, 'pageToken': page}
	res = requests.get(url, params = pms)

	print("Connection status: %s" % res)

	data = res.json()
	full_data.extend(data['items'])

	print("Just downloaded: %s, Total: %s" % (len(data['items']), len(full_data)))

	try:
		page = data['nextPageToken']
	except:
		break<jupyter_output><empty_output><jupyter_text>Data processing<jupyter_code>import pandas as pd

df = pd.DataFrame()

df['comments'] = [k['snippet']['topLevelComment']['snippet']['textDisplay'] for k in full_data]
df['date'] = [k['snippet']['topLevelComment']['snippet']['publishedAt'] for k in full_data]

df = df.set_index(['date'])
df.index = pd.to_datetime(df.index)
def clean(text):
    
    tokens = word_tokenize(text.strip())
    clean = [i.lower() for i in tokens]	
    clean = [i for i in clean if i not in stopwords.words('english')]
    clean = [i.strip(''.join(list(string.punctuations))) for i in clean if i not in punctuations]
    clean = [i for i in clean if len(i) > 1]

    return(" ".join(clean))
df['clean_comments'] = df['comments'].apply(clean)
from nltk.sentiment.vader import SentimentIntensityAnalyzer

sentiment = SentimentIntensityAnalyzer()

df['sentiment'] = df['clean_comments'].apply(lambda txt: sentiment.polarity_scores(txt)['compound'])<jupyter_output><empty_output><jupyter_text>Data Analysis<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import matplotlib


df['sentiment'].resample('M').mean()

matplotlib.style.use('ggplot')

df['sentiment'].resample('M').count().plot()

plt.axhline(0, color='k', lw = 2)
plt.xlabel('Date')
plt.ylabel('Number of comments')


plt.show()<jupyter_output><empty_output><jupyter_text>Sentiment by weekday<jupyter_code>df = pd.read_pickle('ps4-processed.pickle')


dx = df[df.index > '2013-11-18']	# time limit to skip first days ofcomments flood

ax = dx.groupby(dx.index.weekday)['sentiment'].mean().plot(kind = 'bar', title = 'Average sentiment by day of week')

ax.set_xticklabels(['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'])


plt.show()<jupyter_output><empty_output><jupyter_text>Comments in time<jupyter_code>df['sentiment'].resample('M').count().plot()

plt.axhline(0, color='k', lw = 2)

plt.xlabel('Date')

plt.ylabel('Number of comments')

plt.show()
dx = df[df.index > '2013-11-18']

dx['sentiment'].resample('M').count().plot()

plt.axhline(0, color='k', lw = 2)

plt.xlabel('Date')

plt.ylabel('Number of comments')

plt.show()<jupyter_output><empty_output><jupyter_text>Comments by weekday<jupyter_code>dx = df[df.index > '2013-11-15']

ax = dx.groupby(dx.index.weekday)['comments'].count().plot(kind = 'bar', title = 'Number of comments per day')

ax.set_xticklabels(['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday',

'Saturday', 'Sunday'])

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19305103148751357, 'fraction_numerical': 0.011292073832790446, 'mean_word_length': 5.874626865671642, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '474074', 'license': 'MIT', 'n_tokens_mistral': 1794, 'n_tokens_neox': 1669, 'n_words': 312, 'path': 'Chapter05/Chapter5.ipynb', 'repo_name': 'BERGASPAMUNGKAS01/python-sma', 'size': '4605'}