<jupyter_start><jupyter_text>YouTube Dislike Button Remains, with Number of Dislikes Hidden from Public* On November 10, 2021, YouTube announced that video **dislike counts** will be hidden from the public.  The stated benefit is protecting creators (especially smaller ones) from dislike attacks and harassment.* The number of dislikes is only available in Studio to content creators.* Announcement and Comments: https://support.google.com/youtube/thread/134791097/update-to-youtube-dislike-counts?hl=en<jupyter_code>import requests
from lxml import html
import re
import numpy as np
import statistics

import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

#NLP fun
from nltk import tokenize
from flair.models import TextClassifier
from flair.data import Sentence
flair = TextClassifier.load('en-sentiment')

#Monitor apply progress.
from ipywidgets import FloatProgress
from tqdm.notebook import tqdm
tqdm.pandas()

#DataFrame format
pd.options.display.max_rows = 500
pd.options.display.max_columns = 500
#How flair (distilbert) scores a sentence.
sent = Sentence('This change is desireable.')
flair.predict(sent)
sent.labels[0].to_dict()
#We will insert a minus sign for NEGATIVE sentiment.
sent = Sentence('This change is not desireable.')
flair.predict(sent)
mydict = sent.labels[0].to_dict()
mydict['confidence'] = - mydict['confidence'] #flip to negative number.
mydict
#Helper functions

def clean_dt(date_string):
    """Convert date to dd/mm/yy format."""    
    if pd.isna(date_string):
        return date_string
    
    result = re.sub('6 days', '11/16/21',date_string)
    result = re.sub('5 days', '11/17/21', result)
    result = re.sub('4 days', '11/18/21', result)
    result = re.sub('3 days', '11/19/21', result)
    result = re.sub('2 days', '11/20/21', result)
    result = re.sub('1 day', '11/21/21', result)
    result = re.sub('(\d+\shr|\d+\smin)', '11/22/21', result)
    return result

def clean_comm(raw):
    """ Remove hyperlinks and markup """
    
    if pd.isna(raw):
        return raw
    result = re.sub("<[a][^>]*>(.+?)</[a]>", 'Link.', raw)
    result = re.sub("\n", '', result)
    result = re.sub("\.,", '. ', result)
    result = re.sub("\xa0,?", ' ', result)
    return result

def make_sentences(text):
    """ Split text into list of sentences """
    if pd.isna(text):
        return 0    
    sentences = tokenize.sent_tokenize(text)
    return sentences

def get_flair_scores(sentences):
    """ Call predict on every sentence of a text """
    flair_scores = []
    try:        
        for i in range(0, len(sentences)): 
            flair_scores.append(predict_flair(sentences[i]))
    except:
        return 0
    return flair_scores

def predict_flair(sentence):
    """ Predict the sentiment of a sentence """
    if sentence == "":
        return 0
    text = Sentence(sentence)    
    flair.predict(text)
    value = text.labels[0].to_dict()['value'] #flair prediction is method on text.
    if value == 'POSITIVE':
        result = text.to_dict()['labels'][0]['confidence']
    else:
        result = -(text.to_dict()['labels'][0]['confidence'])
    return round(result, 3)

def get_average(scores):
    try:
        result = round(statistics.mean(scores), 3)
    except:
        return 0
    return result

def get_histogram(numpy_array: np.array, log_scale_on: bool, num_bins: int):
    '''Do sanity check'''
    from matplotlib.ticker import MaxNLocator
    fig, ax1 = plt.subplots(1,1,figsize=(12,6))       
    ax1.axvline(x=np.mean(numpy_array), color = 'green', ls='--', alpha=1,  linewidth=3)    
    ax1.axvline(x=np.median(numpy_array), color = 'orange', ls='--', alpha=1,  linewidth=3)
    #ax1.axvline(x=np.min(numpy_array), color = 'black', ls='--', alpha=1, linewidth=3)
    #ax1.axvline(x=np.max(numpy_array), color = 'black', ls='--', alpha=1,  linewidth=3)
    #ax1.set(xlim=(-1.1, 1.1))
    
    ax1.set_title(f'Histogram stats:\n median (orange dashes): {np.round(np.median(numpy_array),3)}; mean (green dashes): {np.round(np.mean(numpy_array),3)}; \nmin: {np.round(np.min(numpy_array),3)}; max:{np.round(np.max(numpy_array),3)}')
    ax1.yaxis.set_major_locator(MaxNLocator(integer=True))
    sns.histplot(numpy_array, ax=ax1, kde=False, log_scale=log_scale_on, bins=num_bins)
    plt.show()
#Open file and read into parser.  With statement automatically closes file.
with open('update_to_youtube_dislike_counts.html', 'r', encoding='UTF-8') as f:
    parser = html.fromstring(f.read())  #f.read() is a string containing the html page.
#Get comment count.
comment = '//div[@class="thread-all-replies__message"]'
qry_result = parser.xpath(comment)
tot_comments = len(qry_result)
print('Total Comments:', tot_comments, '\n')
#Save comments to list "l".
comment_num = 1
l = []
dates = []
for comment in range(1, tot_comments+1):
    commentText = f'//div[@class="thread-all-replies__message"]\
    [{comment_num}]//div[@class="thread-message__payload"]/descendant-or-self::*/text()'  #https://stackoverflow.com/questions/10413649/xpath-to-get-all-the-childrens-text/58616182
    
    commentDate = f'//div[@class="thread-all-replies__message"][{comment_num}]//div[@class="thread-message-header__right-block"]/time/text()'    
    
    comment_i_list = parser.xpath(commentText)
    comment_i_date = parser.xpath(commentDate)[0]
    l.append(",".join(comment_i_list))  #Join collapses list of strings into one string.  Append the string to the list.
    dates.append(comment_i_date)
    comment_num += 1
    
#Load comments and dates into a Pandas dataframe.    
pd.set_option('max_colwidth', 500)
df = pd.DataFrame(l, columns=['comment'])
df.insert(0, 'date', dates, allow_duplicates=True)
df.insert(1, 'clean_date', df.date.apply(clean_dt), allow_duplicates=True)
df['clean_date'] = pd.to_datetime(df['clean_date'])
df.head(2)
df['clean_comment'] = df.comment.progress_apply(clean_comm)
df['sentences'] = df.clean_comment.progress_apply(make_sentences)
df['flair_sco'] = df.sentences.progress_apply(get_flair_scores)
df['flair_sco_avg'] = df.flair_sco.progress_apply(get_average)
df.loc[80:81]
#Explore comment sentiment.  Type in a comment index number below.
comment_index = 80

comment_i = df.loc[comment_index,'sentences']
comment_i_sco = get_flair_scores(df.loc[comment_index,'sentences'])
comment_i_sco_avg = round(statistics.mean(comment_i_sco),3)

print(f"### Comment at index {comment_index} has {len(comment_i_sco)} sentences.  The mean flair score is {comment_i_sco_avg} ### \n")
cnt=1
for element in zip(comment_i, comment_i_sco):
    print(f'S{cnt}:', element)
    cnt+=1

#Plot histogram    
from matplotlib.ticker import MaxNLocator
fig, ax_1 = plt.subplots(1,1,figsize=(12,6))
median_sco = statistics.median(comment_i_sco)
ax_1.axvline(x=median_sco, color = 'orange', ls='--', alpha=1,  linewidth=2)
ax_1.set_title(f'Sentiment: Comment with {len(comment_i_sco)} Sentences\n median sentiment (orange dashes): {np.round(median_sco,3)}')
ax_1.set_xlabel('Sentiment')
ax_1.yaxis.set_major_locator(MaxNLocator(integer=True))
sns.histplot(comment_i_sco, ax=ax_1, kde=False, bins=25)

#label each bar in histogram
for p in ax_1.patches:
    height = p.get_height()
    # adding text to each bar
    if height == 0:
        pass
    else:
        ax_1.text(x = p.get_x()+(p.get_width()/2), y = height-0.2, s=height, ha = 'center', fontsize='large', fontweight='bold')

plt.show()    
#Histogram of mean sentiment for 613 comments.  Means sentiment is the average sentiment of the sentences in a comment.
get_histogram(df.flair_sco_avg.values, log_scale_on=False, num_bins=50)
def plot_boxplot_w_data(colname, df, figsize=(15,3), title='Boxplot', xlimit=1):
    '''Returns boxplot with orange data points overlaid.'''
    fig, ax = plt.subplots(1,1,figsize=figsize)
    ax.set(xlim = (-1,xlimit))
    ax=sns.boxplot(x=colname, data=df, linewidth=3, fliersize=6)
    ax=sns.stripplot(x=colname, data=df, color="orange", jitter=0.2, size=3)  #Credit: https://python-graph-gallery.com/39-hidden-data-under-boxplot/
    ax.set_title(f'{title}')    
    plt.show()
    
plot_boxplot_w_data('flair_sco_avg', df, title='Boxplot: Mean comment sentiment')    
#In theory, these comments should be neutral on average.  
#In practice, we see the sentiment of a sentence (in a given comment) may be inaccurate.
filt = (df['flair_sco_avg'] > -0.01) & (df['flair_sco_avg'] < 0.01)
df.loc[filt].head(10)
#Stripplot with daily boxplot.
fig, ax = plt.subplots(1,1,figsize=(12,6))
ax = sns.stripplot(x='clean_date', y='flair_sco_avg', data=df, jitter=0.2)
ax = sns.boxplot(x='clean_date', y='flair_sco_avg', data=df, boxprops=dict(facecolor=(0,0,0,0)))
ax.set_title('Sentiment by Day:  November 10-22')
ax.set_xlabel('Date')
ax.set_ylabel('Sentiment')
x_dates = df['clean_date'].dt.strftime('%a %d-%b').unique()
ax.set_xticklabels(labels=x_dates, rotation=45)
plt.show()
#Daily sentiment stats.
df.groupby(['clean_date']).agg({'flair_sco_avg':['median', 'count', 'mean']})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15229725219712983, 'fraction_numerical': 0.02391812214929358, 'mean_word_length': 4.563118811881188, 'pattern_counts': {'":': 1, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '429309', 'license': 'MIT', 'n_tokens_mistral': 3371, 'n_tokens_neox': 3213, 'n_words': 721, 'path': 'sentiment_for_hiding_youtube_dislikes_v2.ipynb', 'repo_name': 'jkmackie/youtube_dislike_count_removal_sentiment', 'size': '8989'}