<jupyter_start><jupyter_text>1. Import Meta Data<jupyter_code># All news metadata
allNews=pd.read_pickle('/home/ec2-user/SageMaker/New Uncertainty/parsed_xml.pkl')
print(allNews.info())
# Reg relevant articles
df_regSentsExpand=pd.read_pickle('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/RegSentsExpand_NounChunks3.pkl')
print(df_regSentsExpand.info())
# Merge with metadata
df=df_regSentsExpand.merge(allNews[['ID','Title','StartDate','Text','Newspaper','Year','Month','Type','SourceType']],
                           on='ID',how='left')
print(df.info())
# Remove duplicated articles
IDs_nodup=pd.read_csv('/home/ec2-user/SageMaker/New Uncertainty/IDs_no_duplicates.csv')
print(IDs_nodup.info())
#df['ID']=df['ID'].astype('int64')
IDs_nodup['ID']=IDs_nodup['ID'].astype('str')
df=IDs_nodup.merge(df,on='ID',how='left').reset_index(drop=True)
print(df.info())
# Use filtered noun chunk matches to define reg relevance
df.loc[df['NounChunkMatchFiltered']!=0,'RegRelevance']=1
print("# of reg relevant articles:",df[df['RegRelevance']==1]['ID'].nunique())<jupyter_output># of reg relevant articles: 493418<jupyter_text>2. Calculate Sentiment Scores<jupyter_code># LM uncertainty scores
LMuncertainty=pd.read_csv('/home/ec2-user/SageMaker/New Uncertainty/Sentiment Analysis/LMuncertainty.csv')
print(LMuncertainty.info())
LMuncertainty['ID']=LMuncertainty['ID'].astype('str')
print(LMuncertainty.info())
print(type(LMuncertainty['ID'][0]))
# Merge
df2=df.merge(LMuncertainty,on='ID',how='left')
print(df2.info())
# GI sentiments
GIsentiments=pd.read_csv('/home/ec2-user/SageMaker/New Uncertainty/Sentiment Analysis/GIsentiments.csv')
GIsentiments['ID']=GIsentiments['ID'].astype('str')
print(GIsentiments.info())
# LSD sentiments
LSDsentiments=pd.read_csv('/home/ec2-user/SageMaker/New Uncertainty/Sentiment Analysis/LSDsentiments.csv')
LSDsentiments['ID']=LSDsentiments['ID'].astype('str')
print(LSDsentiments.info())
# LM sentiments
LMsentiments=pd.read_csv('/home/ec2-user/SageMaker/New Uncertainty/Sentiment Analysis/LMsentiments.csv')
LMsentiments['ID']=LMsentiments['ID'].astype('str')
print(LMsentiments.info())
# Merge
df3=df2.merge(GIsentiments,on='ID',how='left').\
    merge(LMsentiments,on='ID',how='left').\
    merge(LSDsentiments,on='ID',how='left')
print(df3.info())
# Reg relevant articles only
df_reg=df3[df3['RegRelevance']==1]
print(df_reg.info())
# Calculate scores
df_reg['UncertaintyScore']=df_reg['UncertaintyCount']/df_reg['TotalWordCount']*100
for dic in ['GI','LSD','LM']:
    df_reg[dic+'score']=(df_reg[dic+'posCount']-df_reg[dic+'negCount'])/df_reg['TotalWordCount']*100
print(df_reg.info())
df_reg.to_pickle('/home/ec2-user/SageMaker/New Uncertainty/Sentiment Analysis/RegRelevant_ArticleSentimentWordsScores.pkl')
sentimentScores=df_reg[['ID','StartDate','Newspaper','UncertaintyScore','GIscore','LMscore','LSDscore']]
print(sentimentScores.info())
sentimentScores.to_csv('/home/ec2-user/SageMaker/New Uncertainty/Sentiment Analysis/RegRelevant_ArticleSentimentScores.csv',index=False)<jupyter_output><empty_output><jupyter_text>3. Calculate Monthly Average Scores<jupyter_code># Calculate average monthly scores
df_reg['ArticleCount']=1
monthlyScores=df_reg[['Year','Month','UncertaintyScore','GIscore','LSDscore','LMscore','ArticleCount']].\
    groupby(['Year','Month']).agg({'UncertaintyScore':'mean','GIscore':'mean',
                                   'LSDscore':'mean','LMscore':'mean','ArticleCount':'sum'}).reset_index()
print(monthlyScores.info())
print(monthlyScores.head())
monthlyScores.to_csv('/home/ec2-user/SageMaker/New Uncertainty/Sentiment Analysis/RegRelevant_MonthlySentimentScores.csv',index=False)<jupyter_output><empty_output><jupyter_text>A1. Examine sentiment words<jupyter_code>import random
df_reg=df_reg.reset_index(drop=True)
sample=random.sample(range(len(df_reg)),10)
for i in sample:
    print(df_reg['RegSentsExpand'][i],df_reg['LMposWords'][i],df_reg['LMnegWords'][i],df_reg['UncertaintyWords'][i])
    print('\n')
for i in range(len(df_reg)):
    if len(df_reg['RegSentsExpand'][i])<500:
        if ('with negation' in df_reg['LMposWords'][i]) | ('with negation' in df_reg['LMnegWords'][i]):
            print(df_reg['ID'][i],df_reg['RegSentsExpand'][i],df_reg['LMposWords'][i],df_reg['LMnegWords'][i],df_reg['UncertaintyWords'][i])
            print('\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18381847352738942, 'fraction_numerical': 0.010772404308961724, 'mean_word_length': 9.032183908045978, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '301170', 'license': 'MIT', 'n_tokens_mistral': 1691, 'n_tokens_neox': 1614, 'n_words': 164, 'path': 'Code/7_Calculate_Sentiment_Scores.ipynb', 'repo_name': 'zhoudanxie/sinclair-xie-sentiment', 'size': '4363'}