<jupyter_start><jupyter_text>1. LM uncertainty term occurences<jupyter_code># All unique LM uncertainty terms and occurences
allMatchWords=[]
for list in df['UncertaintyWords']:
    allMatchWords=allMatchWords+literal_eval(list)
print(len(allMatchWords))
allMatchWordsCount=Counter(allMatchWords)
print(len(allMatchWordsCount))
print(allMatchWordsCount)
for term in allMatchWordsCount.keys():
    doc=nlp(term)
    for token in doc:
        print(token, token.pos_, token.tag_)<jupyter_output>could VERB MD
possible ADJ JJ
exposure NOUN NN
revise NOUN NN
pende PROPN NNP
appear VERB VB
seem VERB VB
confusion NOUN NN
confusing ADJ JJ
may VERB MD
perhaps ADV RB
possibly ADV RB
believe VERB VB
doubt NOUN NN
almost ADV RB
risk NOUN NN
suggest VERB VBP
speculation NOUN NN
unusually ADV RB
confuse PROPN NNP
variation NOUN NN
vary VERB VBP
nearly ADV RB
unsettled ADJ JJ
predict VERB VB
apparent ADJ JJ
reassess PROPN NNP
sometimes ADV RB
dependent ADJ JJ
unidentified ADJ JJ
unusual ADJ JJ
apparently ADV RB
approximately ADV RB
possibility NOUN NN
tentative ADJ JJ
speculate PROPN NNP
maybe ADV RB
fluctuation NOUN NN
roughly ADV RB
probably ADV RB
reconsider NOUN NN
unfamiliarity NOUN NN
probable ADJ JJ
conceivable ADJ JJ
anticipate PROPN NNP
unclear ADJ JJ
clarification NOUN NN
assume PROPN NNP
somewhat ADV RB
unfamiliar ADJ JJ
varied ADJ JJ
unexpected ADJ JJ
tend VERB VB
depend VERB VB
conditional ADJ JJ
uncertainty NOUN NN
tentatively ADV RB
sometime ADV RB
sudden ADJ JJ
reinterpret[...]<jupyter_text>2. Remove certain terms<jupyter_code>term_removed=['could','may','maybe','might']
df=df.reset_index(drop=True)
# Remove certain terms from all matched noun chunks
df['UncertaintyWordsFiltered']=None
df['UncertaintyCountFiltered']=0
for i in range(0,len(df)):
    list=literal_eval(df['UncertaintyWords'][i])
    list_new=[w for w in list if w not in term_removed]
    df['UncertaintyWordsFiltered'][i]=list_new
    df['UncertaintyCountFiltered'][i]=len(df['UncertaintyWordsFiltered'][i])
print(df.info())
print(df.head(10))<jupyter_output>ID  UncertaintyCount       UncertaintyWords  \
0  294326637                 0                     []   
1  294308147                 0                     []   
2  294323196                 1              ['could']   
3  294262284                 2  ['possible', 'could']   
4  294234319                 0                     []   
5  294323101                 1           ['exposure']   
6  294205735                 1             ['revise']   
7  294261564                 0                     []   
8  294337453                 1              ['pende']   
9  294281403                 2     ['appear', 'seem']   

  UncertaintyWordsFiltered  UncertaintyCountFiltered  
0                       []                         0  
1                       []                         0  
2                       []                         0  
3               [possible]                         1  
4                       []                         0  
5               [exposure]                [...]<jupyter_text>3. Calculate uncertainty scores<jupyter_code># All news metadata
allNews=pd.read_pickle('/home/ec2-user/SageMaker/New Uncertainty/parsed_xml.pkl')
print(allNews.info())
# Reg relevant articles
df_regSentsExpand=pd.read_pickle('/home/ec2-user/SageMaker/New Uncertainty/Reg Relevance/RegSentsExpand_NounChunks3.pkl')
print(df_regSentsExpand.info())
# Merge with metadata
df_regSentsExpand=df_regSentsExpand.merge(allNews[['ID','Title','StartDate','Text','Newspaper','Year','Month','Type','SourceType']],
                           on='ID',how='left')
print(df_regSentsExpand.info())
# Remove duplicated articles
IDs_nodup=pd.read_csv('/home/ec2-user/SageMaker/New Uncertainty/IDs_no_duplicates.csv')
print(IDs_nodup.info())
IDs_nodup['ID']=IDs_nodup['ID'].astype('str')
df_reg=IDs_nodup.merge(df_regSentsExpand,on='ID',how='left').reset_index(drop=True)
print(df_reg.info())
# Use filtered noun chunk matches to define reg relevance
df_reg.loc[df_reg['NounChunkMatchFiltered']!=0,'RegRelevance']=1
print("# of reg relevant articles:",df_reg[df_reg['RegRelevance']==1]['ID'].nunique())
# Total word count
GIsentiments=pd.read_csv('/home/ec2-user/SageMaker/New Uncertainty/Sentiment Analysis/GIsentiments.csv')
GIsentiments['ID']=GIsentiments['ID'].astype('str')
print(GIsentiments.info())
# Merge
df['ID']=df['ID'].astype('str')
df=df.merge(GIsentiments[['ID','TotalWordCount']],on='ID',how='left')
print(df.info())
df=df_reg.merge(df,on='ID',how='left')
print(df.info())
df=df[df['RegRelevance']==1].reset_index(drop=True)
print(df.info())
df['UncertaintyScore']=df['UncertaintyCountFiltered']/df['TotalWordCount']*100
print(df.info())<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 493418 entries, 0 to 493417
Data columns (total 21 columns):
 #   Column                       Non-Null Count   Dtype         
---  ------                       --------------   -----         
 0   ID                           493418 non-null  object        
 1   RegSentsExpand               493418 non-null  object        
 2   NounChunksMatch              493418 non-null  int64         
 3   NounChunkMatchWords          493418 non-null  object        
 4   NounChunkMatchWordsFiltered  493418 non-null  object        
 5   NounChunkMatchFiltered       493418 non-null  int64         
 6   Title                        493418 non-null  object        
 7   StartDate                    493418 non-null  datetime64[ns]
 8   Text                         493418 non-null  object        
 9   Newspaper                    493418 non-null  object        
 10  Year                         493418 non-null  int64         
 11  Month                     [...]<jupyter_text>4. Construct uncertainty index<jupyter_code># Change data format
df['StartDate']=df['StartDate'].astype('datetime64[ns]')
df['Year']=df['StartDate'].dt.year
df['Month']=df['StartDate'].dt.month
df['Newspaper']=df['Newspaper'].astype('category')
print(df.info())
# Unique year-month dataframe
df_ym=df[['Year','Month']].drop_duplicates().sort_values(['Year','Month']).reset_index(drop=True).reset_index()
df_ym['YM']=df_ym['index']+1
df_ym=df_ym.drop('index',axis=1)
print(df_ym)
df=df.merge(df_ym[['Year','Month','YM']],on=['Year','Month'],how='left').sort_values(['Year','Month']).reset_index(drop=True)
print(df.info())
# Function to estimate index (suppressing constant)
def estimate_index(var_name):
    FE_OLS=sm.ols(formula=var_name + ' ~ 0+C(YM)+C(Newspaper)',
        data=df).fit()
    print(FE_OLS.summary())

    FE_estimates=pd.DataFrame()
    FE_estimates[var_name+'Index']=FE_OLS.params[0:len(df_ym)]
    FE_estimates=FE_estimates.reset_index().rename(columns={'index':'FE'})
    FE_estimates['YM']=FE_estimates['FE'].str.split("[",expand=True)[1].str.split("]",expand=True)[0].astype('int64')
    
    return FE_estimates
# Uncertainty index
UncertaintyIndex=estimate_index('UncertaintyScore')
print(UncertaintyIndex.info())
UncertaintyIndex=df_ym.merge(UncertaintyIndex,on='YM',how='outer').\
        sort_values(['Year','Month'])
print(UncertaintyIndex.info())
UncertaintyIndex=UncertaintyIndex.rename(columns={'UncertaintyScoreIndex':'UncertaintyIndex'})
print(UncertaintyIndex.head())
# Export
UncertaintyIndex.to_csv('/home/ec2-user/SageMaker/New Uncertainty/Sentiment Analysis/RegRelevant_MonthlyUncertaintyIndex_Robust.csv',index=False)<jupyter_output><empty_output><jupyter_text>Compare with baseline uncertainty index<jupyter_code>UncertaintyIndex_baseline=pd.read_csv('/home/ec2-user/SageMaker/New Uncertainty/Sentiment Analysis/RegRelevant_MonthlySentimentIndex.csv')
print(UncertaintyIndex_baseline.info())
import scipy.stats
import matplotlib.pyplot as plt
print('Baseline & Robust:',scipy.stats.pearsonr(UncertaintyIndex['UncertaintyIndex'], UncertaintyIndex_baseline['UncertaintyIndex']))
y=UncertaintyIndex['UncertaintyIndex']
plt.plot(y)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1219450743260267, 'fraction_numerical': 0.03224993701184178, 'mean_word_length': 3.3287895310796074, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '261781', 'license': 'MIT', 'n_tokens_mistral': 2924, 'n_tokens_neox': 2603, 'n_words': 508, 'path': 'Code/6_1_Modify_LM_Uncertainty.ipynb', 'repo_name': 'zhoudanxie/sinclair-xie-sentiment', 'size': '7938'}