<jupyter_start><jupyter_text>1. SemAxis, pretrained embedding<jupyter_code># define the functions to be used
def polar_dict1(path):
    polar = pd.read_csv(path)
    polar = polar.dropna() # remove NaN values
    po = dict(list(zip(polar.Word, polar.Sim))) # convert the dataframe to key-value pairs
    return po

# calculate the sentiment of the articles
def calculate_polar1(path, text):
    po = polar_dict1(path)
    polar = 0
    for t in text:
        try:
            polar += po.get(t)
        except TypeError:
            polar += 0
            
    return polar

# create the dataframe of the sentiment series
def senti(path, Embedding, GType, BType, Method):
    df1 = pd.DataFrame(df['Date'])
    df1['Score'] = df['corpus'].apply(lambda x: calculate_polar1(path, x)) # get the sentiment of each article
    df1 = df1.groupby(by = ['Date'], as_index = False).mean() # calculate the daily sentiment
    
    df1['Embedding'] = Embedding
    df1['GType'] = GType
    df1['BType'] = BType
    df1['Method'] = Method
    
    return df1
    
    
df11 = senti('SemAxis11.csv', 'PT', 'F', 'F', 'SemAxis')
df11.head()
df12 = senti('SemAxis12.csv', 'PT', 'NF', 'F', 'SemAxis')
df12.head()
df13 = senti('SemAxis13.csv', 'PT', 'F', 'NF', 'SemAxis')
df14 = senti('SemAxis14.csv', 'PT', 'NF', 'NF', 'SemAxis')<jupyter_output><empty_output><jupyter_text>2. SemAxis, Self-trained embedding<jupyter_code>df21 = senti('SemAxis21.csv', 'T', 'F', 'F', 'SemAxis') # F + F, self trained embedding
df22 = senti('SemAxis22.csv', 'T', 'NF', 'F', 'SemAxis') # NF + F, self trained embedding
df23 = senti('SemAxis23.csv', 'T', 'F', 'NF', 'SemAxis') # F + NF, self trained embedding
df24 = senti('SemAxis24.csv', 'T', 'NF', 'NF', 'SemAxis') # NF + NF, self trained embedding<jupyter_output><empty_output><jupyter_text>3. SentiProp, pre-trained embedding<jupyter_code># define the functions to be used
def polar_dict2(path):
    polar = pd.read_csv(path)
    polar = polar.dropna() # remove NaN values
    po = dict(list(zip(polar.words, polar.polarity))) # convert the dataframe to key-value pairs
    return po

# calculate the sentiment of the articles
def calculate_polar2(path, text):
    po = polar_dict2(path)
    polar = 0
    for t in text:
        try:
            polar += po.get(t)
        except TypeError:
            polar += 0
            
    return polar


# create the dataframe of the sentiment series
def senti(path, Embedding, GType, BType, Method):
    df1 = pd.DataFrame(df['Date'])
    df1['Score'] = df['corpus'].apply(lambda x: calculate_polar2(path, x)) # get the sentiment of each article
    df1 = df1.groupby(by = ['Date'], as_index = False).mean() # calculate the daily sentiment
    
    df1['Embedding'] = Embedding
    df1['GType'] = GType
    df1['BType'] = BType
    df1['Method'] = Method
    
    return df1
df31 = senti('SentiProp11.csv', 'PT', 'F', 'F', 'SentiProp') 
df32 = senti('SentiProp12.csv', 'PT', 'NF', 'F', 'SentiProp') 
df33 = senti('SentiProp13.csv', 'PT', 'F', 'NF', 'SentiProp') 
df34 = senti('SentiProp14.csv', 'PT', 'NF', 'NF', 'SentiProp')<jupyter_output><empty_output><jupyter_text>4. SentiProp, self-trained embedding<jupyter_code>df41 = senti('SentiProp21.csv', 'T', 'F', 'F', 'SentiProp') 
df42 = senti('SentiProp22.csv', 'T', 'NF', 'F', 'SentiProp') 
df43 = senti('SentiProp23.csv', 'T', 'F', 'NF', 'SentiProp') 
df44 = senti('SentiProp24.csv', 'T', 'NF', 'NF', 'SentiProp') 
# concate all the dataframes
dfs = [df11, df12, df13, df14, df21, df22, df23, df24, df31, df32, df33, df34, df41, df42, df43, df44]
result = pd.concat(dfs)
res = result.sort_values(by = ['Date', 'Method'])
res.to_csv('WSJ_sentiment.csv', index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16662173092477756, 'fraction_numerical': 0.03612833647883527, 'mean_word_length': 3.9335106382978724, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '359050', 'license': 'MIT', 'n_tokens_mistral': 1428, 'n_tokens_neox': 1357, 'n_words': 353, 'path': 'Code/InductedDicts/WSJ_sentiment_series.ipynb', 'repo_name': 'dordonezc/SSInductionStocks', 'size': '3709'}