<jupyter_start><jupyter_text><jupyter_code>import bs4 as bs
import datetime as dt
import os
import pandas_datareader.data as web
import pickle
import requests
import re
import pandas as pd

os.chdir = ('C:/Users/Anastasia/Desktop/New folder/Разное/Учеба/DS4A')

def save_sp500_tickers():
    resp = requests.get('http://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
    soup = bs.BeautifulSoup(resp.text, 'lxml')
    table = soup.find('table', {'class': 'wikitable sortable'})
    tickers = []
    for row in table.findAll('tr')[1:]:
        ticker = row.findAll('td')[0].text
        ticker = ticker[:-1]
        tickers.append(ticker)
    with open("sp500tickers.pickle", "wb") as f:
        pickle.dump(tickers, f)
    tickers.remove('BRK.B')
    tickers.remove('BF.B')
    tickers.remove('CTVA')
    tickers.remove('DOW')
    tickers.remove('FOXA')
    tickers.remove('FOX')
    tickers.remove('NLOK')
    tickers.remove('LHX')
    tickers.remove('VIAC')
    
    return tickers


# save_sp500_tickers()
def get_data_from_yahoo(reload_sp500=True):
    if reload_sp500:
        tickers = save_sp500_tickers()
    else:
        with open("sp500tickers.pickle", "rb") as f:
            tickers = pickle.load(f)
    if not os.path.exists('stock_dfs'):
        os.makedirs('stock_dfs')

    start = dt.datetime(2018, 1, 1)
    end = dt.datetime(2018, 5, 31)
    df = pd.DataFrame()
    
    for ticker in tickers:
        print(ticker)
        ind_df = web.DataReader(ticker, 'yahoo', start, end)
        ind_df.reset_index(inplace=True)
        #ind_df.set_index("Date", inplace=True)
        ind_df['Return'] = ind_df['Close'] / ind_df['Close'].shift(1) - 1
        ind_df['Symbol'] = ticker
        
        df = df.append(ind_df)
        print(df)
        
    df.to_csv('sp500returns.csv')

get_data_from_yahoo()
import numpy as np
ret = pd.read_csv('sp500returns.csv').iloc[:,1:]

sent_vader = pd.read_csv('scored_news_vader.csv').iloc[:,1:]

# Constructing industry sentiment index:
dates = np.sort(sent_vader.dropna().published.unique())[9:]
ind_names = sent_vader['industry_0'].unique()
sent_ind_vader = pd.DataFrame(index=dates,columns=ind_names)

for ind in ind_names:
    for t in dates:
        sent_ind_vader.loc[t,ind] = sent_vader[sent_vader.published == t][sent_vader.industry_0 == ind].compound.mean()
        
sent_ind_vader = sent_ind_vader.fillna(0)    
sent_ind_vader
news = pd.read_csv('allnews_with_all_labels.csv')
bert_sent = pd.read_csv('bert_predicted_result_40k.csv')
bert_sent = bert_sent.sort_values('index')
news['bert_sent'] = bert_sent['label']
news = news[['Unnamed: 0', 'published', 'company_0', 'industry_0', 'bert_sent']]
news['published'] = sent_vader['published']

# Constructing industry sentiment index:
ind_names = news['industry_0'].unique()
sent_ind_bert = pd.DataFrame(index=dates,columns=ind_names)

for ind in ind_names:
    for t in dates:
        sent_ind_bert.loc[t,ind] = news[news.published == t][news.industry_0 == ind].bert_sent.mean()
        
sent_ind_bert = sent_ind_bert.fillna(0)    
sent_ind_bert
#type(sent_vader['published'])
get_industries = pd.read_csv('sp500companies_industry.csv')[['Symbol','Security','GICS Sector']]
get_industries
ret = pd.merge(ret, get_industries, on='Symbol')
#ret = pd.merge(ret, on='company_0' )
ret['Industry Sentiment vader'] = 0
ret['Industry Sentiment bert'] = 0
get_industries
# Adding Industry Sentiment column to returns data
ind_sent = []
for i in range(51527):
    t = str(ret['Date'][i])
    ind = str(ret['GICS Sector'][i])
    ind_sent.append(sent_ind_vader.loc[t,ind])
    
    
ret['Industry Sentiment vader'] = ind_sent  

ind_sent = []
for i in range(51527):
    t = str(ret['Date'][i])
    ind = str(ret['GICS Sector'][i])
    ind_sent.append(sent_ind_bert.loc[t,ind])
    
    
ret['Industry Sentiment bert'] = ind_sent 
ret
# Constructing Company-specific Sentiment Bert:

comp_names = news['company_0'].unique()
sent_comp_bert = pd.DataFrame(index=dates,columns=comp_names)

for com in comp_names:
    for t in dates:
        sent_comp_bert.loc[t,com] = news[news.published == t][news.company_0 == com].bert_sent.mean()
        
sent_comp_bert = sent_comp_bert.fillna(0)    
sent_comp_bert
# Adding Company Sentiment column to returns data
comp_sent = []
for i in range(51527):
    t = str(ret['Date'][i])
    comp = str(ret['Security'][i])
    if((comp in sent_comp_bert.columns) & (t!='2018-01-02')):
        comp_sent.append(sent_comp_bert.loc[t,comp])
    else:
        comp_sent.append(0)
    
    
ret['Company Sentiment'] = comp_sent    
ret.to_csv('returns_with_sentiment.csv')
ret
import numpy as np
ret['LOGCAP'] = np.log(np.multiply(ret['Close'],ret['Volume']))
ret['Date'] = pd.to_datetime(ret['Date'],format='%Y-%m-%d')
ret
groups = ret.groupby(['Date'])

m = pd.DataFrame()
portfolios = {}
ret_mean = {}
cap_mean = {}
ret_mean_next = {}
cap_mean_next = {}

pnum = 10
print('No portfolios!')
for idx in range(pnum) :
    portfolios[idx] = pd.DataFrame()

for group_name, group in groups :
    g = group.dropna().sort_values(['Company Sentiment'])
    l = int(g.shape[0]/pnum)
    for idx in range(pnum) :
        portfolios[idx] = portfolios[idx].append(g.iloc[idx*l : (idx+1)*l - 1])
for idx in range(pnum) :
    ret_mean[idx] = portfolios[idx].groupby('Date')['Return'].mean()
    #cap_mean[idx] = portfolios[idx].groupby('Date')['LOGCAP'].mean()
ret_mean
ff_fact = pd.read_csv('f-f_factors.csv', sep = '\t', decimal=',')
ff_fact.iloc[:,0] = ff_fact.iloc[:,0].astype(str)
ff_fact.iloc[:,0] = pd.to_datetime(ff_fact.iloc[:,0],format='%Y-%m-%d')
ff_fact['Mkt-RF'] = pd.to_numeric(ff_fact['Mkt-RF'],errors='coerce')
ff_fact['SMB'] = pd.to_numeric(ff_fact['SMB'],errors='coerce')
ff_fact['HML'] = pd.to_numeric(ff_fact['HML'],errors='coerce')
ff_fact['RF'] = pd.to_numeric(ff_fact['RF'],errors='coerce')
ff_fact.iloc[:,1:] = ff_fact.iloc[:,1:]/100
ff_fact
import statsmodels.api as sm
dates = np.sort(ret.dropna().Date.unique())
cpm = {}
mkt = ff_fact['Mkt-RF'][1:]
X = sm.add_constant(mkt)
for idx in range(pnum) :
    ret_mean_next[idx] = np.empty(len(dates))
    cap_mean_next[idx] = np.empty(len(dates))
    for i in range(len(dates) - 1):
        date = dates[i]
        ndate = dates[i+1]
        permno = portfolios[idx].Symbol[portfolios[idx].Date == date]
        p = ret[ret.Date == ndate]
        p = p[p.Symbol.isin(permno)]
        ret_mean_next[idx][i] = p.Return.mean() - ff_fact.iloc[i, 4]
        cap_mean_next[idx][i] = p.LOGCAP.mean()
    cpm[idx] = sm.OLS(ret_mean_next[idx], X).fit()
np.set_printoptions(precision=4)
print('\tRET\talpha\tbeta')
for idx in range(pnum) :
    print("{:d}\t{:.4f}\t{:.4f}\t{:.4f}".format(idx,
        ret_mean_next[idx].mean(), 
        cpm[idx].params[0], cpm[idx].params[1]))
long_short_ret = portfolios[9].groupby('Date')['Return'].mean() - portfolios[0].groupby('Date')['Return'].mean()
long_ret = portfolios[9].groupby('Date')['Return'].mean()
short_ret = -portfolios[0].groupby('Date')['Return'].mean()
long_short = pd.DataFrame(long_short_ret)
strat_sum = pd.DataFrame()
strat_sum['LS ret'] = long_short_ret
strat_sum['Long ret'] = long_ret
strat_sum['Short ret'] = short_ret
strat_sum['MktRF'] = np.array(mkt)
strat_sum['Date'] = np.sort(ret.dropna().Date.unique())
strat_sum['Mkt_cum_ret'] = (1+strat_sum['MktRF']).cumprod() - 1
strat_sum['LS_cum_ret'] = (1+strat_sum['LS ret']).cumprod() - 1
strat_sum['L_cum_ret'] = (1+strat_sum['Long ret']).cumprod() - 1
strat_sum['S_cum_ret'] = (1+strat_sum['Short ret']).cumprod() - 1
print(strat_sum)
import matplotlib.pyplot as plt

strat_sum.plot(x='Date', y=['LS_cum_ret', 'Mkt_cum_ret', 'L_cum_ret', 'S_cum_ret'], figsize=(10,5), grid=True)

strat_sum.plot(x='Date', y=['LS_cum_ret', 'Mkt_cum_ret'], figsize=(10,5), grid=True)
strat_sum.plot(x='Date', y=['Mkt_cum_ret', 'L_cum_ret'], figsize=(10,5), grid=True)
import seaborn as sns
sns.set_style("white")

# Import data
x1 = strat_sum.loc[:, 'LS ret']
x2 = strat_sum.loc[:, 'Long ret']
x3 = strat_sum.loc[:, 'Short ret']

# Plot
kwargs = dict(hist_kws={'alpha':.6}, kde_kws={'linewidth':2})

plt.figure(figsize=(10,7), dpi= 80)
sns.distplot(x1, color="dodgerblue", label="Long-short strategy returns", **kwargs)
sns.distplot(x2, color="orange", label="Long strategy returns", **kwargs)
sns.distplot(x3, color="deeppink", label="Short strategy returns", **kwargs)
plt.legend();<jupyter_output><empty_output><jupyter_text>Inside industry using industry sentiment index<jupyter_code>industrials = ret[ret['GICS Sector'] == 'Materials']
industrials
groups = industrials.groupby(['Date'])

m = pd.DataFrame()
portfolios = {}
ret_mean = {}
cap_mean = {}
ret_mean_next = {}
cap_mean_next = {}

pnum = 10
print('No portfolios!')
for idx in range(pnum) :
    portfolios[idx] = pd.DataFrame()

for group_name, group in groups :
    g = group.dropna().sort_values(['Company Sentiment'])
    l = int(g.shape[0]/pnum)
    for idx in range(pnum) :
        portfolios[idx] = portfolios[idx].append(g.iloc[idx*l : (idx+1)*l - 1])
for idx in range(pnum) :
    ret_mean[idx] = portfolios[idx].groupby('Date')['Return'].mean()
import statsmodels.api as sm
dates = np.sort(ret.dropna().Date.unique())
cpm = {}
mkt = ff_fact['Mkt-RF'][1:]
X = sm.add_constant(mkt)
for idx in range(pnum) :
    ret_mean_next[idx] = np.empty(len(dates))
    cap_mean_next[idx] = np.empty(len(dates))
    for i in range(len(dates) - 1):
        date = dates[i]
        ndate = dates[i+1]
        permno = portfolios[idx].Symbol[portfolios[idx].Date == date]
        p = industrials[industrials.Date == ndate]
        p = p[p.Symbol.isin(permno)]
        ret_mean_next[idx][i] = p.Return.mean() - ff_fact.iloc[i, 4]
        cap_mean_next[idx][i] = p.LOGCAP.mean()
    cpm[idx] = sm.OLS(ret_mean_next[idx], X).fit()
np.set_printoptions(precision=4)
print('\tRET\talpha\tbeta')
for idx in range(pnum) :
    print("{:d}\t{:.4f}\t{:.4f}\t{:.4f}".format(idx,
        ret_mean_next[idx].mean(), 
        cpm[idx].params[0], cpm[idx].params[1]))
long_short_ret = portfolios[9].groupby('Date')['Return'].mean() - portfolios[0].groupby('Date')['Return'].mean()
long_ret = portfolios[9].groupby('Date')['Return'].mean()
short_ret = -portfolios[0].groupby('Date')['Return'].mean()
long_short = pd.DataFrame(long_short_ret)
strat_sum = pd.DataFrame()
strat_sum['LS ret'] = long_short_ret
strat_sum['Long ret'] = long_ret
strat_sum['Short ret'] = short_ret
strat_sum['MktRF'] = np.array(mkt)
strat_sum['Date'] = np.sort(ret.dropna().Date.unique())
strat_sum['Mkt_cum_ret'] = (1+strat_sum['MktRF']).cumprod() - 1
strat_sum['LS_cum_ret'] = (1+strat_sum['LS ret']).cumprod() - 1
strat_sum['L_cum_ret'] = (1+strat_sum['Long ret']).cumprod() - 1
strat_sum['S_cum_ret'] = (1+strat_sum['Short ret']).cumprod() - 1
print(strat_sum)
strategy = pd.DataFrame(columns = ['Name', 'Mean', 'Std', 'SR'])
l_strat = pd.DataFrame(columns = ['Name', 'Mean', 'Std', 'SR'])
s_strat = pd.DataFrame(columns = ['Name', 'Mean', 'Std', 'SR'])

def my(df, name, ret, sent) :
    m = np.mean(ret)
    s = np.std(ret)
    df = df.append({'Name' : name, 'Mean' : m, 'Std' : s, 'SR' : m/s}, ignore_index=True)
    
    median = sent.rolling(5).median()
    msk = sent - median > 0
    msk = msk.shift()[5:]
    ret = ret[5:]

    r_pos = ret[msk]
    m = np.mean(r_pos)
    s = np.std(r_pos)
    df = df.append({'Name' : name + ' HS', 'Mean' : m, 'Std' : s, 'SR' : m/s}, ignore_index=True)

    r_neg = ret[msk == False]
    m = np.mean(r_neg)
    s = np.std(r_neg)
    df = df.append({'Name' : name + ' LS', 'Mean' : m, 'Std' : s, 'SR' : m/s}, ignore_index=True)

    return df, r_pos, r_neg
#sent = sent_ind_bert.loc[list(ret['Date'].unique()[1:]), 'Industrials']
sent = sent_ind_bert.loc[:, 'Materials']
#print(sent_ind_bert.index)
sent
strat_sum['LS ret']
strat_sum.index = strat_sum.index.astype(date)
sent.index = sent.index.astype(date)
strategy = my(strategy, 'Comp Sentiment Index', strat_sum['LS ret'], sent)
print(strategy[0])
sns.set_style("white")

# Import data
x1 = strategy[1]
x2 = strategy[2]

# Plot
kwargs = dict(hist_kws={'alpha':.6}, kde_kws={'linewidth':2})

plt.figure(figsize=(10,7), dpi= 80)
sns.distplot(x1, color="dodgerblue", label="Returns following positive sentiment", **kwargs)
sns.distplot(x2, color="orange", label="Returns following negative sentiment", **kwargs)
plt.legend();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18664397439014507, 'fraction_numerical': 0.016451900478158685, 'mean_word_length': 4.5989110707803995, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '467316', 'license': 'MIT', 'n_tokens_mistral': 5037, 'n_tokens_neox': 4864, 'n_words': 824, 'path': 'Factor Model/Part4_Factor_model.ipynb', 'repo_name': 'Sinba7/The-NLP-News-Sentiment-Trading-Strategy', 'size': '12339'}