<jupyter_start><jupyter_text>sbsitute the dow jones with russessl_2000<jupyter_code># df_stocks['prices'] = df_stocks['close'].apply(np.int64)
df_stocks['prices'] = interpolated_df['close'].apply(np.int64)
df_stocks = df_stocks[['prices', 'articles']]
df_stocks['articles'] = df_stocks['articles'].map(lambda x: x.lstrip('.-'))
df_stocks
df_stocks["num_articles"] = ''
df_stocks["words"] = ''
df_stocks["words_per_article"] = ''
df_stocks
#statistics of the data
total_data_points = len(df_stocks) #total days
total_data_points
total_articles = df_stocks.values[3][1]
total_articles

from nltk import tokenize
p = "Good morning Dr. Adams. The patient is waiting for you in room number 3."

a_list = tokenize.sent_tokenize(total_articles)

print (len(a_list))

print (a_list[1])

print (len(a_list[1].split()))

print (len(total_articles.split()))

for date, row in df_stocks.T.iteritems():
    try:
        a_list = tokenize.sent_tokenize(df_stocks.loc[date, 'articles'])
        words = len(df_stocks.loc[date, 'articles'].split())
        df_stocks.set_value(date, 'num_articles', len(a_list))
        df_stocks.set_value(date, 'words', words)
#         df_stocks.set_value(date, 'words_per_article', words/len(a_list))
        if len(a_list) == 0:
            print(row)
# t_value(date, 'pos', ss['pos'])
    except TypeError:
        print (df_stocks.loc[date, 'articles'])
        print (date)
df_stocks

vocaburary = set()
for i in range(0 ,len(df_stocks)):
    line = df_stocks['articles'].values[i]
#     print(line)
    words = line.split()
#     print(words)
    for word in words:
        if word not in vocaburary:
            vocaburary.add(word)
print (len(vocaburary))
# print (vocaburary)
print (df_stocks['words'].sum())
print (df_stocks['words'].min())
print (df_stocks['words'].max())
print (df_stocks['words'].mean())
print (df_stocks['words'].median())
print (df_stocks['num_articles'].sum())
print (df_stocks['num_articles'].min())
print (df_stocks['num_articles'].max())
print (df_stocks['num_articles'].mean())
print (df_stocks['num_articles'].median())
df = df_stocks[['prices']].copy()
df
# Adding new columns to the data frame
df["compound"] = ''
df["neg"] = ''
df["neu"] = ''
df["pos"] = ''
df
import nltk
nltk.download('vader_lexicon')
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import unicodedata
sid = SentimentIntensityAnalyzer()
for date, row in df_stocks.T.iteritems():
    try:
        sentence = unicodedata.normalize('NFKD', df_stocks.loc[date, 'articles'])#.encode('ascii','ignore')
        ss = sid.polarity_scores(sentence)
        df.set_value(date, 'compound', ss['compound'])
        df.set_value(date, 'neg', ss['neg'])
        df.set_value(date, 'neu', ss['neu'])
        df.set_value(date, 'pos', ss['pos'])
    except TypeError:
        print (df_stocks.loc[date, 'articles'])
        print (date)
df
df['num_articles'] = df_stocks['num_articles']
df['words'] = df_stocks['words']
df
print(df['compound'].mean())
print(df['compound'].median())
df['price_change'] = ''
df
for i in range(0,len(df)):
    if i == 0:
        df['price_change'].values[i] = -82 #price change from Jan 02 to Jan 05
    else:
        df['price_change'].values[i] = df['prices'].values[i] - df['prices'].values[i-1]
df
import scipy.stats as stats
print(stats.pearsonr(df['prices'],df['compound']))

print(stats.pearsonr(df['prices'],df['neg']))

print(stats.pearsonr(df['prices'],df['neu']))

print(stats.pearsonr(df['prices'],df['pos']))


print(stats.pearsonr(df['price_change'],df['compound']))
print(stats.pearsonr(df['price_change'],df['neg']))
print(stats.pearsonr(df['price_change'],df['neu']))
print(stats.pearsonr(df['price_change'],df['pos']))
def crosscorr(datax, datay, lag=0, wrap=False):
    """ Lag-N cross correlation. 
    Shifted data filled with NaNs 
    
    Parameters
    ----------
    lag : int, default 0
    datax, datay : pandas.Series objects of equal length
    Returns
    ----------
    crosscorr : float
    """
    if wrap:
        shiftedy = datay.shift(lag)
        shiftedy.iloc[:lag] = datay.iloc[-lag:].values
        return datax.corr(shiftedy)
    else: 
        return datax.corr(datay.shift(lag))
d1 = df['price_change'].apply(np.float64)
d2 = df['compound'].apply(np.float64)

xcov_daily = [crosscorr(d1,d2,lag = i) for i in range(30)]
print(xcov_daily)
d1 = df['prices'].apply(np.float64)
d2 = df['compound'].apply(np.float64)

xcov_daily = [crosscorr(d1,d2,lag = i) for i in range(30)]
print(xcov_daily)<jupyter_output>[-0.0582321694463992, -0.05834413826017889, -0.057787722688776584, -0.05700705551379429, -0.056487101954878625, -0.05622235099956721, -0.056297101140809415, -0.05605328039861588, -0.05612731475348609, -0.05569103876516831, -0.055432707340622965, -0.05464659808565464, -0.05458345764330806, -0.05429483493581736, -0.05427684914007749, -0.05413610676486024, -0.05471261299478054, -0.054702540293774596, -0.055588909851218817, -0.05558368163007006, -0.055495867558298405, -0.05594133055878906, -0.05623461672470961, -0.05633248101174331, -0.055934979449705324, -0.05575803745658195, -0.05532120569879228, -0.055341199458153294, -0.05535737778508014, -0.055243620910039586]
{'fraction_non_alphanumeric': 0.17417359365938526, 'fraction_numerical': 0.11502029769959404, 'mean_word_length': 4.774553571428571, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '184439', 'license': 'Apache-2.0', 'n_tokens_mistral': 2371, 'n_tokens_neox': 1995, 'n_words': 339, 'path': 'data_stats.ipynb', 'repo_name': 'nazanin-tabatabaei/NewsSentimentAnalysis_StockPredictions', 'size': '5173'}