<jupyter_start><jupyter_text>Dicky-Fuller Test for 'period_df'<jupyter_code>dicky_fuller_test = adfuller(period_df['price'])
dicky_fuller_test<jupyter_output><empty_output><jupyter_text>Solving unstationarity  Differencing<jupyter_code>diff_period_df = period_df['price'].diff()
diff_period_df = diff_period_df.dropna()
dicky_fuller_test = adfuller(diff_period_df)
dicky_fuller_test<jupyter_output><empty_output><jupyter_text>Log Differencing<jupyter_code>log_price = np.log(period_df['price'])
dicky_fuller_test = adfuller(log_price)
dicky_fuller_test
log_diff_period_df = log_price.diff()
log_diff_period_df = log_diff_period_df.dropna()
dicky_fuller_test = adfuller(log_diff_period_df)
dicky_fuller_test
#period_df.drop(period_df.index[0], inplace=True) # make consistent after price difference
pylab.rcParams['figure.figsize'] = (20, 3) 

plt.figure(1)
plt.subplot(121)
plt.title("TS for a period")
plt.plot(period_df['price'])

plt.subplot(122)
plt.title("TS for a period  with date")
plt.plot(period_df['date'], period_df['price'])



plt.figure(2)
plt.subplot(121)
plt.title("TS for a period after differencing")
plt.plot(diff_period_df)

plt.subplot(122)
plt.title("TS for a period after differencing with date")
plt.plot(period_df['date'][1:], diff_period_df)



plt.figure(3)
plt.subplot(121)
plt.title("TS for a period after log differencing")
plt.plot(log_diff_period_df)

plt.subplot(122)
plt.title("TS for a period after log differencing with date")
plt.plot(period_df['date'][1:], log_diff_period_df)<jupyter_output><empty_output><jupyter_text>***<jupyter_code>from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression, LinearRegression<jupyter_output><empty_output><jupyter_text>Tone download<jupyter_code>import time

start = time.time()

#SQLALCHEMY
from sqlalchemy import create_engine
from sqlalchemy import inspect, select, MetaData, and_

db_uri = 'postgresql://ostap:12345@localhost:5432/goog'
engine = create_engine(db_uri)
conn = engine.connect()

inspector = inspect(engine)

# Get table information

meta = MetaData(engine,reflect=True)
table = meta.tables['news']

#print(inspector.get_table_names())
#print(inspector.get_columns('news'))

select_st = select([table.c.DATE, table.c.TONE, table.c.DOCUMENTIDENTIFIER, table.c.ORGANIZATIONS, table.c.GCAM]).where(and_(table.c.DATE < '20160400000000' , table.c.DATE > '20160131234500'))

res = conn.execute(select_st).fetchall()

news = dict()
news['date'] = [datetime.datetime.strptime(el[0], '%Y%m%d%H%M%S') for el in res if '.' not in el[0]]
news['tone'] = [float(el[1].split(',')[0]) for el in res if '.' not in el[0]]
news['source'] = [el[2] for el in res if '.' not in el[0]]
news['org'] = [el[3] for el in res if '.' not in el[0]]
news['words'] = [el[4] for el in res if '.' not in el[0]]

TONE = pd.DataFrame.from_dict(news)
#TONE = TONE[['date', 'tone', 'source']]
del res


TONE_15_M = pd.DataFrame({'date': [], 'tone': [], 'source': [], 'org': [], 'words': []})

for idx, el in enumerate(list(set(TONE.date))):
    one_p_tones = TONE.loc[TONE.date==el].tone
    one_p_words = TONE.loc[TONE.date==el].words
    one_p_source = TONE.loc[TONE.date==el].source.values
    #one_p_org = TONE.loc[TONE.date==el].org.values
    TONE_15_M.loc[idx] = pd.Series({"date": el, "tone": sum(one_p_tones)/len(one_p_tones), "source": one_p_source , "words": one_p_words})

TONE_15_M = TONE_15_M.sort_values(by=['date'])
TONE_15_M.head()
import numpy as np
df_tone = TONE_15_M

start = datetime.datetime.strptime("2016-02-01 09:00:00", '%Y-%m-%d %H:%M:%S')
finish = datetime.datetime.strptime("2016-02-28 18:15:00", '%Y-%m-%d %H:%M:%S')

step = datetime.timedelta(minutes=15)
t = start

time = list()
while t < finish:
    time.append(t)
    t+=step

    
df_tone = df_tone[(df_tone['date'] >= start) & (df_tone['date'] < finish)]
days = intraday[(intraday['date'] >= start) & (intraday['date'] < finish)]

day = pd.DataFrame({'date':[], 'price': []})
f = start

idx = 0
while f < finish:
    try:
        day.loc[idx] = pd.Series({'date': f, 'price': days.loc[days.date == f].values[0][1]})
        
    except IndexError: # if no value found here: days.loc[days.date == f].values[0][1]

        day.loc[idx] = pd.Series({'date': f, 'price': np.nan})

    idx+=1
    f+=step


lack_tone = list(set(time) - set(list(df_tone['date'])))

for x in lack_tone:
    df_tone = df_tone.append({'date': x, 'tone': 0.0}, ignore_index=True)
   



df_tone = df_tone.sort_values(by=['date'])  
day = day.sort_values(by=['date'])



day = day.dropna()
df_tone = df_tone[df_tone.date.isin(day.date)]


f1, ax1 = plt.subplots(figsize = (15,5))

ax1.set_ylabel('tone', color='blue')
ax1.plot(list(range(len(day['price']))), df_tone['tone'], color = 'blue')
ax1.tick_params(axis='y', labelcolor='blue')
ax1.set_xlabel('time', color='green')

ax2 = ax1.twinx()  
ax2.set_ylabel('price', color='red')
ax2.plot(list(range(len(day['price']))), day['price'], color='red')
ax2.tick_params(axis='y', labelcolor='red')

print(len(day['price']))
print(len(df_tone['tone']))<jupyter_output>475
475<jupyter_text>CORRELATIONS EXPERIMENTS<jupyter_code>day = day.reset_index()
df_tone = df_tone.reset_index()

pear_cor = df_tone.tone.corr(day.price)
print( "(PEARSON CORRELATION){}% of variation in Price could be explained by Tone".format(round(pear_cor**2*100, 2)))

pear_cor = df_tone.tone.corr(np.log(day.price))
print( "\n(PEARSON CORRELATION){}% of variation in Log Price could be explained by Tone".format(round(pear_cor**2*100, 2)))
day = day.reset_index()
df_tone = df_tone.reset_index()


price_normalized = pd.Series(day.price - sum(day.price)/len(day))
day.insert(loc=0, column='price_normalized', value=price_normalized)

from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler(feature_range = (0, 10))

price_transformed = pd.Series(scaler.fit_transform(day.price.values.reshape(-1, 1)).reshape(-1))
day['price_transformed'] = price_transformed


tone_normalized = pd.Series(df_tone.tone - sum(df_tone.tone)/len(df_tone))
df_tone.insert(loc=0, column='tone_normalized', value=tone_normalized)
matplotlib.rcParams['figure.figsize'] = (20.0, 10.0)
plt.plot(#list(range(len(day))), day.price, 'red',  #  to gross
        list(range(len(day))), day.price_transformed, 'orange',
        #list(range(len(day))), day.price_normalized, 'yellow', # to gross
        list(range(len(day.price))), df_tone.tone, 'green')
        #list(range(len(day.price))), df_tone.tone_normalized, 'blue') # no need<jupyter_output><empty_output><jupyter_text>Tone:    Normalization doesn't give significant results:  mean = 0.0166    Zero to One transformation: no need Price:    Normalization : to gross    Zero to ten transformation: consider to be important ( aligned with the tone range)<jupyter_code>pear_cor = df_tone.tone.corr(day.price_transformed)
print( "(PEARSON CORRELATION){}% of variation in Price could be explained by Tone".format(round(pear_cor**2*100, 2)))
price_train, price_test = train_test_split(day.price_transformed, test_size=0.3)
tone_train, tone_test = train_test_split(df_tone.tone, test_size=0.3)

lin_reg = LinearRegression()
lin_reg.fit(pd.DataFrame({'tone' : tone_train}), price_train)

plt.plot(list(range(len(price_test))), price_test, 'r--', range(len(price_test)), lin_reg.predict(pd.DataFrame({'tone' : tone_test})))<jupyter_output><empty_output><jupyter_text>Volatility: 1 - grow,  0 - decline<jupyter_code>price_fluc = day.price_transformed
price_fluc = price_fluc.diff(1)[1:]
price_fluc = [1 if el > 0 else 0 for el in price_fluc]

tone_fluc = df_tone.tone
tone_fluc = tone_fluc.diff(1)[1:]
tone_fluc = [1 if el > 0 else 0 for el in tone_fluc]


from scipy.stats.stats import pearsonr
pear_cor = pearsonr(tone_fluc, price_fluc)
print( "(PEARSON CORRELATION){}% of variation in Price could be explained by Tone".format(round(pear_cor[1]**2*100, 2)))
log_reg = LogisticRegression()

price_train, price_test = train_test_split(price_fluc, test_size=0.3)
tone_train, tone_test = train_test_split(tone_fluc, test_size=0.3)
log_reg.fit(pd.DataFrame({'tone' : tone_train}), price_train)
print(accuracy_score(log_reg.predict(pd.DataFrame({'tone':tone_test})), price_test))<jupyter_output>0.48951048951<jupyter_text>Check stationarity<jupyter_code>from statsmodels.tsa.stattools import adfuller
dicky_fuller_test = adfuller(day.price_transformed)
dicky_fuller_test
"The Dicky-Fuller shows p-value: {} in relation to critical values:  {}".format(round(dicky_fuller_test[1], 2), dicky_fuller_test[4])
dicky_fuller_test = adfuller(df_tone.tone)
dicky_fuller_test
"The Dicky-Fuller shows p-value: {} in relation to critical values:  {}".format(round(dicky_fuller_test[1], 2), dicky_fuller_test[4])<jupyter_output><empty_output><jupyter_text>The tone and price are both unit roots based on Dicky-Fuller test   ***   Feature generation<jupyter_code>df_tone.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16088998211091235, 'fraction_numerical': 0.024932915921288015, 'mean_word_length': 6.535804549283909, 'pattern_counts': {'":': 4, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '607918', 'license': 'MIT', 'n_tokens_mistral': 3665, 'n_tokens_neox': 3410, 'n_words': 625, 'path': 'OtherScripts/DataAnalysis/Stock_correlations.ipynb', 'repo_name': 'ostapkharysh/Stock-return-predictability', 'size': '8944'}