<jupyter_start><jupyter_text>Case study 1 Tesla stock prediction<jupyter_code>import numpy as np
import pandas as pd
from pycaret.regression import *
import pandas as pd
import yfinance as yf

# Download additional stock data, which is correlated to Tesla stock, after research we found that NASDAQ is very correlated

tesla_df = yf.download(['TSLA'], 
                      start='2013-01-02', 
                      end='2021-06-30', 
                      progress=False).reset_index()
nasdq_df = yf.download(['^IXIC'], 
                      start='2013-01-02', 
                      end='2021-06-30', 
                      progress=False).reset_index()
events = pd.read_excel('tesla_tweets.xlsx', sheet_name='tesla_tweets')[['Date', 'stock%']]
events = events[events['stock%']!=0]
nasdq_df.columns = ['Date', 'NSDQ-Open', 'NSDQ-High', 'NSDQ-Low', 'NSDQ-Close', 'NSDQ-Adj Close', 'NSDQ-Volume']
tesla_df.columns = ['Date', 'tesla-Open', 'tesla-High', 'tesla-Low', 'tesla-Close', 'tesla-Adj Close', 'tesla-Volume']
event_tesla_merge = pd.merge(tesla_df, events, how='left', on='Date')
data = pd.merge(event_tesla_merge, nasdq_df, how='left', on='Date')
data['stock%']=data['stock%'].fillna(0)
def date_extraction(dataset):
    dataset['Date'] = pd.to_datetime(dataset['Date'])
    dataset['year'] = pd.DatetimeIndex(dataset['Date']).year
    dataset['month'] = pd.DatetimeIndex(dataset['Date']).month
    dataset['day_of_year'] = dataset.Date.dt.dayofyear
    dataset['day_of_week'] = dataset.Date.dt.dayofweek

    # dataset.drop(columns='Date', inplace=True)
date_extraction(data)<jupyter_output><empty_output><jupyter_text>1. Initial Data UnderstandingHere, we get to know our data and try to figure out next steps for modelling.A goal in this step is to get to know what types of data cleaning, preparation and encoding we need to do in order to prepare our data for inclusion in a model.Since we are predicting price, this is a supervised regression problem.  visualization<jupyter_code># plot the data and MA
import plotly.express as px
fig = px.line(data, x="Date", y=["tesla-Close", "NSDQ-Close"], template = 'plotly_dark')
fig.show()<jupyter_output><empty_output><jupyter_text>running preprocessing on our data  Model pipeline<jupyter_code># split data into train-test set
train = data[data['Date'] < '2021-06-15'][['year', 'month', 'day_of_year', 'day_of_week','tesla-Close']]
test = data[data['Date'] >= '2021-06-15'][['year', 'month', 'day_of_year', 'day_of_week','tesla-Close']]
# check shape
train.shape, test.shape

train.columns
# import the regression module
from pycaret.regression import *
# initialize setup
s = setup(data = train, train_size=0.8, test_data = test, target = 'tesla-Close', 
    fold_strategy = 'timeseries', numeric_features = ['year', 'month', 'day_of_year', 'day_of_week'], 
    feature_interaction=True, 
    ignore_low_variance=True,
    remove_multicollinearity=True, multicollinearity_threshold=0.9, 
    fold = 3, transform_target = True, session_id = 123)


    
best = compare_models(sort = 'MAE')
future_dates = pd.date_range(start = '2021-06-15', end = '2021-06-29', freq = 'D')
future_df = pd.DataFrame()
future_df['month'] = [i.month for i in future_dates]
future_df['year'] = [i.year for i in future_dates] 
future_df['day_of_week'] = [i.dayofweek for i in future_dates]
future_df['day_of_year'] = [i.dayofyear for i in future_dates]
xgboost = create_model('xgboost', fold = 10)
final_best = finalize_model(xgboost)
predictions_future = predict_model(xgboost, data=future_df)
pd.merge(test, predictions_future, how='inner', on=['year', 'month', 'day_of_week', 'day_of_year'])[['tesla-Close', 'Label']]
import matplotlib.pyplot as plt

total_data_w_test = pd.merge(data, predictions_future, how='outer', on=['year', 'month', 'day_of_week', 'day_of_year'])
total_data_w_test.plot(x='Date', y=['tesla-Close', 'Label'])
plt.xlim(['2021-6-15', '2021-07-05'])
plt.ylim([500, 800])
# line plot
import plotly.graph_objects as go
fig = go.Figure(px.line(total_data_w_test, x='Date', y=["tesla-Close", "Label"], template = 'plotly_dark'), layout_yaxis_range=[500,1000], layout_xaxis_range=['2021-06-01', '2021-07-05'])
# add a vertical rectange for test-set separation
fig.add_vrect(x0='2021-6-15', x1='2021-06-29', fillcolor="grey", opacity=0.25, line_width=0)
fig.show()<jupyter_output><empty_output><jupyter_text>Feature importance<jupyter_code>plot_model(xgboost, 'feature')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15581499775482713, 'fraction_numerical': 0.03210597215985631, 'mean_word_length': 5.301272984441301, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '547310', 'license': 'MIT', 'n_tokens_mistral': 1670, 'n_tokens_neox': 1558, 'n_words': 346, 'path': 'approach1.ipynb', 'repo_name': 'msalehsaudi/Tesla-predictions', 'size': '4454'}