<jupyter_start><jupyter_text>Importing Libraries<jupyter_code>import pandas as pd
from collections import Counter<jupyter_output><empty_output><jupyter_text>Loading Data and Preprocessing<jupyter_code>NEWS_DATA = "FINAL_DATA/predicted_news.csv"
INDICATOR_DATA = "FINAL_DATA/predicted_indicators.csv"
TWEETS_DATA = "FINAL_DATA/predicted_tweets.csv"
news_df = pd.read_csv(NEWS_DATA, sep="~")
indicator_df = pd.read_csv(INDICATOR_DATA, sep="~")
tweets_df = pd.read_csv(TWEETS_DATA, sep="~")
all_news_data = []
for x in news_df.groupby(["date", "Symbol"]).groups:
    all_news_data.append(
        {
            "date": x[0],
            "Symbol": x[1]
        }
    )
for i,pred in enumerate(news_df.groupby(["date", "Symbol"])['prediction'].agg(lambda x: pd.Series.mode(x)[0]).values):
    all_news_data[i].update({"news_pred": pred})
refined_news_data = pd.DataFrame(all_news_data)
refined_news_data.head()
indicator_df.head()
tweets_df = tweets_df.drop(["index", ], axis=1)
tweets_df = tweets_df[tweets_df.date >= "2019-01-01"]
all_tweets_data = []
for x in tweets_df.groupby(["date", "Symbol"]).groups:
    all_tweets_data.append(
        {
            "date": x[0],
            "Symbol": x[1]
        }
    )
for i,pred in enumerate(tweets_df.groupby(["date", "Symbol"])['change_prediction'].agg(lambda x: pd.Series.mode(x)[0]).values):
    all_tweets_data[i].update({"tweets_pred": pred})
refined_tweets_data = pd.DataFrame(all_tweets_data)
refined_tweets_data.head()<jupyter_output><empty_output><jupyter_text>Combining all data<jupyter_code>combined_data = indicator_df.merge(refined_news_data, right_on=["date", "Symbol"], left_on=["Date", "Symbol"])
combined_data = combined_data.merge(refined_tweets_data, left_on=["Date", "Symbol"], right_on=["date", "Symbol"])
combined_data = combined_data.drop(["date_x", "date_y"], axis=1)
combined_data.head(n=6)<jupyter_output><empty_output><jupyter_text>Classification  Fixed Weights<jupyter_code>combined_data["total_pred"] = 0.5*combined_data["indicators_prediction"] + 0.25*combined_data["news_pred"] + 0.25*combined_data["tweets_pred"]
combined_data["total_pred"] = [1 if x > 0.6 else 0 for x in combined_data["total_pred"].values]
Counter(combined_data["close_price_change"] == combined_data["total_pred"])
accuracy_fixed_weights = 78.75
from sklearn.metrics import confusion_matrix
tn, fp, fn, tp = confusion_matrix(combined_data["close_price_change"], combined_data["total_pred"]).ravel()
print(tn, fp, fn, tp)<jupyter_output>39 2 15 24
{'fraction_non_alphanumeric': 0.1565008025682183, 'fraction_numerical': 0.016051364365971106, 'mean_word_length': 6.790625, 'pattern_counts': {'":': 6, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '317393', 'license': 'MIT', 'n_tokens_mistral': 984, 'n_tokens_neox': 951, 'n_words': 134, 'path': 'Experiments/Combing all three.ipynb', 'repo_name': 'guptachetan1997/BTP-Stock-Analyser', 'size': '2492'}