<jupyter_start><jupyter_text>Sentiment Analysis- Use summarized S-1 raw filings- Add Sentimnet Analysis features - Tokenize sentences - Use positive and negative sentences<jupyter_code>#core
%matplotlib inline
import matplotlib.pyplot as plt

import pandas as pd

import glob
import nasdaq
import ml
from pathlib import Path

#NLP
from nltk.tokenize import sent_tokenize
from nltk.sentiment.vader import SentimentIntensityAnalyzer<jupyter_output><empty_output><jupyter_text>Load Previous Dataset<jupyter_code>#load and sort by date priced
df = pd.read_csv('../Datasets/2 sentiment analysis.csv', index_col='Symbol')
df = df[list(df.columns.drop(['1D', '1W', '1M', '3M'])) + ['1D', '1W', '1M', '3M']]<jupyter_output><empty_output><jupyter_text>Integrating IPO Summaries<jupyter_code>def get_sentiment_df(text):
    '''returns Sentiment Analysis'''
    sid = SentimentIntensityAnalyzer()
    ps_list = []
    sentences = sent_tokenize(text)
    
    for sentence in sentences:
        ps = sid.polarity_scores(sentence)
        ps['sent'] = sentence
        ps_list.append(ps)
    
    df = pd.DataFrame(ps_list)

    return df
def add_sentiment_features(df_sentiment, symbol):
    '''Adds Sentiment Snalysis features, returns DataFrame'''
    
    #filter frames
    df_tmp = pd.DataFrame()

    #sentences features
    df_tmp.at[symbol, 'Summary Mean Sent Len'] = df_sentiment['sent'].str.len().mean()
    df_tmp.at[symbol, 'Summary Sent Count'] = df_sentiment.shape[0]

    #sentiment
    df_sentiment = df_sentiment[['compound', 'neg', 'neu', 'pos']]
    df_tmp = pd.concat([pd.DataFrame(df_sentiment.mean(), columns=[symbol]).T.add_prefix('Summary Signal '), df_tmp], axis=1)
    
    return df_tmp
#add sentiment features
df_sentiment_features = pd.DataFrame()
counter = 0    

for x in df.index:
    try:
        counter += 1
        print('\n( ' + str(counter) + ' / ' + str(df.shape[0]) + ' ) ' + x)

        #check if summary is available
        file_name = "../Summary/" + x + ".txt"
        if Path(file_name).is_file():
            #load data
            with open(file_name, "r", encoding="utf-8") as file:
                text = file.read()

                #get sentiment
                df_sentiment = get_sentiment_df(text)
                df_sentiment_features = pd.concat([df_sentiment_features, add_sentiment_features(df_sentiment, x)], axis=0)
        else:
            print('no S-1 summary for ', x)
    except Exception as e:
        print(x, e)
#combine original data with sentiment features
df1 = pd.concat([df_sentiment_features, df], axis=1)
#save for later improvements
df1.to_csv('../Datasets/3 summarization.csv', index=True, index_label='Symbol')<jupyter_output><empty_output><jupyter_text>df1 = pd.read_csv('../Datasets/3 summarization.csv', index_col='Symbol')<jupyter_code>df1.head()
#quarter and month
months = df1.loc[:, 'Q1':'MKT12'].columns
classes = df1.loc[:, '1D':'3M'].columns
others = df1.columns.difference(months).difference(classes)

#standardize
df1 = pd.concat([ml.standardize(df1[others]), df1[months], df1[classes]], axis=1)
#feature importance
indices = ml.show_feature_importance(df1, '3M')
#feature selection
df1 = ml.select_features(indices.shape[0], indices, df1)
#now run ML flow with sentiment features for summaries
ml.run_ml_flow(df1)<jupyter_output>C:\Users\Void\Anaconda3\lib\site-packages\matplotlib\cbook\deprecation.py:106: MatplotlibDeprecationWarning: Adding an axes using the same arguments as a previous axes currently reuses the earlier instance.  In a future version, a new instance will always be created and returned.  Meanwhile, this warning can be suppressed, and the future behavior ensured, by passing a unique label to each axes instance.
  warnings.warn(message, mplDeprecation, stacklevel=1)
{'fraction_non_alphanumeric': 0.12251392203659507, 'fraction_numerical': 0.012993900822063113, 'mean_word_length': 4.138964577656676, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '71547', 'license': 'MIT', 'n_tokens_mistral': 1262, 'n_tokens_neox': 1225, 'n_words': 304, 'path': 'Notebooks/3 Summarization.ipynb', 'repo_name': 'algonell/IPOMiner', 'size': '3771'}