<jupyter_start><jupyter_text>Data Collecting<jupyter_code># loading news data scrapted by SNA api
df_news = pd.read_csv('./data/apple_news_raw.csv.gz')
# loading articles list previously scraped by scrapy.py we include in ./lib
articles = load('./data/articles.pkl')
df_news['articles'] = articles
df_news.head(3)<jupyter_output><empty_output><jupyter_text>Data Scrubbing<jupyter_code># drop unnecessary columns
cols_to_drop = ['news_url', 'image_url', 'title', 'text', 'topics', 'type', 'tickers']
df_news.drop(cols_to_drop, axis=1, inplace=True)
df_news
miss_articles = df_news['articles'].isna()
print(f"Total Missing Articles: {miss_articles.sum()}")<jupyter_output>Total Missing Articles: 923<jupyter_text>As result of scraping 10,450 websites we got 91.13% of successful scraped articles, but we also end up missing  926 websites. this probably due to websites restriction like cookies and CAPTCHA Blocking,  IP Blocking & Proxy Servers, etc..those 926 row are going to be droped from our dataframe<jupyter_code>df_news.drop(df_news.loc[miss_articles].index, inplace=True)
df_news.reset_index(drop=True, inplace=True)<jupyter_output><empty_output><jupyter_text>DateTime<jupyter_code>df_news.loc[:5,'date']
df_news['date'].str.findall("\s-[0-9]{4}$").apply(lambda x: ''.join(x)).unique()
# we found dates end with noise strings like '-0400' and '-0500', those are going to be removed using
# builin pandas RegEx.
df_news['date'] = df_news['date'].str.replace("\s-[0-9]{4}$", '', regex=True)
# convert date from string date format to DateTime object using standard ISO-8601 formated YYYY-MM-DDTHH:MM:SS.
df_news['date'] = pd.to_datetime(df_news['date'], format="%a, %d %b %Y %H:%M:%S")
df_news.loc[:5,'date']
df_news['date'] = pd.to_datetime(df_news['date']).dt.normalize()
df_news.rename(columns={'source_name':'source'}, inplace=True)<jupyter_output><empty_output><jupyter_text>Using langdetect in order to indentify articles language, ensure articles are only english<jupyter_code>df_news.reset_index(drop=True, inplace=True)
df_news.head()<jupyter_output><empty_output><jupyter_text>PREPROCESSING In this section we start by preparing our data for modeling, first by cleaning and tokenizing the articles text applying the function 'text_tokenize' and lastely evaluate the sentiment analysis applying 'polarity_score function. Text TonkenizationRemove punctuation, special characters, numbers<jupyter_code>prep = NLP_stat()
df_news['articles'] = df_news['articles'].apply(lambda txt: txt.lower())
stop_words=stopwords.words('english')
df_news['articles'] = df_news['articles']\
                            .apply(lambda txt: ' '.join([word for word in txt.split() if word not in stop_words]))
df_news['articles'] = df_news['articles'].apply(lambda txt: sent_tokenize(txt))
df_news['articles'] = df_news['articles'].apply(lambda x: ' '.join(x))
word_tokenizer = RegexpTokenizer('[a-zA-Z]+')
df_news['articles'] = df_news['articles'].apply(lambda txt: word_tokenizer.tokenize(txt))
df_news['articles'] = df_news['articles'].apply(lambda txt: ' '.join(txt))
single_chars = re.compile(r"(^\w{1})(?=\s)|(?<=\s)[\w]{1}(?=\s)")
df_news['articles'] = df_news['articles'].apply(lambda txt: single_chars.sub('', txt))
articles_to_drop = df_news[(df_news['articles'].apply(lambda x: len(x.split()) <= 10))].index
df_news.drop(articles_to_drop, inplace=True)
df_news.reset_index(drop=True, inplace=True)
df_news.to_csv('./data/apple_news_cleaned.csv.gz', compression="gzip")<jupyter_output><empty_output><jupyter_text>Features Engineering<jupyter_code>df_news = pd.read_csv('./data/apple_news_cleaned.csv.gz', index_col=0)
prep = NLP_stat()
df_score = prep.polarity_score(df_news['articles'])
df_news = pd.concat([df_news.drop('articles', axis=1), df_score], axis=1, ignore_index=False, sort=False)
df_news.rename(columns={"text":"articles"}, inplace=True)
# create a Frequency Distribution of postive words
positive_fdist_df = prep.get_freq_dist(
    prep.get_word_list( df_news[(df_news['sentiment'] == 'Positive')].loc[:,'articles'] )
) 
# create a Frequency Distribution of negative words
negative_fdist_df = prep.get_freq_dist(
    prep.get_word_list( df_news[(df_news['sentiment'] == 'Negative')].loc[:,'articles'] )
) 
# create a Frequency Distribution of neutral words
neutral_fdist_df = prep.get_freq_dist(
    prep.get_word_list( df_news[(df_news['sentiment'] == 'Neutral')].loc[:,'articles'] )
) 
# create a Frequency Distribution of all words
fdist_df = prep.get_freq_dist(prep.get_word_list(df_news['articles']))
df_news['sentiment'].value_counts()
df_news.reset_index(drop=True, inplace=True)
df_news['word_count'] = prep.WordsCount(df_news['articles'])<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis<jupyter_code>df_news.head()
sentiment = df_news['sentiment'].value_counts()
fig1 = px.bar(sentiment,
              x=sentiment.index,
              y='sentiment',
              color='sentiment'
             )
fig1.update_layout(
            height=400,
            width=700,
            margin={"r":0,"t":50,"l":0,"b":0},
            paper_bgcolor='white',
            title='Sentiment Distribution',
            xaxis_zeroline=False, 
            yaxis_zeroline=False,
    xaxis=dict(
            title='Sentiment',
            linecolor=None,
            titlefont_size=15,
            tickfont_size=15,
            showgrid=False
            ),
    yaxis=dict(
            title='Sentiment Counts',
            linecolor=None,
            titlefont_size=15,
            tickfont_size=15,
            showgrid=False
            ),
    legend=dict(
            x=0,
            y=0,
            bgcolor='rgba(255, 255, 255, 0)',
            bordercolor='rgba(255, 255, 255, 0)'
            )
                    )
iplot(fig1)
source_top10 = df_news['source'].value_counts(sort=True).nlargest(10)
fig2 = px.pie(source_top10,
              values='source',
              names=source_top10.index,
              title='Top 10 news source',
              color_discrete_sequence=px.colors.sequential.RdBu,
              width=800,
              height=600,
              labels=source_top10.index
             )

fig2.update_traces(hoverinfo='label+percent',
                   textfont_size=12,
                   textposition='inside',
                   textinfo='percent+label'
                  )
iplot(fig2)
sent_vs_source_df = (
    df_news.groupby(['sentiment','source'])
    .size().reset_index()
    .rename(columns={0:'count'})
    .query("count > 50")
                    )
fig3 = px.bar(
    sent_vs_source_df,
    x='source',
    y='count',
    text='count',
    color='sentiment',
    title='News Sources vs Sentiment'
)
fig3.update_layout(
    width=1000,
    height=500,
    margin={"r":50,"t":50,"l":50,"b":50},
    uniformtext_minsize=6,

xaxis=dict(
    title='Finance News Source',
    linecolor=None,
    titlefont_size=15,
    tickfont_size=10,
    showgrid=False,
    tickangle=-45
    ),
yaxis=dict(
    title='Sentiment Analysis Counts',
    linecolor=None,
    titlefont_size=15,
    tickfont_size=10,
    showgrid=False
    ),    
legend=dict(
    yanchor="top",
    xanchor="left",
    x=0.89,
    y=0.99,
    bgcolor='rgba(255, 255, 255, 0)',
    bordercolor='rgba(255, 255, 255, 0)'
            )    
                )
fig3.update_traces(texttemplate='%{text:.3s}', textposition='inside')
iplot(fig3)
word_freq_dist = pd.concat(
    [
    (neutral_fdist_df
     .query("frequency > 1000")
     .assign(sentiment='neutral')
     .sample(15, replace=True)),
    (negative_fdist_df
     .query("frequency > 1000")
     .assign(sentiment='negative')
     .sample(15, replace=True)),        
    (positive_fdist_df
     .query("frequency > 1000")
     .assign(sentiment='positive')
     .sample(15, replace=True))
    ],
    ignore_index=True, axis=0
            )
fig3 = px.bar(
    word_freq_dist,
    x='frequency',
    y='word',
    text='frequency',
    color='sentiment',
    title='Words Frequency Distribution'
)
fig3.update_layout(
    width=900,
    height=800,
    margin={"r":50,"t":50,"l":50,"b":50},
    uniformtext_minsize=6,

xaxis=dict(
    title='Frequency Distribution',
    linecolor=None,
    titlefont_size=15,
    tickfont_size=10,
    showgrid=False,
    tickangle=-45
    ),
yaxis=dict(
    title='Words Sample',
    linecolor=None,
    titlefont_size=15,
    tickfont_size=10,
    showgrid=False
    ),    
legend=dict(
    yanchor="top",
    xanchor="right",
    x=1,
    y=1,
    bgcolor='rgba(255, 255, 255, 0)',
    bordercolor='rgba(255, 255, 255, 0)'
            )    
                )
fig3.update_traces(texttemplate='%{text:.3s}', textposition='inside')
iplot(fig3)
mask = load('./data/aapl_mask.pkl')
fig = plt.subplots(figsize=(20,10))
wc = WordCloud(
    mask=mask,
    ranks_only=True,
    background_color="white",
    collocations=True,
    max_words=200,
    max_font_size=100,
    random_state=42,
    width=1200,
    height=800
    )
wc.generate(' '.join(fdist_df.word.to_list()))
plt.imshow(wc, interpolation="bilinear")
plt.axis('off')
plt.savefig('./img/word_cloud.jpg',format="jpg")
plt.show()<jupyter_output><empty_output><jupyter_text>MODELING<jupyter_code>ohe = OneHotEncoder(sparse=False)
y_ohe = ohe.fit_transform(apple_news_df['sentiment'].values.reshape(-1,1))
max_lenght = 100
tokenizer = Tokenizer(num_words=10000)
tokenizer.fit_on_texts(apple_news_df['articles'])
sequences = tokenizer.texts_to_sequences(apple_news_df['articles'])
word_index = tokenizer.word_index
# total vocabulary size plus 0 for unknown words
vocab_size = len(tokenizer.word_index) + 1
data = pad_sequences(sequences, max_lenght)
print(f"{'Unique Words':25} {len(tokenizer.index_word)}")
print(f"{'Shape of data tensor':25} {data.shape}")
print(f"{'Shape of label tensor':25} {y_ohe.shape}")
sentences = apple_news_df['articles'].apply(lambda x: x.split()).to_list()
word2vec = Word2Vec(sentences,
                     vector_size=max_lenght,
                     window=5,
                     min_count=1,
                     workers=4,
                    )
word2vec.train(sentences, total_examples=word2vec.corpus_count, epochs=5)
pretrained_weights = word2vec.wv.vectors
vocab_size, emdedding_size = pretrained_weights.shape
vectors = np.asarray(word2vec.wv.vectors)
len(word_index), len(word2vec.wv.key_to_index)
embedding_matrix = np.zeros((vocab_size+1, emdedding_size))
for word, idx in word_index.items():
        embedding_matrix[idx] = word2vec.wv[word]
dump(embedding_matrix, './word2vec/embedding_matrix.pkl', compress=5)
X_train, X_test, y_train, y_test = train_test_split(data, y_ohe, test_size=.15, random_state=1, stratify=y_ohe)
X_train, X_val, y_train, y_val   = train_test_split(data, y_ohe, test_size=.15, random_state=1, stratify=y_ohe)

print(f"Train\n\t{'X :':5}{X_train.shape}\n\t{'Y :':5}{y_train.shape}")
print(f"Test\n\t{'X :':5}{X_test.shape}\n\t{'Y :':5}{y_test.shape}")
print(f"Validation\n\t{'X :':5}{X_val.shape}\n\t{'Y :':5}{y_val.shape}")
input_dim = embedding_matrix.shape[0]
embedding_dim = embedding_matrix.shape[1]
max_seq_len = data.shape[1]
EPOCHS = 32
BATCH_SIZE = 1024
BIAS = class_weights_ohe(y_ohe)
auc = metrics=keras.metrics.AUC()
METRICS = [auc]
STEPS_x_EPOCH = X_train.shape[0]//BATCH_SIZE
VAL_SPLIT = X_val.shape[0]//BATCH_SIZE
plot_model = Ploty()
eStop_Val_Loss = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=12,
                            verbose=1,mode='min',baseline=0.999,restore_best_weights=False)
eStop_Loss     = EarlyStopping(monitor='loss', min_delta=0.001, patience=8,
                            verbose=1,mode='min',baseline=0.999,restore_best_weights=False)
eStop_Val_AUC = EarlyStopping(monitor='val_auc_1', min_delta=0.001, patience=12,
                            verbose=1, mode='max', baseline=0.999, restore_best_weights=False)
model = make_model(METRICS,
                   input_dim=input_dim,
                   embedding_dim=embedding_dim,
                   weights=embedding_matrix,
                   input_length=max_seq_len,
                  )
model.layers[-1].bias = BIAS
embedding_init =  keras.initializers.zeros()
model.layers[0].embeddings_initializer = embedding_init
model.summary()
history = model.fit(
                    X_train,
                    y_train,
                    validation_data=(X_val, y_val),
                    validation_split=0.1,
                    epochs=EPOCHS,
                    batch_size=BATCH_SIZE,
                    use_multiprocessing = True,
                    workers=8
                    )
model.save('./model/model.h5')
history = model.fit(
                    X_train,
                    y_train,
                    validation_data=(X_val, y_val),
                    validation_split=0.1,
                    epochs=EPOCHS,
                    batch_size=BATCH_SIZE,
                    use_multiprocessing = True,
                    callbacks=[eStop_Val_AUC ,eStop_Val_Loss],
                    workers=8
                    )
score = model.evaluate(X_test, y_test, steps=X_test.shape[0], workers=8, verbose=0)<jupyter_output><empty_output><jupyter_text>Computer AUC Accuracy<jupyter_code>y_pred = model.predict(X_test, batch_size=1,verbose=0)
classes_accurasy = compute_aucs(y_pred, y_test)
model_stat_df = pd.DataFrame({"accuracy":classes_accurasy, 'classes':ohe.categories_[0], 'bias':BIAS})
fig_cls = px.bar(model_stat_df,
                 x='classes',
                 y=['accuracy', 'bias'],
                 labels={'x': 'Class Name', 'y':'Accuracy'},
                 width=500,
                 height=400,
                 title='Model Results',
                 barmode='group',
                )
fig_cls.show()
print('Accuracy :', score[1])
plot_model.confusion_matrix(y_true=y_test, y_pred=y_pred, target_name=ohe.categories_[0], file_name='./img/cm.jpg')
plot_model.plot_roc_curve(y_true=y_test, y_pred=y_pred, file_name='./img/roc.jpg')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14266226306720275, 'fraction_numerical': 0.027067777139574957, 'mean_word_length': 3.544535073409462, 'pattern_counts': {'":': 14, '<': 42, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '280547', 'license': 'MIT', 'n_tokens_mistral': 5090, 'n_tokens_neox': 4747, 'n_words': 757, 'path': 'project.ipynb', 'repo_name': 'akladyous/stock-market-sentiment-analysis', 'size': '13928'}