<jupyter_start><jupyter_text>Twitter Sentiment Analysis Stock Price Prediction - Crawl data - Sentiment Analysis - Predict Stock  Import module<jupyter_code>from textblob import TextBlob
import math
from collections import Counter, defaultdict
import tweepy as tw
import json
import pandas as pd
import re
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from bs4 import BeautifulSoup
%matplotlib inline
from wordcloud import WordCloud
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.metrics import accuracy_score<jupyter_output><empty_output><jupyter_text>Set up token API<jupyter_code>api_key = "HAcIeikl6eRheQP1oKYGDwPx0"
api_secret = "<KEY>"
access_token = "<KEY>"
access_token_secret = "<KEY>"
auth = tw.OAuthHandler(api_key, api_secret)
auth.set_access_token(access_token, access_token_secret)
api = tw.API(auth,wait_on_rate_limit=True)<jupyter_output><empty_output><jupyter_text>Crawl data<jupyter_code>def apiSearch(kw, number_of_tweets, tweets, user_id, created_at):
    for i in tw.Cursor(api.search, q = kw , lang ="en", since = "2022-01-14",tweet_mode = "extended").items(number_of_tweets):
        tweets.append(i.full_text)
        user_id.append(i.user.id)
        created_at.append(i.created_at)
number_of_tweets = 500
tweets = []
user_id =[]
created_at = []
keyword_list = ['News', 'Stock', 'Customers', 'Employees', 'Company', 'Price', 
                'Shareholders', 'Market', 'Soaring', 'Trends']
company = 'Amazon'
final_kw_list = []

# Crawl data from keywords
for i in range (len(keyword_list)):
    kw = str(company + ' ' + keyword_list[i])
    print(kw)
    final_kw_list.append(kw)
    for i in tw.Cursor(api.search, q = kw , lang ="en", since = "2022-01-14",tweet_mode = "extended").items(number_of_tweets):
        tweets.append(i.full_text)
        user_id.append(i.user.id)
        created_at.append(i.created_at)<jupyter_output>Amazon News
Amazon Stock
Amazon Customers
Amazon Employees
Amazon Company
Amazon Price
Amazon Shareholders
Amazon Market
Amazon Soaring
Amazon Trends<jupyter_text>Dataframe<jupyter_code>df = pd.DataFrame({"Created_at": created_at,"User_id": user_id,"Tweets":tweets})
df<jupyter_output><empty_output><jupyter_text>Clean data<jupyter_code>df = df[~df.Tweets.str.contains("RT")]
df = df.reset_index(drop = True)
def cleanUpTweet(text):
    text = re.sub(r'@[A-Za-z0-9_]+','',text) # Remove mentions @
    text = re.sub(r'#', '', text) # Remove hastag symbol
    text = re.sub(r'https?:\/\/[A-Za-z0-9\.\/]+', '', text) # Remove hyper link
    text = re.sub('[^A-Za-z0-9]+', ' ', text) # Remove special characters
    #Remove stop words
    res = ''
    for word in text.split():
        if word not in stopwords.words('english'):
            res += WordNetLemmatizer().lemmatize(word) + ' '
            
    return res
df['Tweets'] = df['Tweets'].apply(cleanUpTweet)
df = df.drop_duplicates(subset='Tweets')
df = df.reset_index(drop=True)
df<jupyter_output><empty_output><jupyter_text>Create clean data list<jupyter_code>cleaned_data=[]
for tw in df['Tweets']:
    cleaned_data.append(tw)
    
cleaned_data<jupyter_output><empty_output><jupyter_text>WordCloud Visualization<jupyter_code># Plot the word cloud 
allWords = ' '.join( [twts for twts in df['Tweets']] )
positiveWordCloud = WordCloud(max_words = 1000 , width = 1600 , height = 800, collocations=False).generate(allWords)
plt.figure(figsize = (15,15))
plt.imshow(positiveWordCloud, interpolation="bilinear")
plt.axis('off')
plt.show()<jupyter_output><empty_output><jupyter_text>Sentiment Analysis<jupyter_code># Create function to get subjectivity
def getSubjectivity(text):
    return TextBlob(text).sentiment.subjectivity

# Create function to get polarity
def getPolarity(text):
    return TextBlob(text).sentiment.polarity

# Function compute negative, positive
def getAnalysis(score):
    if score < 0: 
        return np.int_(0)
    elif score > 0:
        return np.int_(1)
    else:
        return 'Neutral'
    
df['Subjectivity'] = df['Tweets'].apply(getSubjectivity)
df['Polarity'] = df['Tweets'].apply(getPolarity)
df['Sentiment'] = df['Polarity'].apply(getAnalysis)
df.drop(df.loc[df['Sentiment']=='Neutral'].index, inplace=True)
df = df.reset_index(drop=True)
df
# Save as CSV file
df.to_csv('Amazon.csv')
# Display 5 positive tweets
j = 0
sortedDF = df.sort_values(by=['Polarity'])
positiveTweets = ''
for i in range(sortedDF.shape[0]):
    if sortedDF['Sentiment'][i] == 1:
        print(str(j) + ') ' + sortedDF['Tweets'][i])
        positiveTweets += (' ' + sortedDF['Tweets'][i])
        print()
        j+=1
    if j == 5:
        break


# Display 5 negative tweets
j = 0
sortedDF = df.sort_values(by=['Polarity'])
negativeTweets = ''
for i in range(sortedDF.shape[0]):
    if sortedDF['Sentiment'][i] == 0:
        print(str(j) + ') ' + sortedDF['Tweets'][i] )
        negativeTweets += (' ' + sortedDF['Tweets'][i])
        print()
        j+=1
    if j == 5:
        break
# Plot popularity and subjectivity
plt.figure(figsize=(8,6))
for i in range(df.shape[0]):
    plt.scatter(df['Polarity'][i], df['Subjectivity'][i], color='Blue')
    
plt.title('Sentiment Analysis')
plt.xlabel('Polarity')
plt.ylabel('Subjectivity')
plt.grid(True)
plt.show()<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>class Tokenizer:
  
  def clean(self, text):
      no_html = BeautifulSoup(text).get_text()
      clean = re.sub("[^a-z\s]+", " ", no_html, flags=re.IGNORECASE)
      return re.sub("(\s+)", " ", clean)

 
  def tokenize(self, text):
      clean = self.clean(text).lower()
      stopwords_en = stopwords.words("english")
      return [w for w in re.split("\W+", clean) if not w in stopwords_en]
class MultinomialNaiveBayes:
  
    def __init__(self, classes, tokenizer):
      self.tokenizer = tokenizer
      self.classes = classes
      
    def group_by_class(self, X, y):
      data = dict()
      for c in self.classes:
        data[c] = X[np.where(y == c)]
      return data
           
    def fit(self, X, y):
        self.n_class_items = {}
        self.log_class_priors = {}
        self.word_counts = {}
        self.vocab = set()

        n = len(X)
        
        grouped_data = self.group_by_class(X, y)
        
        for c, data in grouped_data.items():
          self.n_class_items[c] = len(data)
          self.log_class_priors[c] = math.log(self.n_class_items[c] / n)
          self.word_counts[c] = defaultdict(lambda: 0)
          
          for text in data:
            counts = Counter(self.tokenizer.tokenize(text))
            for word, count in counts.items():
                if word not in self.vocab:
                    self.vocab.add(word)

                self.word_counts[c][word] += count
                
        return self
      
    def laplace_smoothing(self, word, text_class):
      num = self.word_counts[text_class][word] + 1
      denom = self.n_class_items[text_class] + len(self.vocab)
      return math.log(num / denom)
      
    def predict(self, X):
        result = []
        for text in X:
          
          class_scores = {c: self.log_class_priors[c] for c in self.classes}

          words = set(self.tokenizer.tokenize(text))
          for word in words:
              if word not in self.vocab: continue

              for c in self.classes:
                
                log_w_given_c = self.laplace_smoothing(word, c)
                class_scores[c] += log_w_given_c
                
          result.append(max(class_scores, key=class_scores.get))

        return result
X = df['Tweets'].values
y = df['Sentiment'].values.astype('float')
RANDOM_SEED = 1
  
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=RANDOM_SEED)

MNB = MultinomialNaiveBayes(
    classes=np.unique(y), 
    tokenizer=Tokenizer()
).fit(X_train, y_train)

y_pred = np.array(MNB.predict(X_test))
print(y_test)
print(y_pred)
accuracy_score(y_test, y_pred)
print(classification_report(y_test, y_pred))
cnf_matrix = confusion_matrix(y_test, y_pred)
cnf_matrix
class_names = ["negative", "positive"]
fig,ax = plt.subplots()


sns.heatmap(pd.DataFrame(cnf_matrix), annot=True, cmap="Blues", fmt="d", cbar=False, xticklabels=class_names, yticklabels=class_names)
ax.xaxis.set_label_position('top')
plt.tight_layout()
plt.ylabel('Actual sentiment')
plt.xlabel('Predicted sentiment');<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13260819254802128, 'fraction_numerical': 0.007868548947003009, 'mean_word_length': 3.692182410423453, 'pattern_counts': {'":': 3, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '38974', 'license': 'MIT', 'n_tokens_mistral': 3156, 'n_tokens_neox': 3026, 'n_words': 617, 'path': 'Twitter Sentiment Analysis Stock Price Prediction .ipynb', 'repo_name': 'tnathu-ai/LSTM-based-sentiment-analysis-for-stock-price-forecast', 'size': '8772'}