<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
import tensorflow as tf

from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam

import nltk
from nltk.corpus import twitter_samples
import matplotlib.pyplot as plt
import random 

nltk.download('stopwords')

import re
import string

from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.tokenize import TweetTokenizer
from google.colab import drive
drive.mount('/content/drive')
df = pd.read_csv('/content/drive/MyDrive/nlp-getting-started/train.csv')
df2 = pd.read_csv('/content/drive/MyDrive/nlp-getting-started/test.csv')
df2.head()
tweet_test = df2['text']
tweet = df['text']
label = df['target']
tweet2 = []
for i in tweet:
  i = re.sub('#', '', i)
  i = re.sub(r'https?:\/\/.*[\r\n]*', '', i)
  tweet2.append(i)
label_final = []
for i in label:
  label_final.append(i)
tokenizer = Tokenizer()
tokenizer.fit_on_texts(tweet2)
stopwords_english = stopwords.words('english')
tweets_clean = []

for word in tweet2:
    if (word not in stopwords_english and  # remove stopwords
        word not in string.punctuation):  # remove punctuation
        tweets_clean.append(word)
stemmer = PorterStemmer() 

# Create an empty list to store the stems
tweets_final = [] 

for word in tweets_clean:
    stem_word = stemmer.stem(word)  # stemming word
    tweets_final.append(stem_word) 
def process_tweet(tweet):
  stemmer = PorterStemmer()
  stopwords_english = stopwords.words('english')
  tweet = re.sub(r'\$\w*', '', tweet)
  tweet = re.sub(r'^RT[\s]+', '', tweet)
  tweet = re.sub(r'https?:\/\/.*[\r\n]*', '', tweet)
  tweet = re.sub(r'#', '', tweet)
  tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True,reduce_len=True)
  tweet_tokens = tokenizer.tokenize(tweet)

  tweets_clean = []
  for word in tweet_tokens:
      if (word not in stopwords_english and  
              word not in string.punctuation): 
          stem_word = stemmer.stem(word)  # stemming word
          tweets_clean.append(stem_word)

  return tweets_clean
def build_freqs(tweets, ys):

    yslist = np.squeeze(ys).tolist()

    freqs = {}
    for y, tweet in zip(yslist, tweets):
        for word in process_tweet(tweet):
            pair = (word, y)
            if pair in freqs:
                freqs[pair] += 1
            else:
                freqs[pair] = 1    
    return freqs
freqs = build_freqs(tweets_final, label_final)
keys = ['earthquak', 'forest', 'flood', 'evacu', 'disast', 'accident', 'wildfir', '...', 'peopl']

data = []

for word in keys:
    
    pos = 0
    neg = 0
    
    if (word, 1) in freqs:
        pos = freqs[(word, 1)]
        
    if (word, 0) in freqs:
        neg = freqs[(word, 0)]
        
    data.append([word, pos, neg])
    
data
fig, ax = plt.subplots(figsize = (8, 8))

x = np.log([x[1] + 1 for x in data])  

y = np.log([x[2] + 1 for x in data]) 

ax.scatter(x, y)  

plt.xlabel("Log Positive count")
plt.ylabel("Log Negative count")

for i in range(0, len(data)):
    ax.annotate(data[i][0], (x[i], y[i]), fontsize=12)

ax.plot([0, 9], [0, 9], color = 'red')
plt.show()
print(freqs)
from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer()
train_x = vectorizer.fit_transform(tweets_final).toarray()
test_x = vectorizer.transform(tweet_test).toarray()
lr = 0.004
n_epochs = 12000

weights_vector = np.random.random(train_x.shape[1])

def cost_function(pred,truth):
    return -truth*np.log(pred) - (1-truth)*np.log(1-pred)

def sigmoid(x):
    return 1.0/(1+np.exp(-x))

def lin_mul(x,weights):
    return np.dot(x,weights)

for ep in range(n_epochs):
    avg_cost = 0
    for i in range(len(train_x)):
        data_point = train_x[i]
        label  = label_final[i]
        pred_prob = sigmoid(lin_mul(weights_vector,data_point))
        avg_cost += cost_function(pred_prob,label)
        weights_vector = weights_vector - lr*(pred_prob - label)* data_point
    if ep%100==0:
        print ("Epoch {} has finished. Error is {}".format(ep+1,avg_cost/4.0))
preds = np.where(sigmoid(lin_mul(test_x,weights_vector))>.75,1,0)
print(preds)
id = pd.DataFrame(df2['id'])
preds = pd.DataFrame(preds, columns=['target'])
final = pd.concat([id, preds], axis=1)
final.to_csv('final_submission.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14081092992507713, 'fraction_numerical': 0.012780960775672102, 'mean_word_length': 3.6938986556359876, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '13836', 'license': 'MIT', 'n_tokens_mistral': 1753, 'n_tokens_neox': 1662, 'n_words': 377, 'path': 'NLP (Logistic_Regression) for Twitter Event Prediction.ipynb', 'repo_name': 'roupenminassian/Freelance', 'size': '4538'}