<jupyter_start><jupyter_text><jupyter_code>import re
import nltk
import string
import warnings
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline
pd.set_option("display.max_colwidth",200)
warnings.filterwarnings("ignore", category= DeprecationWarning)
train = pd.read_csv('/content/train_E6oV3lV.csv')
test = pd.read_csv('/content/test_tweets_anuFYb8.csv')
train.head(5)
test.isnull().any()
train = train.fillna(method='ffill')
test = test.fillna(method='ffill')
train[train['label']==0].head(10)
train[train['label']==1].head(10)
train.shape
test.shape
train['label'].value_counts()
l_train = train['tweet'].str.len()
le_test = test['tweet'].str.len()
plt.hist(l_train, bins=20, label='Train Tweeets')
plt.hist(le_test, bins=20, label='Test Tweets')
plt.legend()
plt.show()
combi = train.append(test, ignore_index=True)
combi.shape
#combi['label'] = combi.groupby('tweet')['label'].ffill()
#combi=combi.dropna()
#combi = combi.reset_index(drop =True)
def remove_pattern(input_txt,pattern):
    r= re.findall(pattern, input_txt)
    for i in r:
        input_txt=re.sub(i, '', input_txt)
    return input_txt
combi['tidy_tweet']= combi['tweet'].str.replace("[^a-zA-Z#]"," ")
combi.head(10)
combi['tidy_tweet1']=np.vectorize(remove_pattern)(combi['tweet'], "@[\w]*" )
combi.head(5)
combi['tidy_tweet1']= combi['tidy_tweet1'].str.replace("[^a-zA-Z#]"," ")
combi.head(10)
combi['tidy_tweet1']= combi['tidy_tweet1'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))
combi.head(10)
token_tweet= combi['tidy_tweet1'].apply(lambda x: x.split())
token_tweet.head()
from nltk.stem.porter import *
stemmer= PorterStemmer()
token_tweet = token_tweet.apply (lambda x: [stemmer.stem(i) for i in x])
for i in range (len(token_tweet)):
    token_tweet[i]= ' '.join(token_tweet[i])
combi['tidy_tweet1']= token_tweet
combi.tail(1000)
#all_word= ' '.join([text for text in combi['tidy_tweet1']]) 
#import WordCloud 
#wordcloud= WordCloud(width=800, height=500, rando_state=21, max_font_size= 110).generate(all_word)
def hashtag_extract(x):
    hashtags=[]
    for i in x:
        ht=re.findall(r"#(\w+)",i)
        hashtags.append(ht)
    return hashtags
HT_regular= hashtag_extract(combi['tidy_tweet1'][combi['label']==0])
HT_negative= hashtag_extract(combi['tidy_tweet1'][combi['label']==1])
HT_regular = sum(HT_regular, [])
HT_negative = sum(HT_negative,[])

HT_regular

a= nltk.FreqDist(HT_regular)
d= pd.DataFrame({'Hashtag': list(a.keys()),'Count': list(a.values())})
d=d.nlargest(columns="Count", n= 20)
plt.figure(figsize=(20,5))
ax=sns.barplot(data=d, x ="Hashtag", y="Count")
ax.set(ylabel="Count")
plt.show()
b= nltk.FreqDist(HT_negative)
e= pd.DataFrame({'Hashtag': list(b.keys()),'Count': list(b.values())})
e=e.nlargest(columns="Count", n= 20)
plt.figure(figsize=(20,5))
ax=sns.barplot(data=d, x ="Hashtag", y="Count")
ax.set(ylabel="Count")
plt.show()
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
import gensim
bow_vectorizer= CountVectorizer(max_df=0.90, min_df=2, max_features=1000, stop_words='english')
bow= bow_vectorizer.fit_transform(combi['tidy_tweet1'])
bow.shape
tfidf_vect= TfidfVectorizer(max_df=0.90 , min_df=2, max_features=1000, stop_words='english')
tfidf= tfidf_vect.fit_transform(combi['tidy_tweet1'])
tfidf.shape
tokenized_tweet=combi['tidy_tweet1'].apply(lambda x: x.split())
model_w2v= gensim.models.Word2Vec(
            tokenized_tweet,
            size=200,
            window=5,
            min_count=2,
            sg=1,
            hs=0,
            negative=10,
            workers=2,
            seed=34)
model_w2v.wv.most_similar(positive="love")
model_w2v['love']
len(model_w2v['love'])
def word_vector(tokens,size):
  vec=np.zeros(size).reshape((1,size))
  count=0.
  for word in tokens:
    try:
      vec+= model_w2v[word].reshape((1,size))
      count+=1.
    except KeyError:
      if count !=0:
        vec/= count
      return vec
wordvec_arrays= np.zeros((len(tokenized_tweet), 200))
for i in range(len(tokenized_tweet)):
  wordvec_arrays[i,:]= word_vector(tokenized_tweet[i],200)
  wordvec_df=pd.DataFrame(wordvec_arrays)

wordvec_df.shape
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
train_bow=bow[:31962,:]
test_bow=bow[31962:,:]
xtrain_bow, xvalid_bow, ytrain, yvalid= train_test_split(train_bow, train['label'], random_state=42, test_size=0.3)
lreg=LogisticRegression()

#lreg.fit(xtrain_bow,ytrain)
#prediction= lreg.predict_proba(test_bow)
#test_pred_int= test_pred[:,1]>=0.3
#test_pred_int= test_pred_int.astype(np.int)
#test['label']= test_pred_int
#submission=test[['id','label']]
#submission.to_csv('sub.csv', index=False)
import math
wordvec_df.head()
wordvec_df.isnull()
wordvec_df.fillna(method='bfill')
train_tfidf= tfidf[:31962,:]
test_tfidf = tfidf[31962:,:]
xtrain_tfidf=train_tfidf[ytrain.index]
xvalid_tfidf= train_tfidf[yvalid.index]
#train_tfidf=np.ceil(train_tfidf)
lreg.fit(xtrain_tfidf,ytrain)
prediction = lreg.predict_proba(xvalid_tfidf)
prediction_int=prediction[:,1]>=0.3
prediction_int=prediction_int.astype(np.int)
f1_score(yvalid,prediction_int)
print(test_tfidf)
train_w2v = wordvec_df.iloc[:49158,:]
test_w2v = wordvec_df.iloc[49158:,:]
xtrain_w2v = train_w2v.iloc[ytrain.index,:1]
xvalid_w2v = train_w2v.iloc[yvalid.index,:]
#lreg.fit(xtrain_w2v, ytrain)
#prediction = lreg.predict_proba(xvalid_w2v)
#prediction_int = prediction[:,1] >= 0.3
#prediction_int = prediction_int.astype(np.int)
#f1_score(yvalid, prediction_int)

from sklearn import svm
svc=svm.SVC(kernel='linear', C=1, probability = True).fit(xtrain_tfidf, ytrain)
prediction1=svc.predict_proba(xvalid_tfidf)
prediction_int1=prediction1[:,1]>=0.3
prediction_int1=prediction_int.astype(np.int)
f1_score(yvalid,prediction_int1)
from sklearn.ensemble import RandomForestClassifier
rf= RandomForestClassifier(n_estimators= 400, random_state=11).fit(xtrain_tfidf,ytrain)
prediction2=rf.predict(xvalid_tfidf)
f1_score(yvalid,prediction2)
from xgboost import XGBClassifier
xbg= XGBClassifier(max_depth=6, n_estimators=1000, nthread=3).fit(xtrain_tfidf,ytrain)
prediction3=xbg.predict(xvalid_tfidf)
f1_score(yvalid,prediction3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16197295147175816, 'fraction_numerical': 0.03054892601431981, 'mean_word_length': 7.058974358974359, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '467247', 'license': 'MIT', 'n_tokens_mistral': 2684, 'n_tokens_neox': 2578, 'n_words': 376, 'path': 'sentiment.ipynb', 'repo_name': 'Shashwat1001/Python', 'size': '6285'}