<jupyter_start><jupyter_text>***   *** ***retweet_count***<jupyter_code>print(df1.retweet_count[0])
isinstance(df1.retweet_count[1], (str))
print(df1.retweet_count.nunique())
df1.retweet_count.unique()
rc=df1['retweet_count']
rc
def convertible(v):
    try:
        int(v)
        return True
    except (TypeError, ValueError):
        return False


res = [int(ele) if convertible(ele)  else int(0) for ele in rc] 
res
res=pd.DataFrame(res)
res.head()
print(res[0].nunique())
res[0].unique()
df1['retweet_count']=res[0]
df1['retweet_count'].unique()<jupyter_output><empty_output><jupyter_text>**   ** ***Orginal_author***<jupyter_code>print(df1.original_author.nunique())
df1.original_author.unique()
oa=df1['original_author']
oa
def convertibles(v):
    try:
        float(v)
        return True
    except (TypeError, ValueError):
        return False


res2 = [str('unkn') if convertibles(ele) else ele  for ele in oa] 
res2
res2=pd.DataFrame(res2)

print(res2[0].nunique())
res2[0].unique()
df1['original_author']=res2[0]
df1['original_author'].nunique()<jupyter_output><empty_output><jupyter_text>**   ** **test_data**<jupyter_code>df2=pd.read_csv("test.csv")
print(df2.shape)
df2.head()

print(df2.retweet_count.nunique())
df2.retweet_count.unique()
rct=df2['retweet_count']
rct
def convertible(v):
    try:
        int(v)
        return True
    except (TypeError, ValueError):
        return False


rest = [int(ele) if convertible(ele)  else int(0) for ele in rct] 
rest
rest=pd.DataFrame(rest)
rest.head()
print(rest[0].nunique())
rest[0].unique()
df2['retweet_count']=rest[0]
df2['retweet_count'].unique()<jupyter_output><empty_output><jupyter_text>**    **<jupyter_code>print(df2.original_author.nunique())
#df2.original_author.unique()
oat=df2['original_author']
oat
def convertibles(v):
    try:
        float(v)
        return True
    except (TypeError, ValueError):
        return False


rest2 = [str('unkn') if convertibles(ele) else ele  for ele in oat] 
rest2
rest2=pd.DataFrame(rest2)

print(rest2[0].nunique())
#rest2[0].unique()
df2['original_author']=rest2[0]
df2['original_author'].nunique()<jupyter_output><empty_output><jupyter_text>**    **<jupyter_code>print(df1.head())
df1.info()
print(df2.head())
df2.info()
np.intersect1d([1,2,3,1,4,3,5,7], [1,3,4,6])        #testing
com=np.intersect1d(df1.original_author, df2.original_author)
com.shape         #246 common in(1528-736)<jupyter_output><empty_output><jupyter_text>**   ** ***tweets***<jupyter_code>x1=df1['original_text']
print(x1.shape)
x1.head
x2=df2['original_text']
print(x2.shape)
x2.tail()
x=pd.concat([x1,x2],axis=0)
x=pd.DataFrame(x)
x.reset_index(inplace=True)
x=x['original_text']
x=pd.DataFrame(x)
print(x.shape)
x.tail()<jupyter_output><empty_output><jupyter_text>**Data cleaning**<jupyter_code>def remove_pattern1(input_txt, pattern):
    r = re.findall(pattern, input_txt)
    for i in r:
        input_txt = re.sub(i, '', input_txt)
        
    return input_txt    

# remove twitter handles (@user)
x['tidy_tweet'] = np.vectorize(remove_pattern1)(x['original_text'], "@[\w]*")
x.head()<jupyter_output><empty_output><jupyter_text>**removing links and mentions**<jupyter_code>def remove_pattern(pattern):
  for i in range(x['tidy_tweet'].shape[0]):
    twt=x['tidy_tweet'][i]
    res=twt.split(pattern, maxsplit=1) 
    x['tidy_tweet'][i]=res[0]
remove_pattern('http')
remove_pattern('pic.twitter')
x.head
x['original_text'][7]
x['tidy_tweet'][7]<jupyter_output><empty_output><jupyter_text>**   ** **NLTK**<jupyter_code>import re
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
corpus = []
for i in range(0, 4622):
    review = re.sub('[^a-zA-Z]', ' ',x['tidy_tweet'][i])
    review = review.lower()
    review = review.split()
    ps = PorterStemmer()
    review = [ps.stem(word) for word in review if not word in set(stopwords.words('english'))]
    review = ' '.join(review)
    corpus.append(review)
corpus[2]
x['tidy_tweet'][2]<jupyter_output><empty_output><jupyter_text>**   ** **Embedding Representation**<jupyter_code>from tensorflow.keras.layers import Embedding
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Sequential
from keras import utils
from tensorflow.keras.preprocessing.text import one_hot
from tensorflow.keras.layers import LSTM
from tensorflow.keras.layers import Dense 
from tensorflow.keras.layers import Dropout
voc_size=5000
onehot_repr=[one_hot(words,voc_size)for words in corpus] 
sent_length=40
embedded_docs=pad_sequences(onehot_repr,padding='pre',maxlen=sent_length)
print(embedded_docs)
embedded_docs[10]
import numpy as np
X_=np.array(embedded_docs)
print(X_.shape)<jupyter_output><empty_output><jupyter_text>**   **<jupyter_code>f1=df1[['retweet_count','original_author']]
print(f1.shape)
f1.head()

f2=df2[['retweet_count','original_author']]
print(f2.shape)
f2.head()
f=pd.concat([f1,f2],axis=0)
f.reset_index(inplace=True)
f=f.drop("index",axis=1)
print(f.shape)
f.head()
f=pd.get_dummies(f, prefix=['original_author'])
print(f.shape)
X_pd=pd.DataFrame(X_)
X=pd.concat([X_pd,f],axis=1)
X.reset_index(inplace=True)
X=X.drop("index",axis=1)
print(X.shape)
X.tail()
xtrain=X.iloc[:3235,: ].values
xtest=X.iloc[3235:,: ].values
print(xtrain.shape)
xtest.shape
from sklearn.model_selection import train_test_split
X_train, X_val, y_train, y_val  = train_test_split(xtrain,y ,test_size=0.15, random_state=0)<jupyter_output><empty_output><jupyter_text>**svm**<jupyter_code>from sklearn.svm import SVC
classifier1 = SVC(kernel = 'poly', random_state = 0, degree=4,C=10)
classifier1.fit(X_train, y_train)

from sklearn.model_selection import cross_val_score
acc=cross_val_score(estimator=classifier1, X=X_train, y=y_train , cv=10)

print(acc.mean())
print(acc.std())
acc

y_pred1 = classifier1.predict(X_val)
y_pred1.shape
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_val, y_pred1)
cm
array([[ 21,  89,   7], 
       [ 30, 190,  23],
       [ 10, 101,  15]])
'''from sklearn.model_selection import GridSearchCV
parameters = [{'C':[15,10]}] #svm-2
grid_search = GridSearchCV(estimator = classifier1,
                           param_grid = parameters,
                           scoring = 'accuracy',
                           cv = 10,
                           n_jobs = -1)
grid_search = grid_search.fit(x, y)
best_accuracy = grid_search.best_score_
best_parameters = grid_search.best_params_
print('done')
print(best_accuracy)
best_parameters'''
226/486
y_pred2 = classifier1.predict(xtest)
y_pred2.shape
ID=df2['id']
ID.shape
pred=pd.DataFrame(y_pred2)
datasets=pd.concat([ID,pred], axis=1)

datasets.columns =['id', 'sentiment_class']
datasets.to_csv('asvm3.csv',index=False)
!cp asvm3.csv "drive/My Drive/"
print(pred[0].value_counts())
sns.countplot( x=-pred[0], data = pred)<jupyter_output><empty_output><jupyter_text>**   **<jupyter_code>from xgboost import XGBClassifier
classifier4=XGBClassifier(n_estimators=600,reg_lambda=0.1 ,max_depth=4,seed=42,learning_rate=0.1,tree_method='gpu_exact',colsample_bytree=0.9)
classifier4.fit(X_train, y_train)
from sklearn.model_selection import cross_val_score
acc=cross_val_score(estimator=classifier4, X=X_train, y=y_train , cv=10)

print(acc.mean())
print(acc.std())
acc
y_pred1 = classifier4.predict(X_val)
print(y_pred1.shape)

from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_val, y_pred1)
cm
array([[  9,  97,  11],
       [ 19, 192,  32],
       [ 17,  88,  21]])

y_pred2 = classifier4.predict(xtest)
y_pred2.shape
ID=df2['id']
ID.shape
pred=pd.DataFrame(y_pred2)
datasets=pd.concat([ID,pred], axis=1)

datasets.columns =['id', 'sentiment_class']
datasets.to_csv('axg23.csv',index=False)
!cp axg23.csv "drive/My Drive/"
print(pred[0].value_counts())
sns.countplot( x=pred[0], data = pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15976631953263906, 'fraction_numerical': 0.03200406400812802, 'mean_word_length': 5.950573698146513, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '36811', 'license': 'Apache-2.0', 'n_tokens_mistral': 3211, 'n_tokens_neox': 3123, 'n_words': 482, 'path': 'Other models/XGB.ipynb', 'repo_name': 'nikhilreddybilla28/Mothers--Day-with-Machine-Learning-HackerEarth-Machine-Learning-challenge-', 'size': '7874'}