<jupyter_start><jupyter_text>Lemmetization<jupyter_code>import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
nltk.download('wordnet')

# sentences = nltk.sent_tokenize(paragraph)

lemmatizer = WordNetLemmatizer()
# Lemmatization
for i in range(len(corpus)):
    words = nltk.word_tokenize(corpus[i])
    words = [lemmatizer.lemmatize(word) for word in words]
    corpus[i] = ' '.join(words)   
corpus[0]<jupyter_output><empty_output><jupyter_text>tfidf model build<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(max_features = 1000, min_df = 0.05, max_df = 0.95, stop_words = stopwords.words('english'))
X_br = vectorizer.fit_transform(corpus).toarray()

X_br.shape
def train_model(classifier, feature_vector_train, label, feature_vector_valid,  valid_y, is_neural_net=False):
    # fit the training dataset on the classifier
    classifier.fit(feature_vector_train, label)
    
    # predict the labels on validation dataset
    predictions = classifier.predict(feature_vector_valid)
    
    if is_neural_net:
        predictions = predictions.argmax(axis=-1)
    
    return metrics.accuracy_score(predictions, valid_y)<jupyter_output><empty_output><jupyter_text>Model Build<jupyter_code>### Logistic Model
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from sklearn.model_selection import train_test_split

train_x_lg,test_x_lg,train_y_lg,test_y_lg= train_test_split(X_br,
                                              y_br,
                                              test_size=.3,
                                              random_state=123)
#Logistic Regression
# Logistic Regression on Count Vectors and TF-IDF
accuracy_L1 = train_model(LogisticRegression(), train_x_lg, train_y_lg, test_x_lg, test_y_lg)
print("LR  for L1, Count Vectors: ", accuracy_L1)
logreg = LogisticRegression()
logreg.fit( train_x_lg, train_y_lg )
#Evalutaing model
#from sklearn import metrics

test_accuracy=metrics.accuracy_score(test_y_lg,logreg.predict(test_x_lg))
print('test_accuracy: ',test_accuracy)

train_accuracy=metrics.accuracy_score(train_y_lg,logreg.predict(train_x_lg))
print('train_accuracy: ',train_accuracy)
import matplotlib.pyplot as plt
import seaborn as sns
# Creating a confusion matrix
cm = metrics.confusion_matrix(test_y_lg,
                            logreg.predict(test_x_lg), [1,5] )
cm


sns.heatmap(cm, annot=True,  fmt='.2f', xticklabels = ["1", "5"] , yticklabels = ["1", "5"] )
plt.ylabel('True label')
plt.xlabel('Predicted label')
###Linear Svc
from sklearn import  svm

train_x_l_svc, test_x_l_svc, train_y_l_svc, test_y_l_svc= train_test_split(X_br,
                                              y_br,
                                              test_size=.3,
                                              random_state=123)
# Linear SVC on Count Vectors and TF-IDF
accuracy_L2 = train_model(svm.LinearSVC(), train_x_l_svc, train_y_l_svc, test_x_l_svc, test_y_l_svc)
print(accuracy_L2)
### Random Forest
from sklearn.ensemble import RandomForestClassifier
train_x_rf, test_x_rf, train_y_rf, test_y_rf= train_test_split(X_br,
                                              y_br,
                                              test_size=.3,
                                              random_state=123)


accuracy_L3 = train_model(RandomForestClassifier(), train_x_rf, train_y_rf, test_x_rf, test_y_rf)
print( accuracy_L3)
models = pd.DataFrame({
    'Model': ['Logistic Regression','Linear SVC','Random Forest'],
    
    'Score': [accuracy_L1,accuracy_L2, accuracy_L3]
    })

models.sort_values(by='Score', ascending=False)<jupyter_output><empty_output><jupyter_text>Final Prediction<jupyter_code>model= svm.LinearSVC().fit(train_x_l_svc, train_y_l_svc)

#Concatenating final prediction with original data set(all obersvations):

predicted=pd.DataFrame(model.predict(X_br))
  
testfile = pd.concat([df, predicted], axis=1)


testfile.rename(columns={0: 'Predicted Stars'},inplace=True)
errors= testfile.loc[testfile.Stars!= testfile['Predicted Stars']]
errors.count()
#Exporting testfile to csv for final submission

testfile.to_csv('Review_submission.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09694117647058824, 'fraction_numerical': 0.01011764705882353, 'mean_word_length': 3.739130434782609, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '425558', 'license': 'MIT', 'n_tokens_mistral': 1466, 'n_tokens_neox': 1431, 'n_words': 261, 'path': 'Bank Reviews Final v.02.ipynb', 'repo_name': 'HardeepSiinghh/Bank-Reviews-Complaints-Analysis', 'size': '4250'}