<jupyter_start><jupyter_text>Modelling- Create all features before splitting- Create Tf-Idf of the entire corpus- And then split it into test train split- Fit the SVR Model and check accuracy- Experiment with various created features<jupyter_code># Importing Necessary Modules
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import spacy
nlp=spacy.load('en_core_web_sm')
from sklearn.model_selection import train_test_split
from spacy.pipeline import SentenceSegmenter
import re
import warnings
warnings.filterwarnings('ignore')
import textblob
from textblob import TextBlob

# Customization of View
pd.options.display.max_columns = 999
# Directly loading with features csv created by Iphone7.ipynb
df= pd.read_csv("withfeatures.csv")
df.columns<jupyter_output><empty_output><jupyter_text>1. Handling Emoji's<jupyter_code>def remove_emoji(text):
    return text.encode('ascii','ignore').decode('ascii').strip()
df['Review_Text']=df.Review_Text.apply(remove_emoji)
df.Review_Text.head()<jupyter_output><empty_output><jupyter_text>2. Handling Punctuations<jupyter_code>def remove_punctuations(text):
    return re.sub('[^\w\s%,-.]',"",text).strip()
df.Review_Text=df.Review_Text.apply(remove_punctuations)
df.Review_Text.head()<jupyter_output><empty_output><jupyter_text>3. Spelling Correction with TextBlob<jupyter_code>def spell_correct(text):
    return str(TextBlob(text).correct())
df.Review_Text=df.Review_Text.apply(spell_correct)
df.Review_Text.head()<jupyter_output><empty_output><jupyter_text>4. Remove Stopwords and Applying Lemmatization of the word<jupyter_code>df.Review_Text=df.Review_Text.apply(lambda text: " ".join(token.lemma_ for token in nlp(text) 
                                           if not token.is_stop)) 
df.Review_Text.head()<jupyter_output><empty_output><jupyter_text>5.Creating Len After Feature<jupyter_code>df["Len_after"] = df.Review_Text.apply(lambda x: len(x.split()))
df.columns<jupyter_output><empty_output><jupyter_text>6. POS Tagging<jupyter_code>def pos(text):
    doc=nlp(text)
    return ' '.join([token.pos_ for token in doc])
df['POS']=df.Review_Text.apply(pos)
df.POS.sample(10)<jupyter_output><empty_output><jupyter_text>Removing Reviews which are not having alteast 2 words<jupyter_code>df=df[df.Len_after>=3]<jupyter_output><empty_output><jupyter_text>7. Percentage of Noun<jupyter_code>def Noun(text):
    text_len=len(text.split())
    noun_count=0
    for word in text.split():
        if word=='NOUN':
            noun_count+=1
    return np.round((noun_count/text_len)*100,2)
df['Perc_Noun']=df.POS.apply(Noun)
df.Perc_Noun.head()<jupyter_output><empty_output><jupyter_text>8. Percentage of Verb<jupyter_code>def Verb(text):
    text_len=len(text.split())
    verb_count=0
    for word in text.split():
        if word=='VERB':
            verb_count+=1
    return np.round((verb_count/text_len)*100,2)
df['Perc_Verb']=df.POS.apply(Verb)
df.Perc_Verb.head()<jupyter_output><empty_output><jupyter_text>9. Percentage of Adverb<jupyter_code>def Adverb(text):
    text_len=len(text.split())
    adv_count=0
    for word in text.split():
        if word=='ADV':
            adv_count+=1
    return np.round((adv_count/text_len)*100,2)
df['Perc_Adverb']=df.POS.apply(Adverb)
df.Perc_Adverb.sample(5)<jupyter_output><empty_output><jupyter_text>Percentage of Adjective<jupyter_code>def Adj(text):
    text_len=len(text.split())
    adj_count=0
    for word in text.split():
        if word=='ADJ':
            adj_count+=1
    return np.round((adj_count/text_len)*100,2)
df['Perc_Adj']=df.POS.apply(Adj)
df.Perc_Adj.head()
df.columns
df=df[['Sr_No', 'Review_Title', 'Review_Text', 'Review_Rating',
       'Review_Date', 'Upvote', 'Downvote', 'Sum_of_Up_Down',
       'Len_before', 'Num_Sentence', 'No_QMark', 'No_ExMark', 'No_Upper',
       'No_proper', 'Emoji_Count', 'Len_after', 'POS', 'Perc_Noun',
       'Perc_Verb', 'Perc_Adverb', 'Perc_Adj', 'h']]
df.head()
df.to_csv("Corrected_New_Features.csv")<jupyter_output><empty_output><jupyter_text>Creating Tf-Idf and Bag of words on the entire df<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer
tfidf= TfidfVectorizer(token_pattern='(?<KEY>')
Matrix=tfidf.fit_transform(df.Review_Text)
unigram=pd.DataFrame(Matrix.toarray(),columns=tfidf.get_feature_names())
feature=df.drop(["Review_Title","Review_Text","Review_Date","POS"],axis=1)
feature.head()
feature.reset_index(drop=True,inplace=True)
feature.head()
main=unigram.join(feature)
main.head()
main.to_csv("Unigram_Feature.csv")
main=main.fillna(value=0)
main.head()
X=main.iloc[:,:-1]
y=main.iloc[:,-1]
X_train_val, X_test, y_train_val, y_test=train_test_split(X,y,test_size=50,random_state=0)
X_train, X_val, y_train, y_val = train_test_split(X_train_val,y_train_val,test_size=50,random_state=0)
X_train.shape,X_val.shape,X_test.shape
y_train.shape,y_val.shape,y_test.shape<jupyter_output><empty_output><jupyter_text>Fitting Models<jupyter_code>from sklearn.svm import SVR
from sklearn.metrics import explained_variance_score,r2_score<jupyter_output><empty_output><jupyter_text>Using Review Rating, Len_after, Num of Sentences to predict the value of h<jupyter_code>predictors=["Review_Rating","Len_after","Num_Sentence"]
train1=X_train[predictors]
val1=X_val[predictors]
test1=X_test[predictors]
clf1=SVR(kernel='rbf').fit(train1,y_train)
ypred_val=clf1.predict(val1)
r2_score(y_val,ypred_val)
ypred_test=clf1.predict(test1)
r2_score(y_test,ypred_test)
ypred_test.argmin()
X_test.iloc[[36]]<jupyter_output><empty_output><jupyter_text>Unigrams,Review Rating,Len of review<jupyter_code>allcol=list(X_train.columns)
X_train['Downvote']
st=" ".join(list(unigram.columns))
list(feature.columns)[:-1]
nonpredictors=["Sr.No","Upvote","Downvote","Sum_of_Up_Down","Len_before","Num_Sentence","No_QMark","No_ExMark","No_Upper","No_proper","Emoji_Count",
              "Perc_Noun","Perc_Verb","Perc_Adverb","Perc_Adj"]
train2=X_train.drop(nonpredictors,axis=1)
val2=X_val.drop(nonpredictors,axis=1)
test2=X_test.drop(nonpredictors,axis=1)
clf2=SVR(kernel='rbf').fit(train2,y_train)
ypred_val2=clf2.predict(val2)
r2_score(y_val,ypred_val2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14117457244272347, 'fraction_numerical': 0.012907389480477573, 'mean_word_length': 7.343203230148048, 'pattern_counts': {'":': 0, '<': 66, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '281399', 'license': 'MIT', 'n_tokens_mistral': 2439, 'n_tokens_neox': 2387, 'n_words': 332, 'path': 'Experiments/Modelling-Iphone.ipynb', 'repo_name': 'veeravignesh1/Capstone-Reviews-Ranker', 'size': '6216'}